drivers/net/mlx5/mlx5_glue.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stdalign.h>
   8 #include <stddef.h>
   9 #include <stdint.h>
  10
  11 /*
  12  * Not needed by this file; included to work around the lack of off_t
  13  * definition for mlx5dv.h with unpatched rdma-core versions.
  14  */
  15 #include <sys/types.h>
  16
  17 /* Verbs headers do not support -pedantic. */
  18 #ifdef PEDANTIC
  19 #pragma GCC diagnostic ignored "-Wpedantic"
  20 #endif
  21 #include <infiniband/mlx5dv.h>
  22 #include <infiniband/verbs.h>
  23 #ifdef PEDANTIC
  24 #pragma GCC diagnostic error "-Wpedantic"
  25 #endif
  26
  27 #include <rte_config.h>
  28
  29 #include "mlx5_autoconf.h"
  30 #include "mlx5_glue.h"
  31
  32 static int
  33 mlx5_glue_fork_init(void)
  34 {
  35         return ibv_fork_init();
  36 }
  37
  38 static struct ibv_pd *
  39 mlx5_glue_alloc_pd(struct ibv_context *context)
  40 {
  41         return ibv_alloc_pd(context);
  42 }
  43
  44 static int
  45 mlx5_glue_dealloc_pd(struct ibv_pd *pd)
  46 {
  47         return ibv_dealloc_pd(pd);
  48 }
  49
  50 static struct ibv_device **
  51 mlx5_glue_get_device_list(int *num_devices)
  52 {
  53         return ibv_get_device_list(num_devices);
  54 }
  55
  56 static void
  57 mlx5_glue_free_device_list(struct ibv_device **list)
  58 {
  59         ibv_free_device_list(list);
  60 }
  61
  62 static struct ibv_context *
  63 mlx5_glue_open_device(struct ibv_device *device)
  64 {
  65         return ibv_open_device(device);
  66 }
  67
  68 static int
  69 mlx5_glue_close_device(struct ibv_context *context)
  70 {
  71         return ibv_close_device(context);
  72 }
  73
  74 static int
  75 mlx5_glue_query_device(struct ibv_context *context,
  76                        struct ibv_device_attr *device_attr)
  77 {
  78         return ibv_query_device(context, device_attr);
  79 }
  80
  81 static int
  82 mlx5_glue_query_device_ex(struct ibv_context *context,
  83                           const struct ibv_query_device_ex_input *input,
  84                           struct ibv_device_attr_ex *attr)
  85 {
  86         return ibv_query_device_ex(context, input, attr);
  87 }
  88
  89 static int
  90 mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num,
  91                      struct ibv_port_attr *port_attr)
  92 {
  93         return ibv_query_port(context, port_num, port_attr);
  94 }
  95
  96 static struct ibv_comp_channel *
  97 mlx5_glue_create_comp_channel(struct ibv_context *context)
  98 {
  99         return ibv_create_comp_channel(context);
 100 }
 101
 102 static int
 103 mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
 104 {
 105         return ibv_destroy_comp_channel(channel);
 106 }
 107
 108 static struct ibv_cq *
 109 mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,
 110                     struct ibv_comp_channel *channel, int comp_vector)
 111 {
 112         return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
 113 }
 114
 115 static int
 116 mlx5_glue_destroy_cq(struct ibv_cq *cq)
 117 {
 118         return ibv_destroy_cq(cq);
 119 }
 120
 121 static int
 122 mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
 123                        void **cq_context)
 124 {
 125         return ibv_get_cq_event(channel, cq, cq_context);
 126 }
 127
 128 static void
 129 mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
 130 {
 131         ibv_ack_cq_events(cq, nevents);
 132 }
 133
 134 static struct ibv_rwq_ind_table *
 135 mlx5_glue_create_rwq_ind_table(struct ibv_context *context,
 136                                struct ibv_rwq_ind_table_init_attr *init_attr)
 137 {
 138         return ibv_create_rwq_ind_table(context, init_attr);
 139 }
 140
 141 static int
 142 mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
 143 {
 144         return ibv_destroy_rwq_ind_table(rwq_ind_table);
 145 }
 146
 147 static struct ibv_wq *
 148 mlx5_glue_create_wq(struct ibv_context *context,
 149                     struct ibv_wq_init_attr *wq_init_attr)
 150 {
 151         return ibv_create_wq(context, wq_init_attr);
 152 }
 153
 154 static int
 155 mlx5_glue_destroy_wq(struct ibv_wq *wq)
 156 {
 157         return ibv_destroy_wq(wq);
 158 }
 159 static int
 160 mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)
 161 {
 162         return ibv_modify_wq(wq, wq_attr);
 163 }
 164
 165 static struct ibv_flow *
 166 mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)
 167 {
 168         return ibv_create_flow(qp, flow);
 169 }
 170
 171 static int
 172 mlx5_glue_destroy_flow(struct ibv_flow *flow_id)
 173 {
 174         return ibv_destroy_flow(flow_id);
 175 }
 176
 177 static int
 178 mlx5_glue_destroy_flow_action(struct ibv_flow_action *action)
 179 {
 180 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 181         return ibv_destroy_flow_action(action);
 182 #else
 183         (void)action;
 184         return ENOTSUP;
 185 #endif
 186 }
 187
 188 static struct ibv_qp *
 189 mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
 190 {
 191         return ibv_create_qp(pd, qp_init_attr);
 192 }
 193
 194 static struct ibv_qp *
 195 mlx5_glue_create_qp_ex(struct ibv_context *context,
 196                        struct ibv_qp_init_attr_ex *qp_init_attr_ex)
 197 {
 198         return ibv_create_qp_ex(context, qp_init_attr_ex);
 199 }
 200
 201 static int
 202 mlx5_glue_destroy_qp(struct ibv_qp *qp)
 203 {
 204         return ibv_destroy_qp(qp);
 205 }
 206
 207 static int
 208 mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
 209 {
 210         return ibv_modify_qp(qp, attr, attr_mask);
 211 }
 212
 213 static struct ibv_mr *
 214 mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)
 215 {
 216         return ibv_reg_mr(pd, addr, length, access);
 217 }
 218
 219 static int
 220 mlx5_glue_dereg_mr(struct ibv_mr *mr)
 221 {
 222         return ibv_dereg_mr(mr);
 223 }
 224
 225 static struct ibv_counter_set *
 226 mlx5_glue_create_counter_set(struct ibv_context *context,
 227                              struct ibv_counter_set_init_attr *init_attr)
 228 {
 229 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 230         (void)context;
 231         (void)init_attr;
 232         return NULL;
 233 #else
 234         return ibv_create_counter_set(context, init_attr);
 235 #endif
 236 }
 237
 238 static int
 239 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 240 {
 241 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 242         (void)cs;
 243         return ENOTSUP;
 244 #else
 245         return ibv_destroy_counter_set(cs);
 246 #endif
 247 }
 248
 249 static int
 250 mlx5_glue_describe_counter_set(struct ibv_context *context,
 251                                uint16_t counter_set_id,
 252                                struct ibv_counter_set_description *cs_desc)
 253 {
 254 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 255         (void)context;
 256         (void)counter_set_id;
 257         (void)cs_desc;
 258         return ENOTSUP;
 259 #else
 260         return ibv_describe_counter_set(context, counter_set_id, cs_desc);
 261 #endif
 262 }
 263
 264 static int
 265 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 266                             struct ibv_counter_set_data *cs_data)
 267 {
 268 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 269         (void)query_attr;
 270         (void)cs_data;
 271         return ENOTSUP;
 272 #else
 273         return ibv_query_counter_set(query_attr, cs_data);
 274 #endif
 275 }
 276
 277 static struct ibv_counters *
 278 mlx5_glue_create_counters(struct ibv_context *context,
 279                           struct ibv_counters_init_attr *init_attr)
 280 {
 281 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 282         (void)context;
 283         (void)init_attr;
 284         return NULL;
 285 #else
 286         return ibv_create_counters(context, init_attr);
 287 #endif
 288 }
 289
 290 static int
 291 mlx5_glue_destroy_counters(struct ibv_counters *counters)
 292 {
 293 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 294         (void)counters;
 295         return ENOTSUP;
 296 #else
 297         return ibv_destroy_counters(counters);
 298 #endif
 299 }
 300
 301 static int
 302 mlx5_glue_attach_counters(struct ibv_counters *counters,
 303                           struct ibv_counter_attach_attr *attr,
 304                           struct ibv_flow *flow)
 305 {
 306 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 307         (void)counters;
 308         (void)attr;
 309         (void)flow;
 310         return ENOTSUP;
 311 #else
 312         return ibv_attach_counters_point_flow(counters, attr, flow);
 313 #endif
 314 }
 315
 316 static int
 317 mlx5_glue_query_counters(struct ibv_counters *counters,
 318                          uint64_t *counters_value,
 319                          uint32_t ncounters,
 320                          uint32_t flags)
 321 {
 322 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 323         (void)counters;
 324         (void)counters_value;
 325         (void)ncounters;
 326         (void)flags;
 327         return ENOTSUP;
 328 #else
 329         return ibv_read_counters(counters, counters_value, ncounters, flags);
 330 #endif
 331 }
 332
 333 static void
 334 mlx5_glue_ack_async_event(struct ibv_async_event *event)
 335 {
 336         ibv_ack_async_event(event);
 337 }
 338
 339 static int
 340 mlx5_glue_get_async_event(struct ibv_context *context,
 341                           struct ibv_async_event *event)
 342 {
 343         return ibv_get_async_event(context, event);
 344 }
 345
 346 static const char *
 347 mlx5_glue_port_state_str(enum ibv_port_state port_state)
 348 {
 349         return ibv_port_state_str(port_state);
 350 }
 351
 352 static struct ibv_cq *
 353 mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq)
 354 {
 355         return ibv_cq_ex_to_cq(cq);
 356 }
 357
 358 static struct ibv_cq_ex *
 359 mlx5_glue_dv_create_cq(struct ibv_context *context,
 360                        struct ibv_cq_init_attr_ex *cq_attr,
 361                        struct mlx5dv_cq_init_attr *mlx5_cq_attr)
 362 {
 363         return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
 364 }
 365
 366 static struct ibv_wq *
 367 mlx5_glue_dv_create_wq(struct ibv_context *context,
 368                        struct ibv_wq_init_attr *wq_attr,
 369                        struct mlx5dv_wq_init_attr *mlx5_wq_attr)
 370 {
 371 #ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 372         (void)context;
 373         (void)wq_attr;
 374         (void)mlx5_wq_attr;
 375         return NULL;
 376 #else
 377         return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr);
 378 #endif
 379 }
 380
 381 static int
 382 mlx5_glue_dv_query_device(struct ibv_context *ctx,
 383                           struct mlx5dv_context *attrs_out)
 384 {
 385         return mlx5dv_query_device(ctx, attrs_out);
 386 }
 387
 388 static int
 389 mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx,
 390                               enum mlx5dv_set_ctx_attr_type type, void *attr)
 391 {
 392         return mlx5dv_set_context_attr(ibv_ctx, type, attr);
 393 }
 394
 395 static int
 396 mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
 397 {
 398         return mlx5dv_init_obj(obj, obj_type);
 399 }
 400
 401 static struct ibv_qp *
 402 mlx5_glue_dv_create_qp(struct ibv_context *context,
 403                        struct ibv_qp_init_attr_ex *qp_init_attr_ex,
 404                        struct mlx5dv_qp_init_attr *dv_qp_init_attr)
 405 {
 406 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 407         return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr);
 408 #else
 409         (void)context;
 410         (void)qp_init_attr_ex;
 411         (void)dv_qp_init_attr;
 412         return NULL;
 413 #endif
 414 }
 415
 416 static struct mlx5dv_flow_matcher *
 417 mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
 418                                  struct mlx5dv_flow_matcher_attr *matcher_attr)
 419 {
 420 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 421         return mlx5dv_create_flow_matcher(context, matcher_attr);
 422 #else
 423         (void)context;
 424         (void)matcher_attr;
 425         return NULL;
 426 #endif
 427 }
 428
 429 static struct ibv_flow *
 430 mlx5_glue_dv_create_flow(struct mlx5dv_flow_matcher *matcher,
 431                          struct mlx5dv_flow_match_parameters *match_value,
 432                          size_t num_actions,
 433                          struct mlx5dv_flow_action_attr *actions_attr)
 434 {
 435 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 436         return mlx5dv_create_flow(matcher, match_value,
 437                                   num_actions, actions_attr);
 438 #else
 439         (void)matcher;
 440         (void)match_value;
 441         (void)num_actions;
 442         (void)actions_attr;
 443         return NULL;
 444 #endif
 445 }
 446
 447 static int
 448 mlx5_glue_dv_destroy_flow_matcher(struct mlx5dv_flow_matcher *matcher)
 449 {
 450 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 451         return mlx5dv_destroy_flow_matcher(matcher);
 452 #else
 453         (void)matcher;
 454         return 0;
 455 #endif
 456 }
 457
 458 static struct ibv_flow_action *
 459 mlx5_glue_dv_create_flow_action_packet_reformat
 460                 (struct ibv_context *ctx,
 461                  size_t data_sz,
 462                  void *data,
 463                  enum mlx5dv_flow_action_packet_reformat_type reformat_type,
 464                  enum mlx5dv_flow_table_type ft_type)
 465 {
 466 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 467         return mlx5dv_create_flow_action_packet_reformat(ctx,
 468                                                          data_sz,
 469                                                          data,
 470                                                          reformat_type,
 471                                                          ft_type);
 472 #else
 473         (void)ctx;
 474         (void)data_sz;
 475         (void)data;
 476         (void)reformat_type;
 477         (void)ft_type;
 478         return NULL;
 479 #endif
 480 }
 481
 482 static struct ibv_flow_action *
 483 mlx5_glue_dv_create_flow_action_modify_header
 484                                         (struct ibv_context *ctx,
 485                                          size_t actions_sz,
 486                                          uint64_t actions[],
 487                                          enum mlx5dv_flow_table_type ft_type)
 488 {
 489 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 490         return mlx5dv_create_flow_action_modify_header(ctx, actions_sz,
 491                                                        actions, ft_type);
 492 #else
 493         (void)ctx;
 494         (void)actions_sz;
 495         (void)actions;
 496         (void)ft_type;
 497         return NULL;
 498 #endif
 499 }
 500
 501 static struct ibv_context *
 502 mlx5_glue_dv_open_device(struct ibv_device *device)
 503 {
 504 #ifdef HAVE_IBV_DEVX_OBJ
 505         return mlx5dv_open_device(device,
 506                                   &(struct mlx5dv_context_attr){
 507                                         .flags = MLX5DV_CONTEXT_FLAGS_DEVX,
 508                                   });
 509 #else
 510         (void)device;
 511         return NULL;
 512 #endif
 513 }
 514
 515 static struct mlx5dv_devx_obj *
 516 mlx5_glue_devx_obj_create(struct ibv_context *ctx,
 517                           const void *in, size_t inlen,
 518                           void *out, size_t outlen)
 519 {
 520 #ifdef HAVE_IBV_DEVX_OBJ
 521         return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen);
 522 #else
 523         (void)ctx;
 524         (void)in;
 525         (void)inlen;
 526         (void)out;
 527         (void)outlen;
 528         return NULL;
 529 #endif
 530 }
 531
 532 static int
 533 mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj)
 534 {
 535 #ifdef HAVE_IBV_DEVX_OBJ
 536         return mlx5dv_devx_obj_destroy(obj);
 537 #else
 538         (void)obj;
 539         return -ENOTSUP;
 540 #endif
 541 }
 542
 543 static int
 544 mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj,
 545                          const void *in, size_t inlen,
 546                          void *out, size_t outlen)
 547 {
 548 #ifdef HAVE_IBV_DEVX_OBJ
 549         return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen);
 550 #else
 551         (void)obj;
 552         (void)in;
 553         (void)inlen;
 554         (void)out;
 555         (void)outlen;
 556         return -ENOTSUP;
 557 #endif
 558 }
 559
 560 static int
 561 mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj,
 562                           const void *in, size_t inlen,
 563                           void *out, size_t outlen)
 564 {
 565 #ifdef HAVE_IBV_DEVX_OBJ
 566         return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen);
 567 #else
 568         (void)obj;
 569         (void)in;
 570         (void)inlen;
 571         (void)out;
 572         (void)outlen;
 573         return -ENOTSUP;
 574 #endif
 575 }
 576
 577 static int
 578 mlx5_glue_devx_general_cmd(struct ibv_context *ctx,
 579                            const void *in, size_t inlen,
 580                            void *out, size_t outlen)
 581 {
 582 #ifdef HAVE_IBV_DEVX_OBJ
 583         return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen);
 584 #else
 585         (void)ctx;
 586         (void)in;
 587         (void)inlen;
 588         (void)out;
 589         (void)outlen;
 590         return -ENOTSUP;
 591 #endif
 592 }
 593
 594 alignas(RTE_CACHE_LINE_SIZE)
 595 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
 596         .version = MLX5_GLUE_VERSION,
 597         .fork_init = mlx5_glue_fork_init,
 598         .alloc_pd = mlx5_glue_alloc_pd,
 599         .dealloc_pd = mlx5_glue_dealloc_pd,
 600         .get_device_list = mlx5_glue_get_device_list,
 601         .free_device_list = mlx5_glue_free_device_list,
 602         .open_device = mlx5_glue_open_device,
 603         .close_device = mlx5_glue_close_device,
 604         .query_device = mlx5_glue_query_device,
 605         .query_device_ex = mlx5_glue_query_device_ex,
 606         .query_port = mlx5_glue_query_port,
 607         .create_comp_channel = mlx5_glue_create_comp_channel,
 608         .destroy_comp_channel = mlx5_glue_destroy_comp_channel,
 609         .create_cq = mlx5_glue_create_cq,
 610         .destroy_cq = mlx5_glue_destroy_cq,
 611         .get_cq_event = mlx5_glue_get_cq_event,
 612         .ack_cq_events = mlx5_glue_ack_cq_events,
 613         .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table,
 614         .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table,
 615         .create_wq = mlx5_glue_create_wq,
 616         .destroy_wq = mlx5_glue_destroy_wq,
 617         .modify_wq = mlx5_glue_modify_wq,
 618         .create_flow = mlx5_glue_create_flow,
 619         .destroy_flow = mlx5_glue_destroy_flow,
 620         .destroy_flow_action = mlx5_glue_destroy_flow_action,
 621         .create_qp = mlx5_glue_create_qp,
 622         .create_qp_ex = mlx5_glue_create_qp_ex,
 623         .destroy_qp = mlx5_glue_destroy_qp,
 624         .modify_qp = mlx5_glue_modify_qp,
 625         .reg_mr = mlx5_glue_reg_mr,
 626         .dereg_mr = mlx5_glue_dereg_mr,
 627         .create_counter_set = mlx5_glue_create_counter_set,
 628         .destroy_counter_set = mlx5_glue_destroy_counter_set,
 629         .describe_counter_set = mlx5_glue_describe_counter_set,
 630         .query_counter_set = mlx5_glue_query_counter_set,
 631         .create_counters = mlx5_glue_create_counters,
 632         .destroy_counters = mlx5_glue_destroy_counters,
 633         .attach_counters = mlx5_glue_attach_counters,
 634         .query_counters = mlx5_glue_query_counters,
 635         .ack_async_event = mlx5_glue_ack_async_event,
 636         .get_async_event = mlx5_glue_get_async_event,
 637         .port_state_str = mlx5_glue_port_state_str,
 638         .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
 639         .dv_create_cq = mlx5_glue_dv_create_cq,
 640         .dv_create_wq = mlx5_glue_dv_create_wq,
 641         .dv_query_device = mlx5_glue_dv_query_device,
 642         .dv_set_context_attr = mlx5_glue_dv_set_context_attr,
 643         .dv_init_obj = mlx5_glue_dv_init_obj,
 644         .dv_create_qp = mlx5_glue_dv_create_qp,
 645         .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
 646         .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
 647         .dv_create_flow = mlx5_glue_dv_create_flow,
 648         .dv_create_flow_action_packet_reformat =
 649                         mlx5_glue_dv_create_flow_action_packet_reformat,
 650         .dv_create_flow_action_modify_header =
 651                         mlx5_glue_dv_create_flow_action_modify_header,
 652         .dv_open_device = mlx5_glue_dv_open_device,
 653         .devx_obj_create = mlx5_glue_devx_obj_create,
 654         .devx_obj_destroy = mlx5_glue_devx_obj_destroy,
 655         .devx_obj_query = mlx5_glue_devx_obj_query,
 656         .devx_obj_modify = mlx5_glue_devx_obj_modify,
 657         .devx_general_cmd = mlx5_glue_devx_general_cmd,
 658 };