drivers/net/mlx5/mlx5_glue.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stdalign.h>
   8 #include <stddef.h>
   9 #include <stdint.h>
  10
  11 /*
  12  * Not needed by this file; included to work around the lack of off_t
  13  * definition for mlx5dv.h with unpatched rdma-core versions.
  14  */
  15 #include <sys/types.h>
  16
  17 /* Verbs headers do not support -pedantic. */
  18 #ifdef PEDANTIC
  19 #pragma GCC diagnostic ignored "-Wpedantic"
  20 #endif
  21 #include <infiniband/mlx5dv.h>
  22 #include <infiniband/verbs.h>
  23 #ifdef PEDANTIC
  24 #pragma GCC diagnostic error "-Wpedantic"
  25 #endif
  26
  27 #include <rte_config.h>
  28
  29 #include "mlx5_autoconf.h"
  30 #include "mlx5_glue.h"
  31
  32 static int
  33 mlx5_glue_fork_init(void)
  34 {
  35         return ibv_fork_init();
  36 }
  37
  38 static struct ibv_pd *
  39 mlx5_glue_alloc_pd(struct ibv_context *context)
  40 {
  41         return ibv_alloc_pd(context);
  42 }
  43
  44 static int
  45 mlx5_glue_dealloc_pd(struct ibv_pd *pd)
  46 {
  47         return ibv_dealloc_pd(pd);
  48 }
  49
  50 static struct ibv_device **
  51 mlx5_glue_get_device_list(int *num_devices)
  52 {
  53         return ibv_get_device_list(num_devices);
  54 }
  55
  56 static void
  57 mlx5_glue_free_device_list(struct ibv_device **list)
  58 {
  59         ibv_free_device_list(list);
  60 }
  61
  62 static struct ibv_context *
  63 mlx5_glue_open_device(struct ibv_device *device)
  64 {
  65         return ibv_open_device(device);
  66 }
  67
  68 static int
  69 mlx5_glue_close_device(struct ibv_context *context)
  70 {
  71         return ibv_close_device(context);
  72 }
  73
  74 static int
  75 mlx5_glue_query_device(struct ibv_context *context,
  76                        struct ibv_device_attr *device_attr)
  77 {
  78         return ibv_query_device(context, device_attr);
  79 }
  80
  81 static int
  82 mlx5_glue_query_device_ex(struct ibv_context *context,
  83                           const struct ibv_query_device_ex_input *input,
  84                           struct ibv_device_attr_ex *attr)
  85 {
  86         return ibv_query_device_ex(context, input, attr);
  87 }
  88
  89 static int
  90 mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num,
  91                      struct ibv_port_attr *port_attr)
  92 {
  93         return ibv_query_port(context, port_num, port_attr);
  94 }
  95
  96 static struct ibv_comp_channel *
  97 mlx5_glue_create_comp_channel(struct ibv_context *context)
  98 {
  99         return ibv_create_comp_channel(context);
 100 }
 101
 102 static int
 103 mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
 104 {
 105         return ibv_destroy_comp_channel(channel);
 106 }
 107
 108 static struct ibv_cq *
 109 mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,
 110                     struct ibv_comp_channel *channel, int comp_vector)
 111 {
 112         return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
 113 }
 114
 115 static int
 116 mlx5_glue_destroy_cq(struct ibv_cq *cq)
 117 {
 118         return ibv_destroy_cq(cq);
 119 }
 120
 121 static int
 122 mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
 123                        void **cq_context)
 124 {
 125         return ibv_get_cq_event(channel, cq, cq_context);
 126 }
 127
 128 static void
 129 mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
 130 {
 131         ibv_ack_cq_events(cq, nevents);
 132 }
 133
 134 static struct ibv_rwq_ind_table *
 135 mlx5_glue_create_rwq_ind_table(struct ibv_context *context,
 136                                struct ibv_rwq_ind_table_init_attr *init_attr)
 137 {
 138         return ibv_create_rwq_ind_table(context, init_attr);
 139 }
 140
 141 static int
 142 mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
 143 {
 144         return ibv_destroy_rwq_ind_table(rwq_ind_table);
 145 }
 146
 147 static struct ibv_wq *
 148 mlx5_glue_create_wq(struct ibv_context *context,
 149                     struct ibv_wq_init_attr *wq_init_attr)
 150 {
 151         return ibv_create_wq(context, wq_init_attr);
 152 }
 153
 154 static int
 155 mlx5_glue_destroy_wq(struct ibv_wq *wq)
 156 {
 157         return ibv_destroy_wq(wq);
 158 }
 159 static int
 160 mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)
 161 {
 162         return ibv_modify_wq(wq, wq_attr);
 163 }
 164
 165 static struct ibv_flow *
 166 mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)
 167 {
 168         return ibv_create_flow(qp, flow);
 169 }
 170
 171 static int
 172 mlx5_glue_destroy_flow(struct ibv_flow *flow_id)
 173 {
 174         return ibv_destroy_flow(flow_id);
 175 }
 176
 177 static int
 178 mlx5_glue_destroy_flow_action(void *action)
 179 {
 180 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 181         struct mlx5dv_flow_action_attr *attr = action;
 182         int res = 0;
 183         switch (attr->type) {
 184         case MLX5DV_FLOW_ACTION_TAG:
 185                 break;
 186         default:
 187                 res = ibv_destroy_flow_action(attr->action);
 188                 break;
 189         }
 190         free(action);
 191         return res;
 192 #else
 193         (void)action;
 194         return ENOTSUP;
 195 #endif
 196 }
 197
 198 static struct ibv_qp *
 199 mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
 200 {
 201         return ibv_create_qp(pd, qp_init_attr);
 202 }
 203
 204 static struct ibv_qp *
 205 mlx5_glue_create_qp_ex(struct ibv_context *context,
 206                        struct ibv_qp_init_attr_ex *qp_init_attr_ex)
 207 {
 208         return ibv_create_qp_ex(context, qp_init_attr_ex);
 209 }
 210
 211 static int
 212 mlx5_glue_destroy_qp(struct ibv_qp *qp)
 213 {
 214         return ibv_destroy_qp(qp);
 215 }
 216
 217 static int
 218 mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
 219 {
 220         return ibv_modify_qp(qp, attr, attr_mask);
 221 }
 222
 223 static struct ibv_mr *
 224 mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)
 225 {
 226         return ibv_reg_mr(pd, addr, length, access);
 227 }
 228
 229 static int
 230 mlx5_glue_dereg_mr(struct ibv_mr *mr)
 231 {
 232         return ibv_dereg_mr(mr);
 233 }
 234
 235 static struct ibv_counter_set *
 236 mlx5_glue_create_counter_set(struct ibv_context *context,
 237                              struct ibv_counter_set_init_attr *init_attr)
 238 {
 239 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 240         (void)context;
 241         (void)init_attr;
 242         return NULL;
 243 #else
 244         return ibv_create_counter_set(context, init_attr);
 245 #endif
 246 }
 247
 248 static int
 249 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 250 {
 251 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 252         (void)cs;
 253         return ENOTSUP;
 254 #else
 255         return ibv_destroy_counter_set(cs);
 256 #endif
 257 }
 258
 259 static int
 260 mlx5_glue_describe_counter_set(struct ibv_context *context,
 261                                uint16_t counter_set_id,
 262                                struct ibv_counter_set_description *cs_desc)
 263 {
 264 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 265         (void)context;
 266         (void)counter_set_id;
 267         (void)cs_desc;
 268         return ENOTSUP;
 269 #else
 270         return ibv_describe_counter_set(context, counter_set_id, cs_desc);
 271 #endif
 272 }
 273
 274 static int
 275 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 276                             struct ibv_counter_set_data *cs_data)
 277 {
 278 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 279         (void)query_attr;
 280         (void)cs_data;
 281         return ENOTSUP;
 282 #else
 283         return ibv_query_counter_set(query_attr, cs_data);
 284 #endif
 285 }
 286
 287 static struct ibv_counters *
 288 mlx5_glue_create_counters(struct ibv_context *context,
 289                           struct ibv_counters_init_attr *init_attr)
 290 {
 291 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 292         (void)context;
 293         (void)init_attr;
 294         return NULL;
 295 #else
 296         return ibv_create_counters(context, init_attr);
 297 #endif
 298 }
 299
 300 static int
 301 mlx5_glue_destroy_counters(struct ibv_counters *counters)
 302 {
 303 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 304         (void)counters;
 305         return ENOTSUP;
 306 #else
 307         return ibv_destroy_counters(counters);
 308 #endif
 309 }
 310
 311 static int
 312 mlx5_glue_attach_counters(struct ibv_counters *counters,
 313                           struct ibv_counter_attach_attr *attr,
 314                           struct ibv_flow *flow)
 315 {
 316 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 317         (void)counters;
 318         (void)attr;
 319         (void)flow;
 320         return ENOTSUP;
 321 #else
 322         return ibv_attach_counters_point_flow(counters, attr, flow);
 323 #endif
 324 }
 325
 326 static int
 327 mlx5_glue_query_counters(struct ibv_counters *counters,
 328                          uint64_t *counters_value,
 329                          uint32_t ncounters,
 330                          uint32_t flags)
 331 {
 332 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 333         (void)counters;
 334         (void)counters_value;
 335         (void)ncounters;
 336         (void)flags;
 337         return ENOTSUP;
 338 #else
 339         return ibv_read_counters(counters, counters_value, ncounters, flags);
 340 #endif
 341 }
 342
 343 static void
 344 mlx5_glue_ack_async_event(struct ibv_async_event *event)
 345 {
 346         ibv_ack_async_event(event);
 347 }
 348
 349 static int
 350 mlx5_glue_get_async_event(struct ibv_context *context,
 351                           struct ibv_async_event *event)
 352 {
 353         return ibv_get_async_event(context, event);
 354 }
 355
 356 static const char *
 357 mlx5_glue_port_state_str(enum ibv_port_state port_state)
 358 {
 359         return ibv_port_state_str(port_state);
 360 }
 361
 362 static struct ibv_cq *
 363 mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq)
 364 {
 365         return ibv_cq_ex_to_cq(cq);
 366 }
 367
 368 static struct ibv_cq_ex *
 369 mlx5_glue_dv_create_cq(struct ibv_context *context,
 370                        struct ibv_cq_init_attr_ex *cq_attr,
 371                        struct mlx5dv_cq_init_attr *mlx5_cq_attr)
 372 {
 373         return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
 374 }
 375
 376 static struct ibv_wq *
 377 mlx5_glue_dv_create_wq(struct ibv_context *context,
 378                        struct ibv_wq_init_attr *wq_attr,
 379                        struct mlx5dv_wq_init_attr *mlx5_wq_attr)
 380 {
 381 #ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 382         (void)context;
 383         (void)wq_attr;
 384         (void)mlx5_wq_attr;
 385         return NULL;
 386 #else
 387         return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr);
 388 #endif
 389 }
 390
 391 static int
 392 mlx5_glue_dv_query_device(struct ibv_context *ctx,
 393                           struct mlx5dv_context *attrs_out)
 394 {
 395         return mlx5dv_query_device(ctx, attrs_out);
 396 }
 397
 398 static int
 399 mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx,
 400                               enum mlx5dv_set_ctx_attr_type type, void *attr)
 401 {
 402         return mlx5dv_set_context_attr(ibv_ctx, type, attr);
 403 }
 404
 405 static int
 406 mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
 407 {
 408         return mlx5dv_init_obj(obj, obj_type);
 409 }
 410
 411 static struct ibv_qp *
 412 mlx5_glue_dv_create_qp(struct ibv_context *context,
 413                        struct ibv_qp_init_attr_ex *qp_init_attr_ex,
 414                        struct mlx5dv_qp_init_attr *dv_qp_init_attr)
 415 {
 416 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 417         return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr);
 418 #else
 419         (void)context;
 420         (void)qp_init_attr_ex;
 421         (void)dv_qp_init_attr;
 422         return NULL;
 423 #endif
 424 }
 425
 426 static struct mlx5dv_flow_matcher *
 427 mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
 428                                  struct mlx5dv_flow_matcher_attr *matcher_attr)
 429 {
 430 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 431         return mlx5dv_create_flow_matcher(context, matcher_attr);
 432 #else
 433         (void)context;
 434         (void)matcher_attr;
 435         return NULL;
 436 #endif
 437 }
 438
 439 static struct ibv_flow *
 440 mlx5_glue_dv_create_flow(struct mlx5dv_flow_matcher *matcher,
 441                          struct mlx5dv_flow_match_parameters *match_value,
 442                          size_t num_actions,
 443                          void *actions[])
 444 {
 445 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 446         struct mlx5dv_flow_action_attr actions_attr[8];
 447
 448         if (num_actions > 8)
 449                 return NULL;
 450         for (size_t i = 0; i < num_actions; i++)
 451                 actions_attr[i] =
 452                         *((struct mlx5dv_flow_action_attr *)(actions[i]));
 453         return mlx5dv_create_flow(matcher, match_value,
 454                                   num_actions, actions_attr);
 455 #else
 456         (void)matcher;
 457         (void)match_value;
 458         (void)num_actions;
 459         (void)actions;
 460         return NULL;
 461 #endif
 462 }
 463
 464 static int
 465 mlx5_glue_dv_destroy_flow_matcher(struct mlx5dv_flow_matcher *matcher)
 466 {
 467 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 468         return mlx5dv_destroy_flow_matcher(matcher);
 469 #else
 470         (void)matcher;
 471         return 0;
 472 #endif
 473 }
 474
 475 static void *
 476 mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset)
 477 {
 478 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 479         struct mlx5dv_flow_action_attr *action;
 480
 481         (void)offset;
 482         action = malloc(sizeof(*action));
 483         if (!action)
 484                 return NULL;
 485         action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
 486         action->obj = counter_obj;
 487         return action;
 488 #else
 489         (void)counter_obj;
 490         (void)offset;
 491         return NULL;
 492 #endif
 493 }
 494
 495 static void *
 496 mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp)
 497 {
 498 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 499         struct mlx5dv_flow_action_attr *action;
 500
 501         action = malloc(sizeof(*action));
 502         if (!action)
 503                 return NULL;
 504         action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
 505         action->obj = qp;
 506         return action;
 507 #else
 508         (void)qp;
 509         return NULL;
 510 #endif
 511 }
 512
 513 static void *
 514 mlx5_glue_dv_create_flow_action_modify_header
 515                                         (struct ibv_context *ctx,
 516                                          size_t actions_sz,
 517                                          uint64_t actions[],
 518                                          enum mlx5dv_flow_table_type ft_type)
 519 {
 520 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 521         struct mlx5dv_flow_action_attr *action;
 522
 523         action = malloc(sizeof(*action));
 524         if (!action)
 525                 return NULL;
 526         action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
 527         action->action = mlx5dv_create_flow_action_modify_header
 528                 (ctx, actions_sz, actions, ft_type);
 529         return action;
 530 #else
 531         (void)ctx;
 532         (void)actions_sz;
 533         (void)actions;
 534         (void)ft_type;
 535         return NULL;
 536 #endif
 537 }
 538
 539 static void *
 540 mlx5_glue_dv_create_flow_action_packet_reformat
 541                 (struct ibv_context *ctx,
 542                  size_t data_sz,
 543                  void *data,
 544                  enum mlx5dv_flow_action_packet_reformat_type reformat_type,
 545                  enum mlx5dv_flow_table_type ft_type)
 546 {
 547 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 548         struct mlx5dv_flow_action_attr *action;
 549
 550         action = malloc(sizeof(*action));
 551         if (!action)
 552                 return NULL;
 553         action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
 554         action->action = mlx5dv_create_flow_action_packet_reformat
 555                 (ctx, data_sz, data, reformat_type, ft_type);
 556         return action;
 557 #else
 558         (void)ctx;
 559         (void)data_sz;
 560         (void)data;
 561         (void)reformat_type;
 562         (void)ft_type;
 563         return NULL;
 564 #endif
 565 }
 566
 567 static void *
 568 mlx5_glue_dv_create_flow_action_tag(uint32_t tag)
 569 {
 570 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 571         struct mlx5dv_flow_action_attr *action;
 572         action = malloc(sizeof(*action));
 573         if (!action)
 574                 return NULL;
 575         action->type = MLX5DV_FLOW_ACTION_TAG;
 576         action->tag_value = tag;
 577         return action;
 578 #endif
 579         (void)tag;
 580         return NULL;
 581 }
 582
 583 static struct ibv_context *
 584 mlx5_glue_dv_open_device(struct ibv_device *device)
 585 {
 586 #ifdef HAVE_IBV_DEVX_OBJ
 587         return mlx5dv_open_device(device,
 588                                   &(struct mlx5dv_context_attr){
 589                                         .flags = MLX5DV_CONTEXT_FLAGS_DEVX,
 590                                   });
 591 #else
 592         (void)device;
 593         return NULL;
 594 #endif
 595 }
 596
 597 static struct mlx5dv_devx_obj *
 598 mlx5_glue_devx_obj_create(struct ibv_context *ctx,
 599                           const void *in, size_t inlen,
 600                           void *out, size_t outlen)
 601 {
 602 #ifdef HAVE_IBV_DEVX_OBJ
 603         return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen);
 604 #else
 605         (void)ctx;
 606         (void)in;
 607         (void)inlen;
 608         (void)out;
 609         (void)outlen;
 610         return NULL;
 611 #endif
 612 }
 613
 614 static int
 615 mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj)
 616 {
 617 #ifdef HAVE_IBV_DEVX_OBJ
 618         return mlx5dv_devx_obj_destroy(obj);
 619 #else
 620         (void)obj;
 621         return -ENOTSUP;
 622 #endif
 623 }
 624
 625 static int
 626 mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj,
 627                          const void *in, size_t inlen,
 628                          void *out, size_t outlen)
 629 {
 630 #ifdef HAVE_IBV_DEVX_OBJ
 631         return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen);
 632 #else
 633         (void)obj;
 634         (void)in;
 635         (void)inlen;
 636         (void)out;
 637         (void)outlen;
 638         return -ENOTSUP;
 639 #endif
 640 }
 641
 642 static int
 643 mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj,
 644                           const void *in, size_t inlen,
 645                           void *out, size_t outlen)
 646 {
 647 #ifdef HAVE_IBV_DEVX_OBJ
 648         return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen);
 649 #else
 650         (void)obj;
 651         (void)in;
 652         (void)inlen;
 653         (void)out;
 654         (void)outlen;
 655         return -ENOTSUP;
 656 #endif
 657 }
 658
 659 static int
 660 mlx5_glue_devx_general_cmd(struct ibv_context *ctx,
 661                            const void *in, size_t inlen,
 662                            void *out, size_t outlen)
 663 {
 664 #ifdef HAVE_IBV_DEVX_OBJ
 665         return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen);
 666 #else
 667         (void)ctx;
 668         (void)in;
 669         (void)inlen;
 670         (void)out;
 671         (void)outlen;
 672         return -ENOTSUP;
 673 #endif
 674 }
 675
 676 alignas(RTE_CACHE_LINE_SIZE)
 677 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
 678         .version = MLX5_GLUE_VERSION,
 679         .fork_init = mlx5_glue_fork_init,
 680         .alloc_pd = mlx5_glue_alloc_pd,
 681         .dealloc_pd = mlx5_glue_dealloc_pd,
 682         .get_device_list = mlx5_glue_get_device_list,
 683         .free_device_list = mlx5_glue_free_device_list,
 684         .open_device = mlx5_glue_open_device,
 685         .close_device = mlx5_glue_close_device,
 686         .query_device = mlx5_glue_query_device,
 687         .query_device_ex = mlx5_glue_query_device_ex,
 688         .query_port = mlx5_glue_query_port,
 689         .create_comp_channel = mlx5_glue_create_comp_channel,
 690         .destroy_comp_channel = mlx5_glue_destroy_comp_channel,
 691         .create_cq = mlx5_glue_create_cq,
 692         .destroy_cq = mlx5_glue_destroy_cq,
 693         .get_cq_event = mlx5_glue_get_cq_event,
 694         .ack_cq_events = mlx5_glue_ack_cq_events,
 695         .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table,
 696         .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table,
 697         .create_wq = mlx5_glue_create_wq,
 698         .destroy_wq = mlx5_glue_destroy_wq,
 699         .modify_wq = mlx5_glue_modify_wq,
 700         .create_flow = mlx5_glue_create_flow,
 701         .destroy_flow = mlx5_glue_destroy_flow,
 702         .destroy_flow_action = mlx5_glue_destroy_flow_action,
 703         .create_qp = mlx5_glue_create_qp,
 704         .create_qp_ex = mlx5_glue_create_qp_ex,
 705         .destroy_qp = mlx5_glue_destroy_qp,
 706         .modify_qp = mlx5_glue_modify_qp,
 707         .reg_mr = mlx5_glue_reg_mr,
 708         .dereg_mr = mlx5_glue_dereg_mr,
 709         .create_counter_set = mlx5_glue_create_counter_set,
 710         .destroy_counter_set = mlx5_glue_destroy_counter_set,
 711         .describe_counter_set = mlx5_glue_describe_counter_set,
 712         .query_counter_set = mlx5_glue_query_counter_set,
 713         .create_counters = mlx5_glue_create_counters,
 714         .destroy_counters = mlx5_glue_destroy_counters,
 715         .attach_counters = mlx5_glue_attach_counters,
 716         .query_counters = mlx5_glue_query_counters,
 717         .ack_async_event = mlx5_glue_ack_async_event,
 718         .get_async_event = mlx5_glue_get_async_event,
 719         .port_state_str = mlx5_glue_port_state_str,
 720         .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
 721         .dv_create_cq = mlx5_glue_dv_create_cq,
 722         .dv_create_wq = mlx5_glue_dv_create_wq,
 723         .dv_query_device = mlx5_glue_dv_query_device,
 724         .dv_set_context_attr = mlx5_glue_dv_set_context_attr,
 725         .dv_init_obj = mlx5_glue_dv_init_obj,
 726         .dv_create_qp = mlx5_glue_dv_create_qp,
 727         .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
 728         .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
 729         .dv_create_flow = mlx5_glue_dv_create_flow,
 730         .dv_create_flow_action_counter =
 731                 mlx5_glue_dv_create_flow_action_counter,
 732         .dv_create_flow_action_dest_ibv_qp =
 733                 mlx5_glue_dv_create_flow_action_dest_ibv_qp,
 734         .dv_create_flow_action_modify_header =
 735                 mlx5_glue_dv_create_flow_action_modify_header,
 736         .dv_create_flow_action_packet_reformat =
 737                 mlx5_glue_dv_create_flow_action_packet_reformat,
 738         .dv_create_flow_action_tag =  mlx5_glue_dv_create_flow_action_tag,
 739         .dv_open_device = mlx5_glue_dv_open_device,
 740         .devx_obj_create = mlx5_glue_devx_obj_create,
 741         .devx_obj_destroy = mlx5_glue_devx_obj_destroy,
 742         .devx_obj_query = mlx5_glue_devx_obj_query,
 743         .devx_obj_modify = mlx5_glue_devx_obj_modify,
 744         .devx_general_cmd = mlx5_glue_devx_general_cmd,
 745 };