drivers/common/mlx5/linux/mlx5_glue.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stdalign.h>
   8 #include <stddef.h>
   9 #include <stdint.h>
  10 #include <stdlib.h>
  11 /*
  12  * Not needed by this file; included to work around the lack of off_t
  13  * definition for mlx5dv.h with unpatched rdma-core versions.
  14  */
  15 #include <sys/types.h>
  16
  17 #include "mlx5_glue.h"
  18
  19 static int
  20 mlx5_glue_fork_init(void)
  21 {
  22         return ibv_fork_init();
  23 }
  24
  25 static struct ibv_pd *
  26 mlx5_glue_alloc_pd(struct ibv_context *context)
  27 {
  28         return ibv_alloc_pd(context);
  29 }
  30
  31 static int
  32 mlx5_glue_dealloc_pd(struct ibv_pd *pd)
  33 {
  34         return ibv_dealloc_pd(pd);
  35 }
  36
  37 static struct ibv_device **
  38 mlx5_glue_get_device_list(int *num_devices)
  39 {
  40         return ibv_get_device_list(num_devices);
  41 }
  42
  43 static void
  44 mlx5_glue_free_device_list(struct ibv_device **list)
  45 {
  46         ibv_free_device_list(list);
  47 }
  48
  49 static struct ibv_context *
  50 mlx5_glue_open_device(struct ibv_device *device)
  51 {
  52         return ibv_open_device(device);
  53 }
  54
  55 static int
  56 mlx5_glue_close_device(struct ibv_context *context)
  57 {
  58         return ibv_close_device(context);
  59 }
  60
  61 static int
  62 mlx5_glue_query_device(struct ibv_context *context,
  63                        struct ibv_device_attr *device_attr)
  64 {
  65         return ibv_query_device(context, device_attr);
  66 }
  67
  68 static int
  69 mlx5_glue_query_device_ex(struct ibv_context *context,
  70                           const struct ibv_query_device_ex_input *input,
  71                           struct ibv_device_attr_ex *attr)
  72 {
  73         return ibv_query_device_ex(context, input, attr);
  74 }
  75
  76 static int
  77 mlx5_glue_query_rt_values_ex(struct ibv_context *context,
  78                           struct ibv_values_ex *values)
  79 {
  80         return ibv_query_rt_values_ex(context, values);
  81 }
  82
  83 static int
  84 mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num,
  85                      struct ibv_port_attr *port_attr)
  86 {
  87         return ibv_query_port(context, port_num, port_attr);
  88 }
  89
  90 static struct ibv_comp_channel *
  91 mlx5_glue_create_comp_channel(struct ibv_context *context)
  92 {
  93         return ibv_create_comp_channel(context);
  94 }
  95
  96 static int
  97 mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
  98 {
  99         return ibv_destroy_comp_channel(channel);
 100 }
 101
 102 static struct ibv_cq *
 103 mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,
 104                     struct ibv_comp_channel *channel, int comp_vector)
 105 {
 106         return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
 107 }
 108
 109 static int
 110 mlx5_glue_destroy_cq(struct ibv_cq *cq)
 111 {
 112         return ibv_destroy_cq(cq);
 113 }
 114
 115 static int
 116 mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
 117                        void **cq_context)
 118 {
 119         return ibv_get_cq_event(channel, cq, cq_context);
 120 }
 121
 122 static void
 123 mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
 124 {
 125         ibv_ack_cq_events(cq, nevents);
 126 }
 127
 128 static struct ibv_rwq_ind_table *
 129 mlx5_glue_create_rwq_ind_table(struct ibv_context *context,
 130                                struct ibv_rwq_ind_table_init_attr *init_attr)
 131 {
 132         return ibv_create_rwq_ind_table(context, init_attr);
 133 }
 134
 135 static int
 136 mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
 137 {
 138         return ibv_destroy_rwq_ind_table(rwq_ind_table);
 139 }
 140
 141 static struct ibv_wq *
 142 mlx5_glue_create_wq(struct ibv_context *context,
 143                     struct ibv_wq_init_attr *wq_init_attr)
 144 {
 145         return ibv_create_wq(context, wq_init_attr);
 146 }
 147
 148 static int
 149 mlx5_glue_destroy_wq(struct ibv_wq *wq)
 150 {
 151         return ibv_destroy_wq(wq);
 152 }
 153 static int
 154 mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)
 155 {
 156         return ibv_modify_wq(wq, wq_attr);
 157 }
 158
 159 static struct ibv_flow *
 160 mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)
 161 {
 162         return ibv_create_flow(qp, flow);
 163 }
 164
 165 static int
 166 mlx5_glue_destroy_flow(struct ibv_flow *flow_id)
 167 {
 168         return ibv_destroy_flow(flow_id);
 169 }
 170
 171 static int
 172 mlx5_glue_destroy_flow_action(void *action)
 173 {
 174 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 175 #ifdef HAVE_MLX5DV_DR
 176         return mlx5dv_dr_action_destroy(action);
 177 #else
 178         struct mlx5dv_flow_action_attr *attr = action;
 179         int res = 0;
 180         switch (attr->type) {
 181         case MLX5DV_FLOW_ACTION_TAG:
 182                 break;
 183         default:
 184                 res = ibv_destroy_flow_action(attr->action);
 185                 break;
 186         }
 187         free(action);
 188         return res;
 189 #endif
 190 #else
 191         (void)action;
 192         return -ENOTSUP;
 193 #endif
 194 }
 195
 196 static struct ibv_qp *
 197 mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
 198 {
 199         return ibv_create_qp(pd, qp_init_attr);
 200 }
 201
 202 static struct ibv_qp *
 203 mlx5_glue_create_qp_ex(struct ibv_context *context,
 204                        struct ibv_qp_init_attr_ex *qp_init_attr_ex)
 205 {
 206         return ibv_create_qp_ex(context, qp_init_attr_ex);
 207 }
 208
 209 static int
 210 mlx5_glue_destroy_qp(struct ibv_qp *qp)
 211 {
 212         return ibv_destroy_qp(qp);
 213 }
 214
 215 static int
 216 mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
 217 {
 218         return ibv_modify_qp(qp, attr, attr_mask);
 219 }
 220
 221 static struct ibv_mr *
 222 mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)
 223 {
 224         return ibv_reg_mr(pd, addr, length, access);
 225 }
 226
 227 static struct ibv_mr *
 228 mlx5_glue_alloc_null_mr(struct ibv_pd *pd)
 229 {
 230 #ifdef HAVE_IBV_DEVX_OBJ
 231         return ibv_alloc_null_mr(pd);
 232 #else
 233         (void)pd;
 234         errno = ENOTSUP;
 235         return NULL;
 236 #endif
 237 }
 238
 239 static int
 240 mlx5_glue_dereg_mr(struct ibv_mr *mr)
 241 {
 242         return ibv_dereg_mr(mr);
 243 }
 244
 245 static struct ibv_counter_set *
 246 mlx5_glue_create_counter_set(struct ibv_context *context,
 247                              struct ibv_counter_set_init_attr *init_attr)
 248 {
 249 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 250         (void)context;
 251         (void)init_attr;
 252         return NULL;
 253 #else
 254         return ibv_create_counter_set(context, init_attr);
 255 #endif
 256 }
 257
 258 static int
 259 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 260 {
 261 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 262         (void)cs;
 263         return -ENOTSUP;
 264 #else
 265         return ibv_destroy_counter_set(cs);
 266 #endif
 267 }
 268
 269 static int
 270 mlx5_glue_describe_counter_set(struct ibv_context *context,
 271                                uint16_t counter_set_id,
 272                                struct ibv_counter_set_description *cs_desc)
 273 {
 274 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 275         (void)context;
 276         (void)counter_set_id;
 277         (void)cs_desc;
 278         return -ENOTSUP;
 279 #else
 280         return ibv_describe_counter_set(context, counter_set_id, cs_desc);
 281 #endif
 282 }
 283
 284 static int
 285 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 286                             struct ibv_counter_set_data *cs_data)
 287 {
 288 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 289         (void)query_attr;
 290         (void)cs_data;
 291         return -ENOTSUP;
 292 #else
 293         return ibv_query_counter_set(query_attr, cs_data);
 294 #endif
 295 }
 296
 297 static struct ibv_counters *
 298 mlx5_glue_create_counters(struct ibv_context *context,
 299                           struct ibv_counters_init_attr *init_attr)
 300 {
 301 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 302         (void)context;
 303         (void)init_attr;
 304         errno = ENOTSUP;
 305         return NULL;
 306 #else
 307         return ibv_create_counters(context, init_attr);
 308 #endif
 309 }
 310
 311 static int
 312 mlx5_glue_destroy_counters(struct ibv_counters *counters)
 313 {
 314 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 315         (void)counters;
 316         return -ENOTSUP;
 317 #else
 318         return ibv_destroy_counters(counters);
 319 #endif
 320 }
 321
 322 static int
 323 mlx5_glue_attach_counters(struct ibv_counters *counters,
 324                           struct ibv_counter_attach_attr *attr,
 325                           struct ibv_flow *flow)
 326 {
 327 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 328         (void)counters;
 329         (void)attr;
 330         (void)flow;
 331         return -ENOTSUP;
 332 #else
 333         return ibv_attach_counters_point_flow(counters, attr, flow);
 334 #endif
 335 }
 336
 337 static int
 338 mlx5_glue_query_counters(struct ibv_counters *counters,
 339                          uint64_t *counters_value,
 340                          uint32_t ncounters,
 341                          uint32_t flags)
 342 {
 343 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 344         (void)counters;
 345         (void)counters_value;
 346         (void)ncounters;
 347         (void)flags;
 348         return -ENOTSUP;
 349 #else
 350         return ibv_read_counters(counters, counters_value, ncounters, flags);
 351 #endif
 352 }
 353
 354 static void
 355 mlx5_glue_ack_async_event(struct ibv_async_event *event)
 356 {
 357         ibv_ack_async_event(event);
 358 }
 359
 360 static int
 361 mlx5_glue_get_async_event(struct ibv_context *context,
 362                           struct ibv_async_event *event)
 363 {
 364         return ibv_get_async_event(context, event);
 365 }
 366
 367 static const char *
 368 mlx5_glue_port_state_str(enum ibv_port_state port_state)
 369 {
 370         return ibv_port_state_str(port_state);
 371 }
 372
 373 static struct ibv_cq *
 374 mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq)
 375 {
 376         return ibv_cq_ex_to_cq(cq);
 377 }
 378
 379 static void *
 380 mlx5_glue_dr_create_flow_action_dest_flow_tbl(void *tbl)
 381 {
 382 #ifdef HAVE_MLX5DV_DR
 383         return mlx5dv_dr_action_create_dest_table(tbl);
 384 #else
 385         (void)tbl;
 386         errno = ENOTSUP;
 387         return NULL;
 388 #endif
 389 }
 390
 391 static void *
 392 mlx5_glue_dr_create_flow_action_dest_port(void *domain, uint32_t port)
 393 {
 394 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
 395         return mlx5dv_dr_action_create_dest_ib_port(domain, port);
 396 #else
 397 #ifdef HAVE_MLX5DV_DR_ESWITCH
 398         return mlx5dv_dr_action_create_dest_vport(domain, port);
 399 #else
 400         (void)domain;
 401         (void)port;
 402         errno = ENOTSUP;
 403         return NULL;
 404 #endif
 405 #endif
 406 }
 407
 408 static void *
 409 mlx5_glue_dr_create_flow_action_drop(void)
 410 {
 411 #ifdef HAVE_MLX5DV_DR_ESWITCH
 412         return mlx5dv_dr_action_create_drop();
 413 #else
 414         errno = ENOTSUP;
 415         return NULL;
 416 #endif
 417 }
 418
 419 static void *
 420 mlx5_glue_dr_create_flow_action_push_vlan(struct mlx5dv_dr_domain *domain,
 421                                           rte_be32_t vlan_tag)
 422 {
 423 #ifdef HAVE_MLX5DV_DR_VLAN
 424         return mlx5dv_dr_action_create_push_vlan(domain, vlan_tag);
 425 #else
 426         (void)domain;
 427         (void)vlan_tag;
 428         errno = ENOTSUP;
 429         return NULL;
 430 #endif
 431 }
 432
 433 static void *
 434 mlx5_glue_dr_create_flow_action_pop_vlan(void)
 435 {
 436 #ifdef HAVE_MLX5DV_DR_VLAN
 437         return mlx5dv_dr_action_create_pop_vlan();
 438 #else
 439         errno = ENOTSUP;
 440         return NULL;
 441 #endif
 442 }
 443
 444 static void *
 445 mlx5_glue_dr_create_flow_tbl(void *domain, uint32_t level)
 446 {
 447 #ifdef HAVE_MLX5DV_DR
 448         return mlx5dv_dr_table_create(domain, level);
 449 #else
 450         (void)domain;
 451         (void)level;
 452         errno = ENOTSUP;
 453         return NULL;
 454 #endif
 455 }
 456
 457 static int
 458 mlx5_glue_dr_destroy_flow_tbl(void *tbl)
 459 {
 460 #ifdef HAVE_MLX5DV_DR
 461         return mlx5dv_dr_table_destroy(tbl);
 462 #else
 463         (void)tbl;
 464         errno = ENOTSUP;
 465         return errno;
 466 #endif
 467 }
 468
 469 static void *
 470 mlx5_glue_dr_create_domain(struct ibv_context *ctx,
 471                            enum  mlx5dv_dr_domain_type domain)
 472 {
 473 #ifdef HAVE_MLX5DV_DR
 474         return mlx5dv_dr_domain_create(ctx, domain);
 475 #else
 476         (void)ctx;
 477         (void)domain;
 478         errno = ENOTSUP;
 479         return NULL;
 480 #endif
 481 }
 482
 483 static int
 484 mlx5_glue_dr_destroy_domain(void *domain)
 485 {
 486 #ifdef HAVE_MLX5DV_DR
 487         return mlx5dv_dr_domain_destroy(domain);
 488 #else
 489         (void)domain;
 490         errno = ENOTSUP;
 491         return errno;
 492 #endif
 493 }
 494
 495 static struct ibv_cq_ex *
 496 mlx5_glue_dv_create_cq(struct ibv_context *context,
 497                        struct ibv_cq_init_attr_ex *cq_attr,
 498                        struct mlx5dv_cq_init_attr *mlx5_cq_attr)
 499 {
 500         return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
 501 }
 502
 503 static struct ibv_wq *
 504 mlx5_glue_dv_create_wq(struct ibv_context *context,
 505                        struct ibv_wq_init_attr *wq_attr,
 506                        struct mlx5dv_wq_init_attr *mlx5_wq_attr)
 507 {
 508 #ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 509         (void)context;
 510         (void)wq_attr;
 511         (void)mlx5_wq_attr;
 512         errno = ENOTSUP;
 513         return NULL;
 514 #else
 515         return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr);
 516 #endif
 517 }
 518
 519 static int
 520 mlx5_glue_dv_query_device(struct ibv_context *ctx,
 521                           struct mlx5dv_context *attrs_out)
 522 {
 523         return mlx5dv_query_device(ctx, attrs_out);
 524 }
 525
 526 static int
 527 mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx,
 528                               enum mlx5dv_set_ctx_attr_type type, void *attr)
 529 {
 530         return mlx5dv_set_context_attr(ibv_ctx, type, attr);
 531 }
 532
 533 static int
 534 mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
 535 {
 536         return mlx5dv_init_obj(obj, obj_type);
 537 }
 538
 539 static struct ibv_qp *
 540 mlx5_glue_dv_create_qp(struct ibv_context *context,
 541                        struct ibv_qp_init_attr_ex *qp_init_attr_ex,
 542                        struct mlx5dv_qp_init_attr *dv_qp_init_attr)
 543 {
 544 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 545         return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr);
 546 #else
 547         (void)context;
 548         (void)qp_init_attr_ex;
 549         (void)dv_qp_init_attr;
 550         errno = ENOTSUP;
 551         return NULL;
 552 #endif
 553 }
 554
 555 static void *
 556 mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
 557                                  struct mlx5dv_flow_matcher_attr *matcher_attr,
 558                                  void *tbl)
 559 {
 560 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 561 #ifdef HAVE_MLX5DV_DR
 562         (void)context;
 563         return mlx5dv_dr_matcher_create(tbl, matcher_attr->priority,
 564                                         matcher_attr->match_criteria_enable,
 565                                         matcher_attr->match_mask);
 566 #else
 567         (void)tbl;
 568         return mlx5dv_create_flow_matcher(context, matcher_attr);
 569 #endif
 570 #else
 571         (void)context;
 572         (void)matcher_attr;
 573         (void)tbl;
 574         errno = ENOTSUP;
 575         return NULL;
 576 #endif
 577 }
 578
 579 static void *
 580 mlx5_glue_dv_create_flow(void *matcher,
 581                          void *match_value,
 582                          size_t num_actions,
 583                          void *actions[])
 584 {
 585 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 586 #ifdef HAVE_MLX5DV_DR
 587         return mlx5dv_dr_rule_create(matcher, match_value, num_actions,
 588                                      (struct mlx5dv_dr_action **)actions);
 589 #else
 590         struct mlx5dv_flow_action_attr actions_attr[8];
 591
 592         if (num_actions > 8)
 593                 return NULL;
 594         for (size_t i = 0; i < num_actions; i++)
 595                 actions_attr[i] =
 596                         *((struct mlx5dv_flow_action_attr *)(actions[i]));
 597         return mlx5dv_create_flow(matcher, match_value,
 598                                   num_actions, actions_attr);
 599 #endif
 600 #else
 601         (void)matcher;
 602         (void)match_value;
 603         (void)num_actions;
 604         (void)actions;
 605         return NULL;
 606 #endif
 607 }
 608
 609 static void *
 610 mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset)
 611 {
 612 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 613 #ifdef HAVE_MLX5DV_DR
 614         return mlx5dv_dr_action_create_flow_counter(counter_obj, offset);
 615 #else
 616         struct mlx5dv_flow_action_attr *action;
 617
 618         (void)offset;
 619         action = malloc(sizeof(*action));
 620         if (!action)
 621                 return NULL;
 622         action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
 623         action->obj = counter_obj;
 624         return action;
 625 #endif
 626 #else
 627         (void)counter_obj;
 628         (void)offset;
 629         errno = ENOTSUP;
 630         return NULL;
 631 #endif
 632 }
 633
 634 static void *
 635 mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp)
 636 {
 637 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 638 #ifdef HAVE_MLX5DV_DR
 639         return mlx5dv_dr_action_create_dest_ibv_qp(qp);
 640 #else
 641         struct mlx5dv_flow_action_attr *action;
 642
 643         action = malloc(sizeof(*action));
 644         if (!action)
 645                 return NULL;
 646         action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
 647         action->obj = qp;
 648         return action;
 649 #endif
 650 #else
 651         (void)qp;
 652         errno = ENOTSUP;
 653         return NULL;
 654 #endif
 655 }
 656
 657 static void *
 658 mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir)
 659 {
 660 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
 661         return mlx5dv_dr_action_create_dest_devx_tir(tir);
 662 #else
 663         (void)tir;
 664         errno = ENOTSUP;
 665         return NULL;
 666 #endif
 667 }
 668
 669 static void *
 670 mlx5_glue_dv_create_flow_action_modify_header
 671                                         (struct ibv_context *ctx,
 672                                          enum mlx5dv_flow_table_type ft_type,
 673                                          void *domain, uint64_t flags,
 674                                          size_t actions_sz,
 675                                          uint64_t actions[])
 676 {
 677 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 678 #ifdef HAVE_MLX5DV_DR
 679         (void)ctx;
 680         (void)ft_type;
 681         return mlx5dv_dr_action_create_modify_header(domain, flags, actions_sz,
 682                                                      (__be64 *)actions);
 683 #else
 684         struct mlx5dv_flow_action_attr *action;
 685
 686         (void)domain;
 687         (void)flags;
 688         action = malloc(sizeof(*action));
 689         if (!action)
 690                 return NULL;
 691         action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
 692         action->action = mlx5dv_create_flow_action_modify_header
 693                 (ctx, actions_sz, actions, ft_type);
 694         return action;
 695 #endif
 696 #else
 697         (void)ctx;
 698         (void)ft_type;
 699         (void)domain;
 700         (void)flags;
 701         (void)actions_sz;
 702         (void)actions;
 703         errno = ENOTSUP;
 704         return NULL;
 705 #endif
 706 }
 707
 708 static void *
 709 mlx5_glue_dv_create_flow_action_packet_reformat
 710                 (struct ibv_context *ctx,
 711                  enum mlx5dv_flow_action_packet_reformat_type reformat_type,
 712                  enum mlx5dv_flow_table_type ft_type,
 713                  struct mlx5dv_dr_domain *domain,
 714                  uint32_t flags, size_t data_sz, void *data)
 715 {
 716 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 717 #ifdef HAVE_MLX5DV_DR
 718         (void)ctx;
 719         (void)ft_type;
 720         return mlx5dv_dr_action_create_packet_reformat(domain, flags,
 721                                                        reformat_type, data_sz,
 722                                                        data);
 723 #else
 724         (void)domain;
 725         (void)flags;
 726         struct mlx5dv_flow_action_attr *action;
 727
 728         action = malloc(sizeof(*action));
 729         if (!action)
 730                 return NULL;
 731         action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
 732         action->action = mlx5dv_create_flow_action_packet_reformat
 733                 (ctx, data_sz, data, reformat_type, ft_type);
 734         return action;
 735 #endif
 736 #else
 737         (void)ctx;
 738         (void)reformat_type;
 739         (void)ft_type;
 740         (void)domain;
 741         (void)flags;
 742         (void)data_sz;
 743         (void)data;
 744         errno = ENOTSUP;
 745         return NULL;
 746 #endif
 747 }
 748
 749 static void *
 750 mlx5_glue_dv_create_flow_action_tag(uint32_t tag)
 751 {
 752 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 753 #ifdef HAVE_MLX5DV_DR
 754         return mlx5dv_dr_action_create_tag(tag);
 755 #else /* HAVE_MLX5DV_DR */
 756         struct mlx5dv_flow_action_attr *action;
 757         action = malloc(sizeof(*action));
 758         if (!action)
 759                 return NULL;
 760         action->type = MLX5DV_FLOW_ACTION_TAG;
 761         action->tag_value = tag;
 762         return action;
 763 #endif /* HAVE_MLX5DV_DR */
 764 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
 765         (void)tag;
 766         errno = ENOTSUP;
 767         return NULL;
 768 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 769 }
 770
 771 static void *
 772 mlx5_glue_dv_create_flow_action_meter(struct mlx5dv_dr_flow_meter_attr *attr)
 773 {
 774 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
 775         return mlx5dv_dr_action_create_flow_meter(attr);
 776 #else
 777         (void)attr;
 778         errno = ENOTSUP;
 779         return NULL;
 780 #endif
 781 }
 782
 783 static int
 784 mlx5_glue_dv_modify_flow_action_meter(void *action,
 785                                       struct mlx5dv_dr_flow_meter_attr *attr,
 786                                       uint64_t modify_bits)
 787 {
 788 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
 789         return mlx5dv_dr_action_modify_flow_meter(action, attr, modify_bits);
 790 #else
 791         (void)action;
 792         (void)attr;
 793         (void)modify_bits;
 794         errno = ENOTSUP;
 795         return errno;
 796 #endif
 797 }
 798
 799 static void *
 800 mlx5_glue_dr_create_flow_action_default_miss(void)
 801 {
 802 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_DEFAULT_MISS)
 803         return mlx5dv_dr_action_create_default_miss();
 804 #else
 805         errno = ENOTSUP;
 806         return NULL;
 807 #endif
 808 }
 809
 810 static int
 811 mlx5_glue_dv_destroy_flow(void *flow_id)
 812 {
 813 #ifdef HAVE_MLX5DV_DR
 814         return mlx5dv_dr_rule_destroy(flow_id);
 815 #else
 816         return ibv_destroy_flow(flow_id);
 817 #endif
 818 }
 819
 820 static int
 821 mlx5_glue_dv_destroy_flow_matcher(void *matcher)
 822 {
 823 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 824 #ifdef HAVE_MLX5DV_DR
 825         return mlx5dv_dr_matcher_destroy(matcher);
 826 #else
 827         return mlx5dv_destroy_flow_matcher(matcher);
 828 #endif
 829 #else
 830         (void)matcher;
 831         errno = ENOTSUP;
 832         return errno;
 833 #endif
 834 }
 835
 836 static struct ibv_context *
 837 mlx5_glue_dv_open_device(struct ibv_device *device)
 838 {
 839 #ifdef HAVE_IBV_DEVX_OBJ
 840         return mlx5dv_open_device(device,
 841                                   &(struct mlx5dv_context_attr){
 842                                         .flags = MLX5DV_CONTEXT_FLAGS_DEVX,
 843                                   });
 844 #else
 845         (void)device;
 846         errno = ENOTSUP;
 847         return NULL;
 848 #endif
 849 }
 850
 851 static struct mlx5dv_devx_obj *
 852 mlx5_glue_devx_obj_create(struct ibv_context *ctx,
 853                           const void *in, size_t inlen,
 854                           void *out, size_t outlen)
 855 {
 856 #ifdef HAVE_IBV_DEVX_OBJ
 857         return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen);
 858 #else
 859         (void)ctx;
 860         (void)in;
 861         (void)inlen;
 862         (void)out;
 863         (void)outlen;
 864         errno = ENOTSUP;
 865         return NULL;
 866 #endif
 867 }
 868
 869 static int
 870 mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj)
 871 {
 872 #ifdef HAVE_IBV_DEVX_OBJ
 873         return mlx5dv_devx_obj_destroy(obj);
 874 #else
 875         (void)obj;
 876         return -ENOTSUP;
 877 #endif
 878 }
 879
 880 static int
 881 mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj,
 882                          const void *in, size_t inlen,
 883                          void *out, size_t outlen)
 884 {
 885 #ifdef HAVE_IBV_DEVX_OBJ
 886         return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen);
 887 #else
 888         (void)obj;
 889         (void)in;
 890         (void)inlen;
 891         (void)out;
 892         (void)outlen;
 893         return -ENOTSUP;
 894 #endif
 895 }
 896
 897 static int
 898 mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj,
 899                           const void *in, size_t inlen,
 900                           void *out, size_t outlen)
 901 {
 902 #ifdef HAVE_IBV_DEVX_OBJ
 903         return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen);
 904 #else
 905         (void)obj;
 906         (void)in;
 907         (void)inlen;
 908         (void)out;
 909         (void)outlen;
 910         return -ENOTSUP;
 911 #endif
 912 }
 913
 914 static int
 915 mlx5_glue_devx_general_cmd(struct ibv_context *ctx,
 916                            const void *in, size_t inlen,
 917                            void *out, size_t outlen)
 918 {
 919 #ifdef HAVE_IBV_DEVX_OBJ
 920         return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen);
 921 #else
 922         (void)ctx;
 923         (void)in;
 924         (void)inlen;
 925         (void)out;
 926         (void)outlen;
 927         return -ENOTSUP;
 928 #endif
 929 }
 930
 931 static struct mlx5dv_devx_cmd_comp *
 932 mlx5_glue_devx_create_cmd_comp(struct ibv_context *ctx)
 933 {
 934 #ifdef HAVE_IBV_DEVX_ASYNC
 935         return mlx5dv_devx_create_cmd_comp(ctx);
 936 #else
 937         (void)ctx;
 938         errno = -ENOTSUP;
 939         return NULL;
 940 #endif
 941 }
 942
 943 static void
 944 mlx5_glue_devx_destroy_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp)
 945 {
 946 #ifdef HAVE_IBV_DEVX_ASYNC
 947         mlx5dv_devx_destroy_cmd_comp(cmd_comp);
 948 #else
 949         (void)cmd_comp;
 950         errno = -ENOTSUP;
 951 #endif
 952 }
 953
 954 static int
 955 mlx5_glue_devx_obj_query_async(struct mlx5dv_devx_obj *obj, const void *in,
 956                                size_t inlen, size_t outlen, uint64_t wr_id,
 957                                struct mlx5dv_devx_cmd_comp *cmd_comp)
 958 {
 959 #ifdef HAVE_IBV_DEVX_ASYNC
 960         return mlx5dv_devx_obj_query_async(obj, in, inlen, outlen, wr_id,
 961                                            cmd_comp);
 962 #else
 963         (void)obj;
 964         (void)in;
 965         (void)inlen;
 966         (void)outlen;
 967         (void)wr_id;
 968         (void)cmd_comp;
 969         return -ENOTSUP;
 970 #endif
 971 }
 972
 973 static int
 974 mlx5_glue_devx_get_async_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp,
 975                                   struct mlx5dv_devx_async_cmd_hdr *cmd_resp,
 976                                   size_t cmd_resp_len)
 977 {
 978 #ifdef HAVE_IBV_DEVX_ASYNC
 979         return mlx5dv_devx_get_async_cmd_comp(cmd_comp, cmd_resp,
 980                                               cmd_resp_len);
 981 #else
 982         (void)cmd_comp;
 983         (void)cmd_resp;
 984         (void)cmd_resp_len;
 985         return -ENOTSUP;
 986 #endif
 987 }
 988
 989 static struct mlx5dv_devx_umem *
 990 mlx5_glue_devx_umem_reg(struct ibv_context *context, void *addr, size_t size,
 991                         uint32_t access)
 992 {
 993 #ifdef HAVE_IBV_DEVX_OBJ
 994         return mlx5dv_devx_umem_reg(context, addr, size, access);
 995 #else
 996         (void)context;
 997         (void)addr;
 998         (void)size;
 999         (void)access;
1000         errno = -ENOTSUP;
1001         return NULL;
1002 #endif
1003 }
1004
1005 static int
1006 mlx5_glue_devx_umem_dereg(struct mlx5dv_devx_umem *dv_devx_umem)
1007 {
1008 #ifdef HAVE_IBV_DEVX_OBJ
1009         return mlx5dv_devx_umem_dereg(dv_devx_umem);
1010 #else
1011         (void)dv_devx_umem;
1012         return -ENOTSUP;
1013 #endif
1014 }
1015
1016 static int
1017 mlx5_glue_devx_qp_query(struct ibv_qp *qp,
1018                         const void *in, size_t inlen,
1019                         void *out, size_t outlen)
1020 {
1021 #ifdef HAVE_IBV_DEVX_QP
1022         return mlx5dv_devx_qp_query(qp, in, inlen, out, outlen);
1023 #else
1024         (void)qp;
1025         (void)in;
1026         (void)inlen;
1027         (void)out;
1028         (void)outlen;
1029         errno = ENOTSUP;
1030         return errno;
1031 #endif
1032 }
1033
1034 static int
1035 mlx5_glue_devx_port_query(struct ibv_context *ctx,
1036                           uint32_t port_num,
1037                           struct mlx5dv_devx_port *mlx5_devx_port)
1038 {
1039 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
1040         return mlx5dv_query_devx_port(ctx, port_num, mlx5_devx_port);
1041 #else
1042         (void)ctx;
1043         (void)port_num;
1044         (void)mlx5_devx_port;
1045         errno = ENOTSUP;
1046         return errno;
1047 #endif
1048 }
1049
1050 static int
1051 mlx5_glue_dr_dump_domain(FILE *file, void *domain)
1052 {
1053 #ifdef HAVE_MLX5_DR_FLOW_DUMP
1054         return mlx5dv_dump_dr_domain(file, domain);
1055 #else
1056         RTE_SET_USED(file);
1057         RTE_SET_USED(domain);
1058         return -ENOTSUP;
1059 #endif
1060 }
1061
1062 static int
1063 mlx5_glue_devx_query_eqn(struct ibv_context *ctx, uint32_t cpus,
1064                          uint32_t *eqn)
1065 {
1066 #ifdef HAVE_IBV_DEVX_OBJ
1067         return mlx5dv_devx_query_eqn(ctx, cpus, eqn);
1068 #else
1069         (void)ctx;
1070         (void)cpus;
1071         (void)eqn;
1072         return -ENOTSUP;
1073 #endif
1074 }
1075
1076 static struct mlx5dv_devx_event_channel *
1077 mlx5_glue_devx_create_event_channel(struct ibv_context *ctx, int flags)
1078 {
1079 #ifdef HAVE_IBV_DEVX_EVENT
1080         return mlx5dv_devx_create_event_channel(ctx, flags);
1081 #else
1082         (void)ctx;
1083         (void)flags;
1084         errno = ENOTSUP;
1085         return NULL;
1086 #endif
1087 }
1088
1089 static void
1090 mlx5_glue_devx_destroy_event_channel(struct mlx5dv_devx_event_channel *eventc)
1091 {
1092 #ifdef HAVE_IBV_DEVX_EVENT
1093         mlx5dv_devx_destroy_event_channel(eventc);
1094 #else
1095         (void)eventc;
1096 #endif
1097 }
1098
1099 static int
1100 mlx5_glue_devx_subscribe_devx_event(struct mlx5dv_devx_event_channel *eventc,
1101                                     struct mlx5dv_devx_obj *obj,
1102                                     uint16_t events_sz, uint16_t events_num[],
1103                                     uint64_t cookie)
1104 {
1105 #ifdef HAVE_IBV_DEVX_EVENT
1106         return mlx5dv_devx_subscribe_devx_event(eventc, obj, events_sz,
1107                                                 events_num, cookie);
1108 #else
1109         (void)eventc;
1110         (void)obj;
1111         (void)events_sz;
1112         (void)events_num;
1113         (void)cookie;
1114         return -ENOTSUP;
1115 #endif
1116 }
1117
1118 static int
1119 mlx5_glue_devx_subscribe_devx_event_fd(struct mlx5dv_devx_event_channel *eventc,
1120                                        int fd, struct mlx5dv_devx_obj *obj,
1121                                        uint16_t event_num)
1122 {
1123 #ifdef HAVE_IBV_DEVX_EVENT
1124         return mlx5dv_devx_subscribe_devx_event_fd(eventc, fd, obj, event_num);
1125 #else
1126         (void)eventc;
1127         (void)fd;
1128         (void)obj;
1129         (void)event_num;
1130         return -ENOTSUP;
1131 #endif
1132 }
1133
1134 static ssize_t
1135 mlx5_glue_devx_get_event(struct mlx5dv_devx_event_channel *eventc,
1136                          struct mlx5dv_devx_async_event_hdr *event_data,
1137                          size_t event_resp_len)
1138 {
1139 #ifdef HAVE_IBV_DEVX_EVENT
1140         return mlx5dv_devx_get_event(eventc, event_data, event_resp_len);
1141 #else
1142         (void)eventc;
1143         (void)event_data;
1144         (void)event_resp_len;
1145         errno = ENOTSUP;
1146         return -1;
1147 #endif
1148 }
1149
1150 static struct mlx5dv_devx_uar *
1151 mlx5_glue_devx_alloc_uar(struct ibv_context *context, uint32_t flags)
1152 {
1153 #ifdef HAVE_IBV_DEVX_OBJ
1154         return mlx5dv_devx_alloc_uar(context, flags);
1155 #else
1156         (void)context;
1157         (void)flags;
1158         errno = ENOTSUP;
1159         return NULL;
1160 #endif
1161 }
1162
1163 static void
1164 mlx5_glue_devx_free_uar(struct mlx5dv_devx_uar *devx_uar)
1165 {
1166 #ifdef HAVE_IBV_DEVX_OBJ
1167         mlx5dv_devx_free_uar(devx_uar);
1168 #else
1169         (void)devx_uar;
1170 #endif
1171 }
1172
1173 static struct mlx5dv_var *
1174 mlx5_glue_dv_alloc_var(struct ibv_context *context, uint32_t flags)
1175 {
1176 #ifdef HAVE_IBV_VAR
1177         return mlx5dv_alloc_var(context, flags);
1178 #else
1179         (void)context;
1180         (void)flags;
1181         errno = ENOTSUP;
1182         return NULL;
1183 #endif
1184 }
1185
1186 static void
1187 mlx5_glue_dv_free_var(struct mlx5dv_var *var)
1188 {
1189 #ifdef HAVE_IBV_VAR
1190         mlx5dv_free_var(var);
1191 #else
1192         (void)var;
1193         errno = ENOTSUP;
1194 #endif
1195 }
1196
1197
1198 static void
1199 mlx5_glue_dr_reclaim_domain_memory(void *domain, uint32_t enable)
1200 {
1201 #ifdef HAVE_MLX5DV_DR_MEM_RECLAIM
1202         mlx5dv_dr_domain_set_reclaim_device_memory(domain, enable);
1203 #else
1204         (void)(enable);
1205         (void)(domain);
1206 #endif
1207 }
1208
1209 __rte_cache_aligned
1210 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue) {
1211         .version = MLX5_GLUE_VERSION,
1212         .fork_init = mlx5_glue_fork_init,
1213         .alloc_pd = mlx5_glue_alloc_pd,
1214         .dealloc_pd = mlx5_glue_dealloc_pd,
1215         .get_device_list = mlx5_glue_get_device_list,
1216         .free_device_list = mlx5_glue_free_device_list,
1217         .open_device = mlx5_glue_open_device,
1218         .close_device = mlx5_glue_close_device,
1219         .query_device = mlx5_glue_query_device,
1220         .query_device_ex = mlx5_glue_query_device_ex,
1221         .query_rt_values_ex = mlx5_glue_query_rt_values_ex,
1222         .query_port = mlx5_glue_query_port,
1223         .create_comp_channel = mlx5_glue_create_comp_channel,
1224         .destroy_comp_channel = mlx5_glue_destroy_comp_channel,
1225         .create_cq = mlx5_glue_create_cq,
1226         .destroy_cq = mlx5_glue_destroy_cq,
1227         .get_cq_event = mlx5_glue_get_cq_event,
1228         .ack_cq_events = mlx5_glue_ack_cq_events,
1229         .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table,
1230         .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table,
1231         .create_wq = mlx5_glue_create_wq,
1232         .destroy_wq = mlx5_glue_destroy_wq,
1233         .modify_wq = mlx5_glue_modify_wq,
1234         .create_flow = mlx5_glue_create_flow,
1235         .destroy_flow = mlx5_glue_destroy_flow,
1236         .destroy_flow_action = mlx5_glue_destroy_flow_action,
1237         .create_qp = mlx5_glue_create_qp,
1238         .create_qp_ex = mlx5_glue_create_qp_ex,
1239         .destroy_qp = mlx5_glue_destroy_qp,
1240         .modify_qp = mlx5_glue_modify_qp,
1241         .reg_mr = mlx5_glue_reg_mr,
1242         .alloc_null_mr = mlx5_glue_alloc_null_mr,
1243         .dereg_mr = mlx5_glue_dereg_mr,
1244         .create_counter_set = mlx5_glue_create_counter_set,
1245         .destroy_counter_set = mlx5_glue_destroy_counter_set,
1246         .describe_counter_set = mlx5_glue_describe_counter_set,
1247         .query_counter_set = mlx5_glue_query_counter_set,
1248         .create_counters = mlx5_glue_create_counters,
1249         .destroy_counters = mlx5_glue_destroy_counters,
1250         .attach_counters = mlx5_glue_attach_counters,
1251         .query_counters = mlx5_glue_query_counters,
1252         .ack_async_event = mlx5_glue_ack_async_event,
1253         .get_async_event = mlx5_glue_get_async_event,
1254         .port_state_str = mlx5_glue_port_state_str,
1255         .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
1256         .dr_create_flow_action_dest_flow_tbl =
1257                 mlx5_glue_dr_create_flow_action_dest_flow_tbl,
1258         .dr_create_flow_action_dest_port =
1259                 mlx5_glue_dr_create_flow_action_dest_port,
1260         .dr_create_flow_action_drop =
1261                 mlx5_glue_dr_create_flow_action_drop,
1262         .dr_create_flow_action_push_vlan =
1263                 mlx5_glue_dr_create_flow_action_push_vlan,
1264         .dr_create_flow_action_pop_vlan =
1265                 mlx5_glue_dr_create_flow_action_pop_vlan,
1266         .dr_create_flow_tbl = mlx5_glue_dr_create_flow_tbl,
1267         .dr_destroy_flow_tbl = mlx5_glue_dr_destroy_flow_tbl,
1268         .dr_create_domain = mlx5_glue_dr_create_domain,
1269         .dr_destroy_domain = mlx5_glue_dr_destroy_domain,
1270         .dv_create_cq = mlx5_glue_dv_create_cq,
1271         .dv_create_wq = mlx5_glue_dv_create_wq,
1272         .dv_query_device = mlx5_glue_dv_query_device,
1273         .dv_set_context_attr = mlx5_glue_dv_set_context_attr,
1274         .dv_init_obj = mlx5_glue_dv_init_obj,
1275         .dv_create_qp = mlx5_glue_dv_create_qp,
1276         .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
1277         .dv_create_flow = mlx5_glue_dv_create_flow,
1278         .dv_create_flow_action_counter =
1279                 mlx5_glue_dv_create_flow_action_counter,
1280         .dv_create_flow_action_dest_ibv_qp =
1281                 mlx5_glue_dv_create_flow_action_dest_ibv_qp,
1282         .dv_create_flow_action_dest_devx_tir =
1283                 mlx5_glue_dv_create_flow_action_dest_devx_tir,
1284         .dv_create_flow_action_modify_header =
1285                 mlx5_glue_dv_create_flow_action_modify_header,
1286         .dv_create_flow_action_packet_reformat =
1287                 mlx5_glue_dv_create_flow_action_packet_reformat,
1288         .dv_create_flow_action_tag =  mlx5_glue_dv_create_flow_action_tag,
1289         .dv_create_flow_action_meter = mlx5_glue_dv_create_flow_action_meter,
1290         .dv_modify_flow_action_meter = mlx5_glue_dv_modify_flow_action_meter,
1291         .dr_create_flow_action_default_miss =
1292                 mlx5_glue_dr_create_flow_action_default_miss,
1293         .dv_destroy_flow = mlx5_glue_dv_destroy_flow,
1294         .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
1295         .dv_open_device = mlx5_glue_dv_open_device,
1296         .devx_obj_create = mlx5_glue_devx_obj_create,
1297         .devx_obj_destroy = mlx5_glue_devx_obj_destroy,
1298         .devx_obj_query = mlx5_glue_devx_obj_query,
1299         .devx_obj_modify = mlx5_glue_devx_obj_modify,
1300         .devx_general_cmd = mlx5_glue_devx_general_cmd,
1301         .devx_create_cmd_comp = mlx5_glue_devx_create_cmd_comp,
1302         .devx_destroy_cmd_comp = mlx5_glue_devx_destroy_cmd_comp,
1303         .devx_obj_query_async = mlx5_glue_devx_obj_query_async,
1304         .devx_get_async_cmd_comp = mlx5_glue_devx_get_async_cmd_comp,
1305         .devx_umem_reg = mlx5_glue_devx_umem_reg,
1306         .devx_umem_dereg = mlx5_glue_devx_umem_dereg,
1307         .devx_qp_query = mlx5_glue_devx_qp_query,
1308         .devx_port_query = mlx5_glue_devx_port_query,
1309         .dr_dump_domain = mlx5_glue_dr_dump_domain,
1310         .dr_reclaim_domain_memory = mlx5_glue_dr_reclaim_domain_memory,
1311         .devx_query_eqn = mlx5_glue_devx_query_eqn,
1312         .devx_create_event_channel = mlx5_glue_devx_create_event_channel,
1313         .devx_destroy_event_channel = mlx5_glue_devx_destroy_event_channel,
1314         .devx_subscribe_devx_event = mlx5_glue_devx_subscribe_devx_event,
1315         .devx_subscribe_devx_event_fd = mlx5_glue_devx_subscribe_devx_event_fd,
1316         .devx_get_event = mlx5_glue_devx_get_event,
1317         .devx_alloc_uar = mlx5_glue_devx_alloc_uar,
1318         .devx_free_uar = mlx5_glue_devx_free_uar,
1319         .dv_alloc_var = mlx5_glue_dv_alloc_var,
1320         .dv_free_var = mlx5_glue_dv_free_var,
1321 };