drivers/common/mlx5/mlx5_glue.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stdalign.h>
   8 #include <stddef.h>
   9 #include <stdint.h>
  10 #include <stdlib.h>
  11 /*
  12  * Not needed by this file; included to work around the lack of off_t
  13  * definition for mlx5dv.h with unpatched rdma-core versions.
  14  */
  15 #include <sys/types.h>
  16
  17 #include "mlx5_glue.h"
  18
  19 static int
  20 mlx5_glue_fork_init(void)
  21 {
  22         return ibv_fork_init();
  23 }
  24
  25 static struct ibv_pd *
  26 mlx5_glue_alloc_pd(struct ibv_context *context)
  27 {
  28         return ibv_alloc_pd(context);
  29 }
  30
  31 static int
  32 mlx5_glue_dealloc_pd(struct ibv_pd *pd)
  33 {
  34         return ibv_dealloc_pd(pd);
  35 }
  36
  37 static struct ibv_device **
  38 mlx5_glue_get_device_list(int *num_devices)
  39 {
  40         return ibv_get_device_list(num_devices);
  41 }
  42
  43 static void
  44 mlx5_glue_free_device_list(struct ibv_device **list)
  45 {
  46         ibv_free_device_list(list);
  47 }
  48
  49 static struct ibv_context *
  50 mlx5_glue_open_device(struct ibv_device *device)
  51 {
  52         return ibv_open_device(device);
  53 }
  54
  55 static int
  56 mlx5_glue_close_device(struct ibv_context *context)
  57 {
  58         return ibv_close_device(context);
  59 }
  60
  61 static int
  62 mlx5_glue_query_device(struct ibv_context *context,
  63                        struct ibv_device_attr *device_attr)
  64 {
  65         return ibv_query_device(context, device_attr);
  66 }
  67
  68 static int
  69 mlx5_glue_query_device_ex(struct ibv_context *context,
  70                           const struct ibv_query_device_ex_input *input,
  71                           struct ibv_device_attr_ex *attr)
  72 {
  73         return ibv_query_device_ex(context, input, attr);
  74 }
  75
  76 static int
  77 mlx5_glue_query_rt_values_ex(struct ibv_context *context,
  78                           struct ibv_values_ex *values)
  79 {
  80         return ibv_query_rt_values_ex(context, values);
  81 }
  82
  83 static int
  84 mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num,
  85                      struct ibv_port_attr *port_attr)
  86 {
  87         return ibv_query_port(context, port_num, port_attr);
  88 }
  89
  90 static struct ibv_comp_channel *
  91 mlx5_glue_create_comp_channel(struct ibv_context *context)
  92 {
  93         return ibv_create_comp_channel(context);
  94 }
  95
  96 static int
  97 mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
  98 {
  99         return ibv_destroy_comp_channel(channel);
 100 }
 101
 102 static struct ibv_cq *
 103 mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,
 104                     struct ibv_comp_channel *channel, int comp_vector)
 105 {
 106         return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
 107 }
 108
 109 static int
 110 mlx5_glue_destroy_cq(struct ibv_cq *cq)
 111 {
 112         return ibv_destroy_cq(cq);
 113 }
 114
 115 static int
 116 mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
 117                        void **cq_context)
 118 {
 119         return ibv_get_cq_event(channel, cq, cq_context);
 120 }
 121
 122 static void
 123 mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
 124 {
 125         ibv_ack_cq_events(cq, nevents);
 126 }
 127
 128 static struct ibv_rwq_ind_table *
 129 mlx5_glue_create_rwq_ind_table(struct ibv_context *context,
 130                                struct ibv_rwq_ind_table_init_attr *init_attr)
 131 {
 132         return ibv_create_rwq_ind_table(context, init_attr);
 133 }
 134
 135 static int
 136 mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
 137 {
 138         return ibv_destroy_rwq_ind_table(rwq_ind_table);
 139 }
 140
 141 static struct ibv_wq *
 142 mlx5_glue_create_wq(struct ibv_context *context,
 143                     struct ibv_wq_init_attr *wq_init_attr)
 144 {
 145         return ibv_create_wq(context, wq_init_attr);
 146 }
 147
 148 static int
 149 mlx5_glue_destroy_wq(struct ibv_wq *wq)
 150 {
 151         return ibv_destroy_wq(wq);
 152 }
 153 static int
 154 mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)
 155 {
 156         return ibv_modify_wq(wq, wq_attr);
 157 }
 158
 159 static struct ibv_flow *
 160 mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)
 161 {
 162         return ibv_create_flow(qp, flow);
 163 }
 164
 165 static int
 166 mlx5_glue_destroy_flow(struct ibv_flow *flow_id)
 167 {
 168         return ibv_destroy_flow(flow_id);
 169 }
 170
 171 static int
 172 mlx5_glue_destroy_flow_action(void *action)
 173 {
 174 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 175 #ifdef HAVE_MLX5DV_DR
 176         return mlx5dv_dr_action_destroy(action);
 177 #else
 178         struct mlx5dv_flow_action_attr *attr = action;
 179         int res = 0;
 180         switch (attr->type) {
 181         case MLX5DV_FLOW_ACTION_TAG:
 182                 break;
 183         default:
 184                 res = ibv_destroy_flow_action(attr->action);
 185                 break;
 186         }
 187         free(action);
 188         return res;
 189 #endif
 190 #else
 191         (void)action;
 192         return ENOTSUP;
 193 #endif
 194 }
 195
 196 static struct ibv_qp *
 197 mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
 198 {
 199         return ibv_create_qp(pd, qp_init_attr);
 200 }
 201
 202 static struct ibv_qp *
 203 mlx5_glue_create_qp_ex(struct ibv_context *context,
 204                        struct ibv_qp_init_attr_ex *qp_init_attr_ex)
 205 {
 206         return ibv_create_qp_ex(context, qp_init_attr_ex);
 207 }
 208
 209 static int
 210 mlx5_glue_destroy_qp(struct ibv_qp *qp)
 211 {
 212         return ibv_destroy_qp(qp);
 213 }
 214
 215 static int
 216 mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
 217 {
 218         return ibv_modify_qp(qp, attr, attr_mask);
 219 }
 220
 221 static struct ibv_mr *
 222 mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)
 223 {
 224         return ibv_reg_mr(pd, addr, length, access);
 225 }
 226
 227 static struct ibv_mr *
 228 mlx5_glue_alloc_null_mr(struct ibv_pd *pd)
 229 {
 230 #ifdef HAVE_IBV_DEVX_OBJ
 231         return ibv_alloc_null_mr(pd);
 232 #else
 233         (void)pd;
 234         errno = ENOTSUP;
 235         return NULL;
 236 #endif
 237 }
 238
 239 static int
 240 mlx5_glue_dereg_mr(struct ibv_mr *mr)
 241 {
 242         return ibv_dereg_mr(mr);
 243 }
 244
 245 static struct ibv_counter_set *
 246 mlx5_glue_create_counter_set(struct ibv_context *context,
 247                              struct ibv_counter_set_init_attr *init_attr)
 248 {
 249 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 250         (void)context;
 251         (void)init_attr;
 252         return NULL;
 253 #else
 254         return ibv_create_counter_set(context, init_attr);
 255 #endif
 256 }
 257
 258 static int
 259 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 260 {
 261 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 262         (void)cs;
 263         return ENOTSUP;
 264 #else
 265         return ibv_destroy_counter_set(cs);
 266 #endif
 267 }
 268
 269 static int
 270 mlx5_glue_describe_counter_set(struct ibv_context *context,
 271                                uint16_t counter_set_id,
 272                                struct ibv_counter_set_description *cs_desc)
 273 {
 274 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 275         (void)context;
 276         (void)counter_set_id;
 277         (void)cs_desc;
 278         return ENOTSUP;
 279 #else
 280         return ibv_describe_counter_set(context, counter_set_id, cs_desc);
 281 #endif
 282 }
 283
 284 static int
 285 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 286                             struct ibv_counter_set_data *cs_data)
 287 {
 288 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 289         (void)query_attr;
 290         (void)cs_data;
 291         return ENOTSUP;
 292 #else
 293         return ibv_query_counter_set(query_attr, cs_data);
 294 #endif
 295 }
 296
 297 static struct ibv_counters *
 298 mlx5_glue_create_counters(struct ibv_context *context,
 299                           struct ibv_counters_init_attr *init_attr)
 300 {
 301 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 302         (void)context;
 303         (void)init_attr;
 304         errno = ENOTSUP;
 305         return NULL;
 306 #else
 307         return ibv_create_counters(context, init_attr);
 308 #endif
 309 }
 310
 311 static int
 312 mlx5_glue_destroy_counters(struct ibv_counters *counters)
 313 {
 314 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 315         (void)counters;
 316         return ENOTSUP;
 317 #else
 318         return ibv_destroy_counters(counters);
 319 #endif
 320 }
 321
 322 static int
 323 mlx5_glue_attach_counters(struct ibv_counters *counters,
 324                           struct ibv_counter_attach_attr *attr,
 325                           struct ibv_flow *flow)
 326 {
 327 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 328         (void)counters;
 329         (void)attr;
 330         (void)flow;
 331         return ENOTSUP;
 332 #else
 333         return ibv_attach_counters_point_flow(counters, attr, flow);
 334 #endif
 335 }
 336
 337 static int
 338 mlx5_glue_query_counters(struct ibv_counters *counters,
 339                          uint64_t *counters_value,
 340                          uint32_t ncounters,
 341                          uint32_t flags)
 342 {
 343 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 344         (void)counters;
 345         (void)counters_value;
 346         (void)ncounters;
 347         (void)flags;
 348         return ENOTSUP;
 349 #else
 350         return ibv_read_counters(counters, counters_value, ncounters, flags);
 351 #endif
 352 }
 353
 354 static void
 355 mlx5_glue_ack_async_event(struct ibv_async_event *event)
 356 {
 357         ibv_ack_async_event(event);
 358 }
 359
 360 static int
 361 mlx5_glue_get_async_event(struct ibv_context *context,
 362                           struct ibv_async_event *event)
 363 {
 364         return ibv_get_async_event(context, event);
 365 }
 366
 367 static const char *
 368 mlx5_glue_port_state_str(enum ibv_port_state port_state)
 369 {
 370         return ibv_port_state_str(port_state);
 371 }
 372
 373 static struct ibv_cq *
 374 mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq)
 375 {
 376         return ibv_cq_ex_to_cq(cq);
 377 }
 378
 379 static void *
 380 mlx5_glue_dr_create_flow_action_dest_flow_tbl(void *tbl)
 381 {
 382 #ifdef HAVE_MLX5DV_DR
 383         return mlx5dv_dr_action_create_dest_table(tbl);
 384 #else
 385         (void)tbl;
 386         errno = ENOTSUP;
 387         return NULL;
 388 #endif
 389 }
 390
 391 static void *
 392 mlx5_glue_dr_create_flow_action_dest_port(void *domain, uint32_t port)
 393 {
 394 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
 395         return mlx5dv_dr_action_create_dest_ib_port(domain, port);
 396 #else
 397 #ifdef HAVE_MLX5DV_DR_ESWITCH
 398         return mlx5dv_dr_action_create_dest_vport(domain, port);
 399 #else
 400         (void)domain;
 401         (void)port;
 402         errno = ENOTSUP;
 403         return NULL;
 404 #endif
 405 #endif
 406 }
 407
 408 static void *
 409 mlx5_glue_dr_create_flow_action_drop(void)
 410 {
 411 #ifdef HAVE_MLX5DV_DR_ESWITCH
 412         return mlx5dv_dr_action_create_drop();
 413 #else
 414         errno = ENOTSUP;
 415         return NULL;
 416 #endif
 417 }
 418
 419 static void *
 420 mlx5_glue_dr_create_flow_action_push_vlan(struct mlx5dv_dr_domain *domain,
 421                                           rte_be32_t vlan_tag)
 422 {
 423 #ifdef HAVE_MLX5DV_DR_VLAN
 424         return mlx5dv_dr_action_create_push_vlan(domain, vlan_tag);
 425 #else
 426         (void)domain;
 427         (void)vlan_tag;
 428         errno = ENOTSUP;
 429         return NULL;
 430 #endif
 431 }
 432
 433 static void *
 434 mlx5_glue_dr_create_flow_action_pop_vlan(void)
 435 {
 436 #ifdef HAVE_MLX5DV_DR_VLAN
 437         return mlx5dv_dr_action_create_pop_vlan();
 438 #else
 439         errno = ENOTSUP;
 440         return NULL;
 441 #endif
 442 }
 443
 444 static void *
 445 mlx5_glue_dr_create_flow_tbl(void *domain, uint32_t level)
 446 {
 447 #ifdef HAVE_MLX5DV_DR
 448         return mlx5dv_dr_table_create(domain, level);
 449 #else
 450         (void)domain;
 451         (void)level;
 452         errno = ENOTSUP;
 453         return NULL;
 454 #endif
 455 }
 456
 457 static int
 458 mlx5_glue_dr_destroy_flow_tbl(void *tbl)
 459 {
 460 #ifdef HAVE_MLX5DV_DR
 461         return mlx5dv_dr_table_destroy(tbl);
 462 #else
 463         (void)tbl;
 464         errno = ENOTSUP;
 465         return errno;
 466 #endif
 467 }
 468
 469 static void *
 470 mlx5_glue_dr_create_domain(struct ibv_context *ctx,
 471                            enum  mlx5dv_dr_domain_type domain)
 472 {
 473 #ifdef HAVE_MLX5DV_DR
 474         return mlx5dv_dr_domain_create(ctx, domain);
 475 #else
 476         (void)ctx;
 477         (void)domain;
 478         errno = ENOTSUP;
 479         return NULL;
 480 #endif
 481 }
 482
 483 static int
 484 mlx5_glue_dr_destroy_domain(void *domain)
 485 {
 486 #ifdef HAVE_MLX5DV_DR
 487         return mlx5dv_dr_domain_destroy(domain);
 488 #else
 489         (void)domain;
 490         errno = ENOTSUP;
 491         return errno;
 492 #endif
 493 }
 494
 495 static struct ibv_cq_ex *
 496 mlx5_glue_dv_create_cq(struct ibv_context *context,
 497                        struct ibv_cq_init_attr_ex *cq_attr,
 498                        struct mlx5dv_cq_init_attr *mlx5_cq_attr)
 499 {
 500         return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
 501 }
 502
 503 static struct ibv_wq *
 504 mlx5_glue_dv_create_wq(struct ibv_context *context,
 505                        struct ibv_wq_init_attr *wq_attr,
 506                        struct mlx5dv_wq_init_attr *mlx5_wq_attr)
 507 {
 508 #ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 509         (void)context;
 510         (void)wq_attr;
 511         (void)mlx5_wq_attr;
 512         errno = ENOTSUP;
 513         return NULL;
 514 #else
 515         return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr);
 516 #endif
 517 }
 518
 519 static int
 520 mlx5_glue_dv_query_device(struct ibv_context *ctx,
 521                           struct mlx5dv_context *attrs_out)
 522 {
 523         return mlx5dv_query_device(ctx, attrs_out);
 524 }
 525
 526 static int
 527 mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx,
 528                               enum mlx5dv_set_ctx_attr_type type, void *attr)
 529 {
 530         return mlx5dv_set_context_attr(ibv_ctx, type, attr);
 531 }
 532
 533 static int
 534 mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
 535 {
 536         return mlx5dv_init_obj(obj, obj_type);
 537 }
 538
 539 static struct ibv_qp *
 540 mlx5_glue_dv_create_qp(struct ibv_context *context,
 541                        struct ibv_qp_init_attr_ex *qp_init_attr_ex,
 542                        struct mlx5dv_qp_init_attr *dv_qp_init_attr)
 543 {
 544 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 545         return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr);
 546 #else
 547         (void)context;
 548         (void)qp_init_attr_ex;
 549         (void)dv_qp_init_attr;
 550         errno = ENOTSUP;
 551         return NULL;
 552 #endif
 553 }
 554
 555 static void *
 556 mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
 557                                  struct mlx5dv_flow_matcher_attr *matcher_attr,
 558                                  void *tbl)
 559 {
 560 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 561 #ifdef HAVE_MLX5DV_DR
 562         (void)context;
 563         return mlx5dv_dr_matcher_create(tbl, matcher_attr->priority,
 564                                         matcher_attr->match_criteria_enable,
 565                                         matcher_attr->match_mask);
 566 #else
 567         (void)tbl;
 568         return mlx5dv_create_flow_matcher(context, matcher_attr);
 569 #endif
 570 #else
 571         (void)context;
 572         (void)matcher_attr;
 573         (void)tbl;
 574         errno = ENOTSUP;
 575         return NULL;
 576 #endif
 577 }
 578
 579 static void *
 580 mlx5_glue_dv_create_flow(void *matcher,
 581                          void *match_value,
 582                          size_t num_actions,
 583                          void *actions[])
 584 {
 585 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 586 #ifdef HAVE_MLX5DV_DR
 587         return mlx5dv_dr_rule_create(matcher, match_value, num_actions,
 588                                      (struct mlx5dv_dr_action **)actions);
 589 #else
 590         struct mlx5dv_flow_action_attr actions_attr[8];
 591
 592         if (num_actions > 8)
 593                 return NULL;
 594         for (size_t i = 0; i < num_actions; i++)
 595                 actions_attr[i] =
 596                         *((struct mlx5dv_flow_action_attr *)(actions[i]));
 597         return mlx5dv_create_flow(matcher, match_value,
 598                                   num_actions, actions_attr);
 599 #endif
 600 #else
 601         (void)matcher;
 602         (void)match_value;
 603         (void)num_actions;
 604         (void)actions;
 605         return NULL;
 606 #endif
 607 }
 608
 609 static void *
 610 mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset)
 611 {
 612 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 613 #ifdef HAVE_MLX5DV_DR
 614         return mlx5dv_dr_action_create_flow_counter(counter_obj, offset);
 615 #else
 616         struct mlx5dv_flow_action_attr *action;
 617
 618         (void)offset;
 619         action = malloc(sizeof(*action));
 620         if (!action)
 621                 return NULL;
 622         action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
 623         action->obj = counter_obj;
 624         return action;
 625 #endif
 626 #else
 627         (void)counter_obj;
 628         (void)offset;
 629         errno = ENOTSUP;
 630         return NULL;
 631 #endif
 632 }
 633
 634 static void *
 635 mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp)
 636 {
 637 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 638 #ifdef HAVE_MLX5DV_DR
 639         return mlx5dv_dr_action_create_dest_ibv_qp(qp);
 640 #else
 641         struct mlx5dv_flow_action_attr *action;
 642
 643         action = malloc(sizeof(*action));
 644         if (!action)
 645                 return NULL;
 646         action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
 647         action->obj = qp;
 648         return action;
 649 #endif
 650 #else
 651         (void)qp;
 652         errno = ENOTSUP;
 653         return NULL;
 654 #endif
 655 }
 656
 657 static void *
 658 mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir)
 659 {
 660 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
 661         return mlx5dv_dr_action_create_dest_devx_tir(tir);
 662 #else
 663         (void)tir;
 664         errno = ENOTSUP;
 665         return NULL;
 666 #endif
 667 }
 668
 669 static void *
 670 mlx5_glue_dv_create_flow_action_modify_header
 671                                         (struct ibv_context *ctx,
 672                                          enum mlx5dv_flow_table_type ft_type,
 673                                          void *domain, uint64_t flags,
 674                                          size_t actions_sz,
 675                                          uint64_t actions[])
 676 {
 677 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 678 #ifdef HAVE_MLX5DV_DR
 679         (void)ctx;
 680         (void)ft_type;
 681         return mlx5dv_dr_action_create_modify_header(domain, flags, actions_sz,
 682                                                      (__be64 *)actions);
 683 #else
 684         struct mlx5dv_flow_action_attr *action;
 685
 686         (void)domain;
 687         (void)flags;
 688         action = malloc(sizeof(*action));
 689         if (!action)
 690                 return NULL;
 691         action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
 692         action->action = mlx5dv_create_flow_action_modify_header
 693                 (ctx, actions_sz, actions, ft_type);
 694         return action;
 695 #endif
 696 #else
 697         (void)ctx;
 698         (void)ft_type;
 699         (void)domain;
 700         (void)flags;
 701         (void)actions_sz;
 702         (void)actions;
 703         errno = ENOTSUP;
 704         return NULL;
 705 #endif
 706 }
 707
 708 static void *
 709 mlx5_glue_dv_create_flow_action_packet_reformat
 710                 (struct ibv_context *ctx,
 711                  enum mlx5dv_flow_action_packet_reformat_type reformat_type,
 712                  enum mlx5dv_flow_table_type ft_type,
 713                  struct mlx5dv_dr_domain *domain,
 714                  uint32_t flags, size_t data_sz, void *data)
 715 {
 716 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 717 #ifdef HAVE_MLX5DV_DR
 718         (void)ctx;
 719         (void)ft_type;
 720         return mlx5dv_dr_action_create_packet_reformat(domain, flags,
 721                                                        reformat_type, data_sz,
 722                                                        data);
 723 #else
 724         (void)domain;
 725         (void)flags;
 726         struct mlx5dv_flow_action_attr *action;
 727
 728         action = malloc(sizeof(*action));
 729         if (!action)
 730                 return NULL;
 731         action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
 732         action->action = mlx5dv_create_flow_action_packet_reformat
 733                 (ctx, data_sz, data, reformat_type, ft_type);
 734         return action;
 735 #endif
 736 #else
 737         (void)ctx;
 738         (void)reformat_type;
 739         (void)ft_type;
 740         (void)domain;
 741         (void)flags;
 742         (void)data_sz;
 743         (void)data;
 744         errno = ENOTSUP;
 745         return NULL;
 746 #endif
 747 }
 748
 749 static void *
 750 mlx5_glue_dv_create_flow_action_tag(uint32_t tag)
 751 {
 752 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 753 #ifdef HAVE_MLX5DV_DR
 754         return mlx5dv_dr_action_create_tag(tag);
 755 #else
 756         struct mlx5dv_flow_action_attr *action;
 757         action = malloc(sizeof(*action));
 758         if (!action)
 759                 return NULL;
 760         action->type = MLX5DV_FLOW_ACTION_TAG;
 761         action->tag_value = tag;
 762         return action;
 763 #endif
 764 #endif
 765         (void)tag;
 766         errno = ENOTSUP;
 767         return NULL;
 768 }
 769
 770 static void *
 771 mlx5_glue_dv_create_flow_action_meter(struct mlx5dv_dr_flow_meter_attr *attr)
 772 {
 773 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
 774         return mlx5dv_dr_action_create_flow_meter(attr);
 775 #else
 776         (void)attr;
 777         errno = ENOTSUP;
 778         return NULL;
 779 #endif
 780 }
 781
 782 static int
 783 mlx5_glue_dv_modify_flow_action_meter(void *action,
 784                                       struct mlx5dv_dr_flow_meter_attr *attr,
 785                                       uint64_t modify_bits)
 786 {
 787 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
 788         return mlx5dv_dr_action_modify_flow_meter(action, attr, modify_bits);
 789 #else
 790         (void)action;
 791         (void)attr;
 792         (void)modify_bits;
 793         errno = ENOTSUP;
 794         return errno;
 795 #endif
 796 }
 797
 798 static int
 799 mlx5_glue_dv_destroy_flow(void *flow_id)
 800 {
 801 #ifdef HAVE_MLX5DV_DR
 802         return mlx5dv_dr_rule_destroy(flow_id);
 803 #else
 804         return ibv_destroy_flow(flow_id);
 805 #endif
 806 }
 807
 808 static int
 809 mlx5_glue_dv_destroy_flow_matcher(void *matcher)
 810 {
 811 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 812 #ifdef HAVE_MLX5DV_DR
 813         return mlx5dv_dr_matcher_destroy(matcher);
 814 #else
 815         return mlx5dv_destroy_flow_matcher(matcher);
 816 #endif
 817 #else
 818         (void)matcher;
 819         errno = ENOTSUP;
 820         return errno;
 821 #endif
 822 }
 823
 824 static struct ibv_context *
 825 mlx5_glue_dv_open_device(struct ibv_device *device)
 826 {
 827 #ifdef HAVE_IBV_DEVX_OBJ
 828         return mlx5dv_open_device(device,
 829                                   &(struct mlx5dv_context_attr){
 830                                         .flags = MLX5DV_CONTEXT_FLAGS_DEVX,
 831                                   });
 832 #else
 833         (void)device;
 834         errno = ENOTSUP;
 835         return NULL;
 836 #endif
 837 }
 838
 839 static struct mlx5dv_devx_obj *
 840 mlx5_glue_devx_obj_create(struct ibv_context *ctx,
 841                           const void *in, size_t inlen,
 842                           void *out, size_t outlen)
 843 {
 844 #ifdef HAVE_IBV_DEVX_OBJ
 845         return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen);
 846 #else
 847         (void)ctx;
 848         (void)in;
 849         (void)inlen;
 850         (void)out;
 851         (void)outlen;
 852         errno = ENOTSUP;
 853         return NULL;
 854 #endif
 855 }
 856
 857 static int
 858 mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj)
 859 {
 860 #ifdef HAVE_IBV_DEVX_OBJ
 861         return mlx5dv_devx_obj_destroy(obj);
 862 #else
 863         (void)obj;
 864         return -ENOTSUP;
 865 #endif
 866 }
 867
 868 static int
 869 mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj,
 870                          const void *in, size_t inlen,
 871                          void *out, size_t outlen)
 872 {
 873 #ifdef HAVE_IBV_DEVX_OBJ
 874         return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen);
 875 #else
 876         (void)obj;
 877         (void)in;
 878         (void)inlen;
 879         (void)out;
 880         (void)outlen;
 881         return -ENOTSUP;
 882 #endif
 883 }
 884
 885 static int
 886 mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj,
 887                           const void *in, size_t inlen,
 888                           void *out, size_t outlen)
 889 {
 890 #ifdef HAVE_IBV_DEVX_OBJ
 891         return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen);
 892 #else
 893         (void)obj;
 894         (void)in;
 895         (void)inlen;
 896         (void)out;
 897         (void)outlen;
 898         return -ENOTSUP;
 899 #endif
 900 }
 901
 902 static int
 903 mlx5_glue_devx_general_cmd(struct ibv_context *ctx,
 904                            const void *in, size_t inlen,
 905                            void *out, size_t outlen)
 906 {
 907 #ifdef HAVE_IBV_DEVX_OBJ
 908         return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen);
 909 #else
 910         (void)ctx;
 911         (void)in;
 912         (void)inlen;
 913         (void)out;
 914         (void)outlen;
 915         return -ENOTSUP;
 916 #endif
 917 }
 918
 919 static struct mlx5dv_devx_cmd_comp *
 920 mlx5_glue_devx_create_cmd_comp(struct ibv_context *ctx)
 921 {
 922 #ifdef HAVE_IBV_DEVX_ASYNC
 923         return mlx5dv_devx_create_cmd_comp(ctx);
 924 #else
 925         (void)ctx;
 926         errno = -ENOTSUP;
 927         return NULL;
 928 #endif
 929 }
 930
 931 static void
 932 mlx5_glue_devx_destroy_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp)
 933 {
 934 #ifdef HAVE_IBV_DEVX_ASYNC
 935         mlx5dv_devx_destroy_cmd_comp(cmd_comp);
 936 #else
 937         (void)cmd_comp;
 938         errno = -ENOTSUP;
 939 #endif
 940 }
 941
 942 static int
 943 mlx5_glue_devx_obj_query_async(struct mlx5dv_devx_obj *obj, const void *in,
 944                                size_t inlen, size_t outlen, uint64_t wr_id,
 945                                struct mlx5dv_devx_cmd_comp *cmd_comp)
 946 {
 947 #ifdef HAVE_IBV_DEVX_ASYNC
 948         return mlx5dv_devx_obj_query_async(obj, in, inlen, outlen, wr_id,
 949                                            cmd_comp);
 950 #else
 951         (void)obj;
 952         (void)in;
 953         (void)inlen;
 954         (void)outlen;
 955         (void)wr_id;
 956         (void)cmd_comp;
 957         return -ENOTSUP;
 958 #endif
 959 }
 960
 961 static int
 962 mlx5_glue_devx_get_async_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp,
 963                                   struct mlx5dv_devx_async_cmd_hdr *cmd_resp,
 964                                   size_t cmd_resp_len)
 965 {
 966 #ifdef HAVE_IBV_DEVX_ASYNC
 967         return mlx5dv_devx_get_async_cmd_comp(cmd_comp, cmd_resp,
 968                                               cmd_resp_len);
 969 #else
 970         (void)cmd_comp;
 971         (void)cmd_resp;
 972         (void)cmd_resp_len;
 973         return -ENOTSUP;
 974 #endif
 975 }
 976
 977 static struct mlx5dv_devx_umem *
 978 mlx5_glue_devx_umem_reg(struct ibv_context *context, void *addr, size_t size,
 979                         uint32_t access)
 980 {
 981 #ifdef HAVE_IBV_DEVX_OBJ
 982         return mlx5dv_devx_umem_reg(context, addr, size, access);
 983 #else
 984         (void)context;
 985         (void)addr;
 986         (void)size;
 987         (void)access;
 988         errno = -ENOTSUP;
 989         return NULL;
 990 #endif
 991 }
 992
 993 static int
 994 mlx5_glue_devx_umem_dereg(struct mlx5dv_devx_umem *dv_devx_umem)
 995 {
 996 #ifdef HAVE_IBV_DEVX_OBJ
 997         return mlx5dv_devx_umem_dereg(dv_devx_umem);
 998 #else
 999         (void)dv_devx_umem;
1000         return -ENOTSUP;
1001 #endif
1002 }
1003
1004 static int
1005 mlx5_glue_devx_qp_query(struct ibv_qp *qp,
1006                         const void *in, size_t inlen,
1007                         void *out, size_t outlen)
1008 {
1009 #ifdef HAVE_IBV_DEVX_OBJ
1010         return mlx5dv_devx_qp_query(qp, in, inlen, out, outlen);
1011 #else
1012         (void)qp;
1013         (void)in;
1014         (void)inlen;
1015         (void)out;
1016         (void)outlen;
1017         errno = ENOTSUP;
1018         return errno;
1019 #endif
1020 }
1021
1022 static int
1023 mlx5_glue_devx_port_query(struct ibv_context *ctx,
1024                           uint32_t port_num,
1025                           struct mlx5dv_devx_port *mlx5_devx_port)
1026 {
1027 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
1028         return mlx5dv_query_devx_port(ctx, port_num, mlx5_devx_port);
1029 #else
1030         (void)ctx;
1031         (void)port_num;
1032         (void)mlx5_devx_port;
1033         errno = ENOTSUP;
1034         return errno;
1035 #endif
1036 }
1037
1038 static int
1039 mlx5_glue_dr_dump_domain(FILE *file, void *domain)
1040 {
1041 #ifdef HAVE_MLX5_DR_FLOW_DUMP
1042         return mlx5dv_dump_dr_domain(file, domain);
1043 #else
1044         RTE_SET_USED(file);
1045         RTE_SET_USED(domain);
1046         return -ENOTSUP;
1047 #endif
1048 }
1049
1050 static int
1051 mlx5_glue_devx_query_eqn(struct ibv_context *ctx, uint32_t cpus,
1052                          uint32_t *eqn)
1053 {
1054 #ifdef HAVE_IBV_DEVX_OBJ
1055         return mlx5dv_devx_query_eqn(ctx, cpus, eqn);
1056 #else
1057         (void)ctx;
1058         (void)cpus;
1059         (void)eqn;
1060         return -ENOTSUP;
1061 #endif
1062 }
1063
1064 static struct mlx5dv_devx_event_channel *
1065 mlx5_glue_devx_create_event_channel(struct ibv_context *ctx, int flags)
1066 {
1067 #ifdef HAVE_IBV_DEVX_EVENT
1068         return mlx5dv_devx_create_event_channel(ctx, flags);
1069 #else
1070         (void)ctx;
1071         (void)flags;
1072         errno = ENOTSUP;
1073         return NULL;
1074 #endif
1075 }
1076
1077 static void
1078 mlx5_glue_devx_destroy_event_channel(struct mlx5dv_devx_event_channel *eventc)
1079 {
1080 #ifdef HAVE_IBV_DEVX_EVENT
1081         mlx5dv_devx_destroy_event_channel(eventc);
1082 #else
1083         (void)eventc;
1084 #endif
1085 }
1086
1087 static int
1088 mlx5_glue_devx_subscribe_devx_event(struct mlx5dv_devx_event_channel *eventc,
1089                                     struct mlx5dv_devx_obj *obj,
1090                                     uint16_t events_sz, uint16_t events_num[],
1091                                     uint64_t cookie)
1092 {
1093 #ifdef HAVE_IBV_DEVX_EVENT
1094         return mlx5dv_devx_subscribe_devx_event(eventc, obj, events_sz,
1095                                                 events_num, cookie);
1096 #else
1097         (void)eventc;
1098         (void)obj;
1099         (void)events_sz;
1100         (void)events_num;
1101         (void)cookie;
1102         return -ENOTSUP;
1103 #endif
1104 }
1105
1106 static int
1107 mlx5_glue_devx_subscribe_devx_event_fd(struct mlx5dv_devx_event_channel *eventc,
1108                                        int fd, struct mlx5dv_devx_obj *obj,
1109                                        uint16_t event_num)
1110 {
1111 #ifdef HAVE_IBV_DEVX_EVENT
1112         return mlx5dv_devx_subscribe_devx_event_fd(eventc, fd, obj, event_num);
1113 #else
1114         (void)eventc;
1115         (void)fd;
1116         (void)obj;
1117         (void)event_num;
1118         return -ENOTSUP;
1119 #endif
1120 }
1121
1122 static ssize_t
1123 mlx5_glue_devx_get_event(struct mlx5dv_devx_event_channel *eventc,
1124                          struct mlx5dv_devx_async_event_hdr *event_data,
1125                          size_t event_resp_len)
1126 {
1127 #ifdef HAVE_IBV_DEVX_EVENT
1128         return mlx5dv_devx_get_event(eventc, event_data, event_resp_len);
1129 #else
1130         (void)eventc;
1131         (void)event_data;
1132         (void)event_resp_len;
1133         errno = ENOTSUP;
1134         return -1;
1135 #endif
1136 }
1137
1138 static struct mlx5dv_devx_uar *
1139 mlx5_glue_devx_alloc_uar(struct ibv_context *context, uint32_t flags)
1140 {
1141 #ifdef HAVE_IBV_DEVX_OBJ
1142         return mlx5dv_devx_alloc_uar(context, flags);
1143 #else
1144         (void)context;
1145         (void)flags;
1146         errno = ENOTSUP;
1147         return NULL;
1148 #endif
1149 }
1150
1151 static void
1152 mlx5_glue_devx_free_uar(struct mlx5dv_devx_uar *devx_uar)
1153 {
1154 #ifdef HAVE_IBV_DEVX_OBJ
1155         mlx5dv_devx_free_uar(devx_uar);
1156 #else
1157         (void)devx_uar;
1158 #endif
1159 }
1160
1161 static struct mlx5dv_var *
1162 mlx5_glue_dv_alloc_var(struct ibv_context *context, uint32_t flags)
1163 {
1164 #ifdef HAVE_IBV_VAR
1165         return mlx5dv_alloc_var(context, flags);
1166 #else
1167         (void)context;
1168         (void)flags;
1169         errno = ENOTSUP;
1170         return NULL;
1171 #endif
1172 }
1173
1174 static void
1175 mlx5_glue_dv_free_var(struct mlx5dv_var *var)
1176 {
1177 #ifdef HAVE_IBV_VAR
1178         mlx5dv_free_var(var);
1179 #else
1180         (void)var;
1181         errno = ENOTSUP;
1182 #endif
1183 }
1184
1185 alignas(RTE_CACHE_LINE_SIZE)
1186 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
1187         .version = MLX5_GLUE_VERSION,
1188         .fork_init = mlx5_glue_fork_init,
1189         .alloc_pd = mlx5_glue_alloc_pd,
1190         .dealloc_pd = mlx5_glue_dealloc_pd,
1191         .get_device_list = mlx5_glue_get_device_list,
1192         .free_device_list = mlx5_glue_free_device_list,
1193         .open_device = mlx5_glue_open_device,
1194         .close_device = mlx5_glue_close_device,
1195         .query_device = mlx5_glue_query_device,
1196         .query_device_ex = mlx5_glue_query_device_ex,
1197         .query_rt_values_ex = mlx5_glue_query_rt_values_ex,
1198         .query_port = mlx5_glue_query_port,
1199         .create_comp_channel = mlx5_glue_create_comp_channel,
1200         .destroy_comp_channel = mlx5_glue_destroy_comp_channel,
1201         .create_cq = mlx5_glue_create_cq,
1202         .destroy_cq = mlx5_glue_destroy_cq,
1203         .get_cq_event = mlx5_glue_get_cq_event,
1204         .ack_cq_events = mlx5_glue_ack_cq_events,
1205         .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table,
1206         .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table,
1207         .create_wq = mlx5_glue_create_wq,
1208         .destroy_wq = mlx5_glue_destroy_wq,
1209         .modify_wq = mlx5_glue_modify_wq,
1210         .create_flow = mlx5_glue_create_flow,
1211         .destroy_flow = mlx5_glue_destroy_flow,
1212         .destroy_flow_action = mlx5_glue_destroy_flow_action,
1213         .create_qp = mlx5_glue_create_qp,
1214         .create_qp_ex = mlx5_glue_create_qp_ex,
1215         .destroy_qp = mlx5_glue_destroy_qp,
1216         .modify_qp = mlx5_glue_modify_qp,
1217         .reg_mr = mlx5_glue_reg_mr,
1218         .alloc_null_mr = mlx5_glue_alloc_null_mr,
1219         .dereg_mr = mlx5_glue_dereg_mr,
1220         .create_counter_set = mlx5_glue_create_counter_set,
1221         .destroy_counter_set = mlx5_glue_destroy_counter_set,
1222         .describe_counter_set = mlx5_glue_describe_counter_set,
1223         .query_counter_set = mlx5_glue_query_counter_set,
1224         .create_counters = mlx5_glue_create_counters,
1225         .destroy_counters = mlx5_glue_destroy_counters,
1226         .attach_counters = mlx5_glue_attach_counters,
1227         .query_counters = mlx5_glue_query_counters,
1228         .ack_async_event = mlx5_glue_ack_async_event,
1229         .get_async_event = mlx5_glue_get_async_event,
1230         .port_state_str = mlx5_glue_port_state_str,
1231         .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
1232         .dr_create_flow_action_dest_flow_tbl =
1233                 mlx5_glue_dr_create_flow_action_dest_flow_tbl,
1234         .dr_create_flow_action_dest_port =
1235                 mlx5_glue_dr_create_flow_action_dest_port,
1236         .dr_create_flow_action_drop =
1237                 mlx5_glue_dr_create_flow_action_drop,
1238         .dr_create_flow_action_push_vlan =
1239                 mlx5_glue_dr_create_flow_action_push_vlan,
1240         .dr_create_flow_action_pop_vlan =
1241                 mlx5_glue_dr_create_flow_action_pop_vlan,
1242         .dr_create_flow_tbl = mlx5_glue_dr_create_flow_tbl,
1243         .dr_destroy_flow_tbl = mlx5_glue_dr_destroy_flow_tbl,
1244         .dr_create_domain = mlx5_glue_dr_create_domain,
1245         .dr_destroy_domain = mlx5_glue_dr_destroy_domain,
1246         .dv_create_cq = mlx5_glue_dv_create_cq,
1247         .dv_create_wq = mlx5_glue_dv_create_wq,
1248         .dv_query_device = mlx5_glue_dv_query_device,
1249         .dv_set_context_attr = mlx5_glue_dv_set_context_attr,
1250         .dv_init_obj = mlx5_glue_dv_init_obj,
1251         .dv_create_qp = mlx5_glue_dv_create_qp,
1252         .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
1253         .dv_create_flow = mlx5_glue_dv_create_flow,
1254         .dv_create_flow_action_counter =
1255                 mlx5_glue_dv_create_flow_action_counter,
1256         .dv_create_flow_action_dest_ibv_qp =
1257                 mlx5_glue_dv_create_flow_action_dest_ibv_qp,
1258         .dv_create_flow_action_dest_devx_tir =
1259                 mlx5_glue_dv_create_flow_action_dest_devx_tir,
1260         .dv_create_flow_action_modify_header =
1261                 mlx5_glue_dv_create_flow_action_modify_header,
1262         .dv_create_flow_action_packet_reformat =
1263                 mlx5_glue_dv_create_flow_action_packet_reformat,
1264         .dv_create_flow_action_tag =  mlx5_glue_dv_create_flow_action_tag,
1265         .dv_create_flow_action_meter = mlx5_glue_dv_create_flow_action_meter,
1266         .dv_modify_flow_action_meter = mlx5_glue_dv_modify_flow_action_meter,
1267         .dv_destroy_flow = mlx5_glue_dv_destroy_flow,
1268         .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
1269         .dv_open_device = mlx5_glue_dv_open_device,
1270         .devx_obj_create = mlx5_glue_devx_obj_create,
1271         .devx_obj_destroy = mlx5_glue_devx_obj_destroy,
1272         .devx_obj_query = mlx5_glue_devx_obj_query,
1273         .devx_obj_modify = mlx5_glue_devx_obj_modify,
1274         .devx_general_cmd = mlx5_glue_devx_general_cmd,
1275         .devx_create_cmd_comp = mlx5_glue_devx_create_cmd_comp,
1276         .devx_destroy_cmd_comp = mlx5_glue_devx_destroy_cmd_comp,
1277         .devx_obj_query_async = mlx5_glue_devx_obj_query_async,
1278         .devx_get_async_cmd_comp = mlx5_glue_devx_get_async_cmd_comp,
1279         .devx_umem_reg = mlx5_glue_devx_umem_reg,
1280         .devx_umem_dereg = mlx5_glue_devx_umem_dereg,
1281         .devx_qp_query = mlx5_glue_devx_qp_query,
1282         .devx_port_query = mlx5_glue_devx_port_query,
1283         .dr_dump_domain = mlx5_glue_dr_dump_domain,
1284         .devx_query_eqn = mlx5_glue_devx_query_eqn,
1285         .devx_create_event_channel = mlx5_glue_devx_create_event_channel,
1286         .devx_destroy_event_channel = mlx5_glue_devx_destroy_event_channel,
1287         .devx_subscribe_devx_event = mlx5_glue_devx_subscribe_devx_event,
1288         .devx_subscribe_devx_event_fd = mlx5_glue_devx_subscribe_devx_event_fd,
1289         .devx_get_event = mlx5_glue_devx_get_event,
1290         .devx_alloc_uar = mlx5_glue_devx_alloc_uar,
1291         .devx_free_uar = mlx5_glue_devx_free_uar,
1292         .dv_alloc_var = mlx5_glue_dv_alloc_var,
1293         .dv_free_var = mlx5_glue_dv_free_var,
1294 };