drivers/net/mlx5/mlx5_glue.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stdalign.h>
   8 #include <stddef.h>
   9 #include <stdint.h>
  10 #include <stdlib.h>
  11
  12 /*
  13  * Not needed by this file; included to work around the lack of off_t
  14  * definition for mlx5dv.h with unpatched rdma-core versions.
  15  */
  16 #include <sys/types.h>
  17
  18 /* Verbs headers do not support -pedantic. */
  19 #ifdef PEDANTIC
  20 #pragma GCC diagnostic ignored "-Wpedantic"
  21 #endif
  22 #include <infiniband/mlx5dv.h>
  23 #include <infiniband/verbs.h>
  24 #ifdef PEDANTIC
  25 #pragma GCC diagnostic error "-Wpedantic"
  26 #endif
  27
  28 #include <rte_config.h>
  29
  30 #include "mlx5_autoconf.h"
  31 #include "mlx5_glue.h"
  32
  33 static int
  34 mlx5_glue_fork_init(void)
  35 {
  36         return ibv_fork_init();
  37 }
  38
  39 static struct ibv_pd *
  40 mlx5_glue_alloc_pd(struct ibv_context *context)
  41 {
  42         return ibv_alloc_pd(context);
  43 }
  44
  45 static int
  46 mlx5_glue_dealloc_pd(struct ibv_pd *pd)
  47 {
  48         return ibv_dealloc_pd(pd);
  49 }
  50
  51 static struct ibv_device **
  52 mlx5_glue_get_device_list(int *num_devices)
  53 {
  54         return ibv_get_device_list(num_devices);
  55 }
  56
  57 static void
  58 mlx5_glue_free_device_list(struct ibv_device **list)
  59 {
  60         ibv_free_device_list(list);
  61 }
  62
  63 static struct ibv_context *
  64 mlx5_glue_open_device(struct ibv_device *device)
  65 {
  66         return ibv_open_device(device);
  67 }
  68
  69 static int
  70 mlx5_glue_close_device(struct ibv_context *context)
  71 {
  72         return ibv_close_device(context);
  73 }
  74
  75 static int
  76 mlx5_glue_query_device(struct ibv_context *context,
  77                        struct ibv_device_attr *device_attr)
  78 {
  79         return ibv_query_device(context, device_attr);
  80 }
  81
  82 static int
  83 mlx5_glue_query_device_ex(struct ibv_context *context,
  84                           const struct ibv_query_device_ex_input *input,
  85                           struct ibv_device_attr_ex *attr)
  86 {
  87         return ibv_query_device_ex(context, input, attr);
  88 }
  89
  90 static int
  91 mlx5_glue_query_rt_values_ex(struct ibv_context *context,
  92                           struct ibv_values_ex *values)
  93 {
  94         return ibv_query_rt_values_ex(context, values);
  95 }
  96
  97 static int
  98 mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num,
  99                      struct ibv_port_attr *port_attr)
 100 {
 101         return ibv_query_port(context, port_num, port_attr);
 102 }
 103
 104 static struct ibv_comp_channel *
 105 mlx5_glue_create_comp_channel(struct ibv_context *context)
 106 {
 107         return ibv_create_comp_channel(context);
 108 }
 109
 110 static int
 111 mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
 112 {
 113         return ibv_destroy_comp_channel(channel);
 114 }
 115
 116 static struct ibv_cq *
 117 mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,
 118                     struct ibv_comp_channel *channel, int comp_vector)
 119 {
 120         return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
 121 }
 122
 123 static int
 124 mlx5_glue_destroy_cq(struct ibv_cq *cq)
 125 {
 126         return ibv_destroy_cq(cq);
 127 }
 128
 129 static int
 130 mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
 131                        void **cq_context)
 132 {
 133         return ibv_get_cq_event(channel, cq, cq_context);
 134 }
 135
 136 static void
 137 mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
 138 {
 139         ibv_ack_cq_events(cq, nevents);
 140 }
 141
 142 static struct ibv_rwq_ind_table *
 143 mlx5_glue_create_rwq_ind_table(struct ibv_context *context,
 144                                struct ibv_rwq_ind_table_init_attr *init_attr)
 145 {
 146         return ibv_create_rwq_ind_table(context, init_attr);
 147 }
 148
 149 static int
 150 mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
 151 {
 152         return ibv_destroy_rwq_ind_table(rwq_ind_table);
 153 }
 154
 155 static struct ibv_wq *
 156 mlx5_glue_create_wq(struct ibv_context *context,
 157                     struct ibv_wq_init_attr *wq_init_attr)
 158 {
 159         return ibv_create_wq(context, wq_init_attr);
 160 }
 161
 162 static int
 163 mlx5_glue_destroy_wq(struct ibv_wq *wq)
 164 {
 165         return ibv_destroy_wq(wq);
 166 }
 167 static int
 168 mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)
 169 {
 170         return ibv_modify_wq(wq, wq_attr);
 171 }
 172
 173 static struct ibv_flow *
 174 mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)
 175 {
 176         return ibv_create_flow(qp, flow);
 177 }
 178
 179 static int
 180 mlx5_glue_destroy_flow(struct ibv_flow *flow_id)
 181 {
 182         return ibv_destroy_flow(flow_id);
 183 }
 184
 185 static int
 186 mlx5_glue_destroy_flow_action(void *action)
 187 {
 188 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 189 #ifdef HAVE_MLX5DV_DR
 190         return mlx5dv_dr_action_destroy(action);
 191 #else
 192         struct mlx5dv_flow_action_attr *attr = action;
 193         int res = 0;
 194         switch (attr->type) {
 195         case MLX5DV_FLOW_ACTION_TAG:
 196                 break;
 197         default:
 198                 res = ibv_destroy_flow_action(attr->action);
 199                 break;
 200         }
 201         free(action);
 202         return res;
 203 #endif
 204 #else
 205         (void)action;
 206         return ENOTSUP;
 207 #endif
 208 }
 209
 210 static struct ibv_qp *
 211 mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
 212 {
 213         return ibv_create_qp(pd, qp_init_attr);
 214 }
 215
 216 static struct ibv_qp *
 217 mlx5_glue_create_qp_ex(struct ibv_context *context,
 218                        struct ibv_qp_init_attr_ex *qp_init_attr_ex)
 219 {
 220         return ibv_create_qp_ex(context, qp_init_attr_ex);
 221 }
 222
 223 static int
 224 mlx5_glue_destroy_qp(struct ibv_qp *qp)
 225 {
 226         return ibv_destroy_qp(qp);
 227 }
 228
 229 static int
 230 mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
 231 {
 232         return ibv_modify_qp(qp, attr, attr_mask);
 233 }
 234
 235 static struct ibv_mr *
 236 mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)
 237 {
 238         return ibv_reg_mr(pd, addr, length, access);
 239 }
 240
 241 static int
 242 mlx5_glue_dereg_mr(struct ibv_mr *mr)
 243 {
 244         return ibv_dereg_mr(mr);
 245 }
 246
 247 static struct ibv_counter_set *
 248 mlx5_glue_create_counter_set(struct ibv_context *context,
 249                              struct ibv_counter_set_init_attr *init_attr)
 250 {
 251 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 252         (void)context;
 253         (void)init_attr;
 254         return NULL;
 255 #else
 256         return ibv_create_counter_set(context, init_attr);
 257 #endif
 258 }
 259
 260 static int
 261 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 262 {
 263 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 264         (void)cs;
 265         return ENOTSUP;
 266 #else
 267         return ibv_destroy_counter_set(cs);
 268 #endif
 269 }
 270
 271 static int
 272 mlx5_glue_describe_counter_set(struct ibv_context *context,
 273                                uint16_t counter_set_id,
 274                                struct ibv_counter_set_description *cs_desc)
 275 {
 276 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 277         (void)context;
 278         (void)counter_set_id;
 279         (void)cs_desc;
 280         return ENOTSUP;
 281 #else
 282         return ibv_describe_counter_set(context, counter_set_id, cs_desc);
 283 #endif
 284 }
 285
 286 static int
 287 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 288                             struct ibv_counter_set_data *cs_data)
 289 {
 290 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 291         (void)query_attr;
 292         (void)cs_data;
 293         return ENOTSUP;
 294 #else
 295         return ibv_query_counter_set(query_attr, cs_data);
 296 #endif
 297 }
 298
 299 static struct ibv_counters *
 300 mlx5_glue_create_counters(struct ibv_context *context,
 301                           struct ibv_counters_init_attr *init_attr)
 302 {
 303 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 304         (void)context;
 305         (void)init_attr;
 306         errno = ENOTSUP;
 307         return NULL;
 308 #else
 309         return ibv_create_counters(context, init_attr);
 310 #endif
 311 }
 312
 313 static int
 314 mlx5_glue_destroy_counters(struct ibv_counters *counters)
 315 {
 316 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 317         (void)counters;
 318         return ENOTSUP;
 319 #else
 320         return ibv_destroy_counters(counters);
 321 #endif
 322 }
 323
 324 static int
 325 mlx5_glue_attach_counters(struct ibv_counters *counters,
 326                           struct ibv_counter_attach_attr *attr,
 327                           struct ibv_flow *flow)
 328 {
 329 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 330         (void)counters;
 331         (void)attr;
 332         (void)flow;
 333         return ENOTSUP;
 334 #else
 335         return ibv_attach_counters_point_flow(counters, attr, flow);
 336 #endif
 337 }
 338
 339 static int
 340 mlx5_glue_query_counters(struct ibv_counters *counters,
 341                          uint64_t *counters_value,
 342                          uint32_t ncounters,
 343                          uint32_t flags)
 344 {
 345 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
 346         (void)counters;
 347         (void)counters_value;
 348         (void)ncounters;
 349         (void)flags;
 350         return ENOTSUP;
 351 #else
 352         return ibv_read_counters(counters, counters_value, ncounters, flags);
 353 #endif
 354 }
 355
 356 static void
 357 mlx5_glue_ack_async_event(struct ibv_async_event *event)
 358 {
 359         ibv_ack_async_event(event);
 360 }
 361
 362 static int
 363 mlx5_glue_get_async_event(struct ibv_context *context,
 364                           struct ibv_async_event *event)
 365 {
 366         return ibv_get_async_event(context, event);
 367 }
 368
 369 static const char *
 370 mlx5_glue_port_state_str(enum ibv_port_state port_state)
 371 {
 372         return ibv_port_state_str(port_state);
 373 }
 374
 375 static struct ibv_cq *
 376 mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq)
 377 {
 378         return ibv_cq_ex_to_cq(cq);
 379 }
 380
 381 static void *
 382 mlx5_glue_dr_create_flow_action_dest_flow_tbl(void *tbl)
 383 {
 384 #ifdef HAVE_MLX5DV_DR
 385         return mlx5dv_dr_action_create_dest_table(tbl);
 386 #else
 387         (void)tbl;
 388         errno = ENOTSUP;
 389         return NULL;
 390 #endif
 391 }
 392
 393 static void *
 394 mlx5_glue_dr_create_flow_action_dest_vport(void *domain, uint32_t vport)
 395 {
 396 #ifdef HAVE_MLX5DV_DR_ESWITCH
 397         return mlx5dv_dr_action_create_dest_vport(domain, vport);
 398 #else
 399         (void)domain;
 400         (void)vport;
 401         errno = ENOTSUP;
 402         return NULL;
 403 #endif
 404 }
 405
 406 static void *
 407 mlx5_glue_dr_create_flow_action_drop(void)
 408 {
 409 #ifdef HAVE_MLX5DV_DR_ESWITCH
 410         return mlx5dv_dr_action_create_drop();
 411 #else
 412         errno = ENOTSUP;
 413         return NULL;
 414 #endif
 415 }
 416
 417 static void *
 418 mlx5_glue_dr_create_flow_action_push_vlan(struct mlx5dv_dr_domain *domain,
 419                                           rte_be32_t vlan_tag)
 420 {
 421 #ifdef HAVE_MLX5DV_DR_VLAN
 422         return mlx5dv_dr_action_create_push_vlan(domain, vlan_tag);
 423 #else
 424         (void)domain;
 425         (void)vlan_tag;
 426         errno = ENOTSUP;
 427         return NULL;
 428 #endif
 429 }
 430
 431 static void *
 432 mlx5_glue_dr_create_flow_action_pop_vlan(void)
 433 {
 434 #ifdef HAVE_MLX5DV_DR_VLAN
 435         return mlx5dv_dr_action_create_pop_vlan();
 436 #else
 437         errno = ENOTSUP;
 438         return NULL;
 439 #endif
 440 }
 441
 442 static void *
 443 mlx5_glue_dr_create_flow_tbl(void *domain, uint32_t level)
 444 {
 445 #ifdef HAVE_MLX5DV_DR
 446         return mlx5dv_dr_table_create(domain, level);
 447 #else
 448         (void)domain;
 449         (void)level;
 450         errno = ENOTSUP;
 451         return NULL;
 452 #endif
 453 }
 454
 455 static int
 456 mlx5_glue_dr_destroy_flow_tbl(void *tbl)
 457 {
 458 #ifdef HAVE_MLX5DV_DR
 459         return mlx5dv_dr_table_destroy(tbl);
 460 #else
 461         (void)tbl;
 462         errno = ENOTSUP;
 463         return errno;
 464 #endif
 465 }
 466
 467 static void *
 468 mlx5_glue_dr_create_domain(struct ibv_context *ctx,
 469                            enum  mlx5dv_dr_domain_type domain)
 470 {
 471 #ifdef HAVE_MLX5DV_DR
 472         return mlx5dv_dr_domain_create(ctx, domain);
 473 #else
 474         (void)ctx;
 475         (void)domain;
 476         errno = ENOTSUP;
 477         return NULL;
 478 #endif
 479 }
 480
 481 static int
 482 mlx5_glue_dr_destroy_domain(void *domain)
 483 {
 484 #ifdef HAVE_MLX5DV_DR
 485         return mlx5dv_dr_domain_destroy(domain);
 486 #else
 487         (void)domain;
 488         errno = ENOTSUP;
 489         return errno;
 490 #endif
 491 }
 492
 493 static struct ibv_cq_ex *
 494 mlx5_glue_dv_create_cq(struct ibv_context *context,
 495                        struct ibv_cq_init_attr_ex *cq_attr,
 496                        struct mlx5dv_cq_init_attr *mlx5_cq_attr)
 497 {
 498         return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
 499 }
 500
 501 static struct ibv_wq *
 502 mlx5_glue_dv_create_wq(struct ibv_context *context,
 503                        struct ibv_wq_init_attr *wq_attr,
 504                        struct mlx5dv_wq_init_attr *mlx5_wq_attr)
 505 {
 506 #ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 507         (void)context;
 508         (void)wq_attr;
 509         (void)mlx5_wq_attr;
 510         errno = ENOTSUP;
 511         return NULL;
 512 #else
 513         return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr);
 514 #endif
 515 }
 516
 517 static int
 518 mlx5_glue_dv_query_device(struct ibv_context *ctx,
 519                           struct mlx5dv_context *attrs_out)
 520 {
 521         return mlx5dv_query_device(ctx, attrs_out);
 522 }
 523
 524 static int
 525 mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx,
 526                               enum mlx5dv_set_ctx_attr_type type, void *attr)
 527 {
 528         return mlx5dv_set_context_attr(ibv_ctx, type, attr);
 529 }
 530
 531 static int
 532 mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
 533 {
 534         return mlx5dv_init_obj(obj, obj_type);
 535 }
 536
 537 static struct ibv_qp *
 538 mlx5_glue_dv_create_qp(struct ibv_context *context,
 539                        struct ibv_qp_init_attr_ex *qp_init_attr_ex,
 540                        struct mlx5dv_qp_init_attr *dv_qp_init_attr)
 541 {
 542 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 543         return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr);
 544 #else
 545         (void)context;
 546         (void)qp_init_attr_ex;
 547         (void)dv_qp_init_attr;
 548         errno = ENOTSUP;
 549         return NULL;
 550 #endif
 551 }
 552
 553 static void *
 554 mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
 555                                  struct mlx5dv_flow_matcher_attr *matcher_attr,
 556                                  void *tbl)
 557 {
 558 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 559 #ifdef HAVE_MLX5DV_DR
 560         (void)context;
 561         return mlx5dv_dr_matcher_create(tbl, matcher_attr->priority,
 562                                         matcher_attr->match_criteria_enable,
 563                                         matcher_attr->match_mask);
 564 #else
 565         (void)tbl;
 566         return mlx5dv_create_flow_matcher(context, matcher_attr);
 567 #endif
 568 #else
 569         (void)context;
 570         (void)matcher_attr;
 571         (void)tbl;
 572         errno = ENOTSUP;
 573         return NULL;
 574 #endif
 575 }
 576
 577 static void *
 578 mlx5_glue_dv_create_flow(void *matcher,
 579                          void *match_value,
 580                          size_t num_actions,
 581                          void *actions[])
 582 {
 583 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 584 #ifdef HAVE_MLX5DV_DR
 585         return mlx5dv_dr_rule_create(matcher, match_value, num_actions,
 586                                      (struct mlx5dv_dr_action **)actions);
 587 #else
 588         struct mlx5dv_flow_action_attr actions_attr[8];
 589
 590         if (num_actions > 8)
 591                 return NULL;
 592         for (size_t i = 0; i < num_actions; i++)
 593                 actions_attr[i] =
 594                         *((struct mlx5dv_flow_action_attr *)(actions[i]));
 595         return mlx5dv_create_flow(matcher, match_value,
 596                                   num_actions, actions_attr);
 597 #endif
 598 #else
 599         (void)matcher;
 600         (void)match_value;
 601         (void)num_actions;
 602         (void)actions;
 603         return NULL;
 604 #endif
 605 }
 606
 607 static void *
 608 mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset)
 609 {
 610 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 611 #ifdef HAVE_MLX5DV_DR
 612         return mlx5dv_dr_action_create_flow_counter(counter_obj, offset);
 613 #else
 614         struct mlx5dv_flow_action_attr *action;
 615
 616         (void)offset;
 617         action = malloc(sizeof(*action));
 618         if (!action)
 619                 return NULL;
 620         action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
 621         action->obj = counter_obj;
 622         return action;
 623 #endif
 624 #else
 625         (void)counter_obj;
 626         (void)offset;
 627         errno = ENOTSUP;
 628         return NULL;
 629 #endif
 630 }
 631
 632 static void *
 633 mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp)
 634 {
 635 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 636 #ifdef HAVE_MLX5DV_DR
 637         return mlx5dv_dr_action_create_dest_ibv_qp(qp);
 638 #else
 639         struct mlx5dv_flow_action_attr *action;
 640
 641         action = malloc(sizeof(*action));
 642         if (!action)
 643                 return NULL;
 644         action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
 645         action->obj = qp;
 646         return action;
 647 #endif
 648 #else
 649         (void)qp;
 650         errno = ENOTSUP;
 651         return NULL;
 652 #endif
 653 }
 654
 655 static void *
 656 mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir)
 657 {
 658 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
 659         return mlx5dv_dr_action_create_dest_devx_tir(tir);
 660 #else
 661         (void)tir;
 662         errno = ENOTSUP;
 663         return NULL;
 664 #endif
 665 }
 666
 667 static void *
 668 mlx5_glue_dv_create_flow_action_modify_header
 669                                         (struct ibv_context *ctx,
 670                                          enum mlx5dv_flow_table_type ft_type,
 671                                          void *domain, uint64_t flags,
 672                                          size_t actions_sz,
 673                                          uint64_t actions[])
 674 {
 675 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 676 #ifdef HAVE_MLX5DV_DR
 677         (void)ctx;
 678         (void)ft_type;
 679         return mlx5dv_dr_action_create_modify_header(domain, flags, actions_sz,
 680                                                      (__be64 *)actions);
 681 #else
 682         struct mlx5dv_flow_action_attr *action;
 683
 684         (void)domain;
 685         (void)flags;
 686         action = malloc(sizeof(*action));
 687         if (!action)
 688                 return NULL;
 689         action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
 690         action->action = mlx5dv_create_flow_action_modify_header
 691                 (ctx, actions_sz, actions, ft_type);
 692         return action;
 693 #endif
 694 #else
 695         (void)ctx;
 696         (void)ft_type;
 697         (void)domain;
 698         (void)flags;
 699         (void)actions_sz;
 700         (void)actions;
 701         errno = ENOTSUP;
 702         return NULL;
 703 #endif
 704 }
 705
 706 static void *
 707 mlx5_glue_dv_create_flow_action_packet_reformat
 708                 (struct ibv_context *ctx,
 709                  enum mlx5dv_flow_action_packet_reformat_type reformat_type,
 710                  enum mlx5dv_flow_table_type ft_type,
 711                  struct mlx5dv_dr_domain *domain,
 712                  uint32_t flags, size_t data_sz, void *data)
 713 {
 714 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 715 #ifdef HAVE_MLX5DV_DR
 716         (void)ctx;
 717         (void)ft_type;
 718         return mlx5dv_dr_action_create_packet_reformat(domain, flags,
 719                                                        reformat_type, data_sz,
 720                                                        data);
 721 #else
 722         (void)domain;
 723         (void)flags;
 724         struct mlx5dv_flow_action_attr *action;
 725
 726         action = malloc(sizeof(*action));
 727         if (!action)
 728                 return NULL;
 729         action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
 730         action->action = mlx5dv_create_flow_action_packet_reformat
 731                 (ctx, data_sz, data, reformat_type, ft_type);
 732         return action;
 733 #endif
 734 #else
 735         (void)ctx;
 736         (void)reformat_type;
 737         (void)ft_type;
 738         (void)domain;
 739         (void)flags;
 740         (void)data_sz;
 741         (void)data;
 742         errno = ENOTSUP;
 743         return NULL;
 744 #endif
 745 }
 746
 747 static void *
 748 mlx5_glue_dv_create_flow_action_tag(uint32_t tag)
 749 {
 750 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 751 #ifdef HAVE_MLX5DV_DR
 752         return mlx5dv_dr_action_create_tag(tag);
 753 #else
 754         struct mlx5dv_flow_action_attr *action;
 755         action = malloc(sizeof(*action));
 756         if (!action)
 757                 return NULL;
 758         action->type = MLX5DV_FLOW_ACTION_TAG;
 759         action->tag_value = tag;
 760         return action;
 761 #endif
 762 #endif
 763         (void)tag;
 764         errno = ENOTSUP;
 765         return NULL;
 766 }
 767
 768 static int
 769 mlx5_glue_dv_destroy_flow(void *flow_id)
 770 {
 771 #ifdef HAVE_MLX5DV_DR
 772         return mlx5dv_dr_rule_destroy(flow_id);
 773 #else
 774         return ibv_destroy_flow(flow_id);
 775 #endif
 776 }
 777
 778 static int
 779 mlx5_glue_dv_destroy_flow_matcher(void *matcher)
 780 {
 781 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 782 #ifdef HAVE_MLX5DV_DR
 783         return mlx5dv_dr_matcher_destroy(matcher);
 784 #else
 785         return mlx5dv_destroy_flow_matcher(matcher);
 786 #endif
 787 #else
 788         (void)matcher;
 789         errno = ENOTSUP;
 790         return errno;
 791 #endif
 792 }
 793
 794 static struct ibv_context *
 795 mlx5_glue_dv_open_device(struct ibv_device *device)
 796 {
 797 #ifdef HAVE_IBV_DEVX_OBJ
 798         return mlx5dv_open_device(device,
 799                                   &(struct mlx5dv_context_attr){
 800                                         .flags = MLX5DV_CONTEXT_FLAGS_DEVX,
 801                                   });
 802 #else
 803         (void)device;
 804         errno = ENOTSUP;
 805         return NULL;
 806 #endif
 807 }
 808
 809 static struct mlx5dv_devx_obj *
 810 mlx5_glue_devx_obj_create(struct ibv_context *ctx,
 811                           const void *in, size_t inlen,
 812                           void *out, size_t outlen)
 813 {
 814 #ifdef HAVE_IBV_DEVX_OBJ
 815         return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen);
 816 #else
 817         (void)ctx;
 818         (void)in;
 819         (void)inlen;
 820         (void)out;
 821         (void)outlen;
 822         errno = ENOTSUP;
 823         return NULL;
 824 #endif
 825 }
 826
 827 static int
 828 mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj)
 829 {
 830 #ifdef HAVE_IBV_DEVX_OBJ
 831         return mlx5dv_devx_obj_destroy(obj);
 832 #else
 833         (void)obj;
 834         return -ENOTSUP;
 835 #endif
 836 }
 837
 838 static int
 839 mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj,
 840                          const void *in, size_t inlen,
 841                          void *out, size_t outlen)
 842 {
 843 #ifdef HAVE_IBV_DEVX_OBJ
 844         return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen);
 845 #else
 846         (void)obj;
 847         (void)in;
 848         (void)inlen;
 849         (void)out;
 850         (void)outlen;
 851         return -ENOTSUP;
 852 #endif
 853 }
 854
 855 static int
 856 mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj,
 857                           const void *in, size_t inlen,
 858                           void *out, size_t outlen)
 859 {
 860 #ifdef HAVE_IBV_DEVX_OBJ
 861         return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen);
 862 #else
 863         (void)obj;
 864         (void)in;
 865         (void)inlen;
 866         (void)out;
 867         (void)outlen;
 868         return -ENOTSUP;
 869 #endif
 870 }
 871
 872 static int
 873 mlx5_glue_devx_general_cmd(struct ibv_context *ctx,
 874                            const void *in, size_t inlen,
 875                            void *out, size_t outlen)
 876 {
 877 #ifdef HAVE_IBV_DEVX_OBJ
 878         return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen);
 879 #else
 880         (void)ctx;
 881         (void)in;
 882         (void)inlen;
 883         (void)out;
 884         (void)outlen;
 885         return -ENOTSUP;
 886 #endif
 887 }
 888
 889 static struct mlx5dv_devx_cmd_comp *
 890 mlx5_glue_devx_create_cmd_comp(struct ibv_context *ctx)
 891 {
 892 #ifdef HAVE_IBV_DEVX_ASYNC
 893         return mlx5dv_devx_create_cmd_comp(ctx);
 894 #else
 895         (void)ctx;
 896         errno = -ENOTSUP;
 897         return NULL;
 898 #endif
 899 }
 900
 901 static void
 902 mlx5_glue_devx_destroy_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp)
 903 {
 904 #ifdef HAVE_IBV_DEVX_ASYNC
 905         mlx5dv_devx_destroy_cmd_comp(cmd_comp);
 906 #else
 907         (void)cmd_comp;
 908         errno = -ENOTSUP;
 909 #endif
 910 }
 911
 912 static int
 913 mlx5_glue_devx_obj_query_async(struct mlx5dv_devx_obj *obj, const void *in,
 914                                size_t inlen, size_t outlen, uint64_t wr_id,
 915                                struct mlx5dv_devx_cmd_comp *cmd_comp)
 916 {
 917 #ifdef HAVE_IBV_DEVX_ASYNC
 918         return mlx5dv_devx_obj_query_async(obj, in, inlen, outlen, wr_id,
 919                                            cmd_comp);
 920 #else
 921         (void)obj;
 922         (void)in;
 923         (void)inlen;
 924         (void)outlen;
 925         (void)wr_id;
 926         (void)cmd_comp;
 927         return -ENOTSUP;
 928 #endif
 929 }
 930
 931 static int
 932 mlx5_glue_devx_get_async_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp,
 933                                   struct mlx5dv_devx_async_cmd_hdr *cmd_resp,
 934                                   size_t cmd_resp_len)
 935 {
 936 #ifdef HAVE_IBV_DEVX_ASYNC
 937         return mlx5dv_devx_get_async_cmd_comp(cmd_comp, cmd_resp,
 938                                               cmd_resp_len);
 939 #else
 940         (void)cmd_comp;
 941         (void)cmd_resp;
 942         (void)cmd_resp_len;
 943         return -ENOTSUP;
 944 #endif
 945 }
 946
 947 static struct mlx5dv_devx_umem *
 948 mlx5_glue_devx_umem_reg(struct ibv_context *context, void *addr, size_t size,
 949                         uint32_t access)
 950 {
 951 #ifdef HAVE_IBV_DEVX_OBJ
 952         return mlx5dv_devx_umem_reg(context, addr, size, access);
 953 #else
 954         (void)context;
 955         (void)addr;
 956         (void)size;
 957         (void)access;
 958         errno = -ENOTSUP;
 959         return NULL;
 960 #endif
 961 }
 962
 963 static int
 964 mlx5_glue_devx_umem_dereg(struct mlx5dv_devx_umem *dv_devx_umem)
 965 {
 966 #ifdef HAVE_IBV_DEVX_OBJ
 967         return mlx5dv_devx_umem_dereg(dv_devx_umem);
 968 #else
 969         (void)dv_devx_umem;
 970         return -ENOTSUP;
 971 #endif
 972 }
 973
 974 static int
 975 mlx5_glue_devx_qp_query(struct ibv_qp *qp,
 976                         const void *in, size_t inlen,
 977                         void *out, size_t outlen)
 978 {
 979 #ifdef HAVE_IBV_DEVX_OBJ
 980         return mlx5dv_devx_qp_query(qp, in, inlen, out, outlen);
 981 #else
 982         (void)qp;
 983         (void)in;
 984         (void)inlen;
 985         (void)out;
 986         (void)outlen;
 987         errno = ENOTSUP;
 988         return errno;
 989 #endif
 990 }
 991
 992 static int
 993 mlx5_glue_devx_port_query(struct ibv_context *ctx,
 994                           uint32_t port_num,
 995                           struct mlx5dv_devx_port *mlx5_devx_port)
 996 {
 997 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
 998         return mlx5dv_query_devx_port(ctx, port_num, mlx5_devx_port);
 999 #else
1000         (void)ctx;
1001         (void)port_num;
1002         (void)mlx5_devx_port;
1003         errno = ENOTSUP;
1004         return errno;
1005 #endif
1006 }
1007
1008 alignas(RTE_CACHE_LINE_SIZE)
1009 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
1010         .version = MLX5_GLUE_VERSION,
1011         .fork_init = mlx5_glue_fork_init,
1012         .alloc_pd = mlx5_glue_alloc_pd,
1013         .dealloc_pd = mlx5_glue_dealloc_pd,
1014         .get_device_list = mlx5_glue_get_device_list,
1015         .free_device_list = mlx5_glue_free_device_list,
1016         .open_device = mlx5_glue_open_device,
1017         .close_device = mlx5_glue_close_device,
1018         .query_device = mlx5_glue_query_device,
1019         .query_device_ex = mlx5_glue_query_device_ex,
1020         .query_rt_values_ex = mlx5_glue_query_rt_values_ex,
1021         .query_port = mlx5_glue_query_port,
1022         .create_comp_channel = mlx5_glue_create_comp_channel,
1023         .destroy_comp_channel = mlx5_glue_destroy_comp_channel,
1024         .create_cq = mlx5_glue_create_cq,
1025         .destroy_cq = mlx5_glue_destroy_cq,
1026         .get_cq_event = mlx5_glue_get_cq_event,
1027         .ack_cq_events = mlx5_glue_ack_cq_events,
1028         .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table,
1029         .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table,
1030         .create_wq = mlx5_glue_create_wq,
1031         .destroy_wq = mlx5_glue_destroy_wq,
1032         .modify_wq = mlx5_glue_modify_wq,
1033         .create_flow = mlx5_glue_create_flow,
1034         .destroy_flow = mlx5_glue_destroy_flow,
1035         .destroy_flow_action = mlx5_glue_destroy_flow_action,
1036         .create_qp = mlx5_glue_create_qp,
1037         .create_qp_ex = mlx5_glue_create_qp_ex,
1038         .destroy_qp = mlx5_glue_destroy_qp,
1039         .modify_qp = mlx5_glue_modify_qp,
1040         .reg_mr = mlx5_glue_reg_mr,
1041         .dereg_mr = mlx5_glue_dereg_mr,
1042         .create_counter_set = mlx5_glue_create_counter_set,
1043         .destroy_counter_set = mlx5_glue_destroy_counter_set,
1044         .describe_counter_set = mlx5_glue_describe_counter_set,
1045         .query_counter_set = mlx5_glue_query_counter_set,
1046         .create_counters = mlx5_glue_create_counters,
1047         .destroy_counters = mlx5_glue_destroy_counters,
1048         .attach_counters = mlx5_glue_attach_counters,
1049         .query_counters = mlx5_glue_query_counters,
1050         .ack_async_event = mlx5_glue_ack_async_event,
1051         .get_async_event = mlx5_glue_get_async_event,
1052         .port_state_str = mlx5_glue_port_state_str,
1053         .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
1054         .dr_create_flow_action_dest_flow_tbl =
1055                 mlx5_glue_dr_create_flow_action_dest_flow_tbl,
1056         .dr_create_flow_action_dest_vport =
1057                 mlx5_glue_dr_create_flow_action_dest_vport,
1058         .dr_create_flow_action_drop =
1059                 mlx5_glue_dr_create_flow_action_drop,
1060         .dr_create_flow_action_push_vlan =
1061                 mlx5_glue_dr_create_flow_action_push_vlan,
1062         .dr_create_flow_action_pop_vlan =
1063                 mlx5_glue_dr_create_flow_action_pop_vlan,
1064         .dr_create_flow_tbl = mlx5_glue_dr_create_flow_tbl,
1065         .dr_destroy_flow_tbl = mlx5_glue_dr_destroy_flow_tbl,
1066         .dr_create_domain = mlx5_glue_dr_create_domain,
1067         .dr_destroy_domain = mlx5_glue_dr_destroy_domain,
1068         .dv_create_cq = mlx5_glue_dv_create_cq,
1069         .dv_create_wq = mlx5_glue_dv_create_wq,
1070         .dv_query_device = mlx5_glue_dv_query_device,
1071         .dv_set_context_attr = mlx5_glue_dv_set_context_attr,
1072         .dv_init_obj = mlx5_glue_dv_init_obj,
1073         .dv_create_qp = mlx5_glue_dv_create_qp,
1074         .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
1075         .dv_create_flow = mlx5_glue_dv_create_flow,
1076         .dv_create_flow_action_counter =
1077                 mlx5_glue_dv_create_flow_action_counter,
1078         .dv_create_flow_action_dest_ibv_qp =
1079                 mlx5_glue_dv_create_flow_action_dest_ibv_qp,
1080         .dv_create_flow_action_dest_devx_tir =
1081                 mlx5_glue_dv_create_flow_action_dest_devx_tir,
1082         .dv_create_flow_action_modify_header =
1083                 mlx5_glue_dv_create_flow_action_modify_header,
1084         .dv_create_flow_action_packet_reformat =
1085                 mlx5_glue_dv_create_flow_action_packet_reformat,
1086         .dv_create_flow_action_tag =  mlx5_glue_dv_create_flow_action_tag,
1087         .dv_destroy_flow = mlx5_glue_dv_destroy_flow,
1088         .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
1089         .dv_open_device = mlx5_glue_dv_open_device,
1090         .devx_obj_create = mlx5_glue_devx_obj_create,
1091         .devx_obj_destroy = mlx5_glue_devx_obj_destroy,
1092         .devx_obj_query = mlx5_glue_devx_obj_query,
1093         .devx_obj_modify = mlx5_glue_devx_obj_modify,
1094         .devx_general_cmd = mlx5_glue_devx_general_cmd,
1095         .devx_create_cmd_comp = mlx5_glue_devx_create_cmd_comp,
1096         .devx_destroy_cmd_comp = mlx5_glue_devx_destroy_cmd_comp,
1097         .devx_obj_query_async = mlx5_glue_devx_obj_query_async,
1098         .devx_get_async_cmd_comp = mlx5_glue_devx_get_async_cmd_comp,
1099         .devx_umem_reg = mlx5_glue_devx_umem_reg,
1100         .devx_umem_dereg = mlx5_glue_devx_umem_dereg,
1101         .devx_qp_query = mlx5_glue_devx_qp_query,
1102         .devx_port_query = mlx5_glue_devx_port_query,
1103 };