drivers/net/mlx5/mlx5.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <unistd.h>
   8 #include <string.h>
   9 #include <assert.h>
  10 #include <dlfcn.h>
  11 #include <stdint.h>
  12 #include <stdlib.h>
  13 #include <errno.h>
  14 #include <net/if.h>
  15 #include <sys/mman.h>
  16 #include <linux/rtnetlink.h>
  17
  18 /* Verbs header. */
  19 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  20 #ifdef PEDANTIC
  21 #pragma GCC diagnostic ignored "-Wpedantic"
  22 #endif
  23 #include <infiniband/verbs.h>
  24 #ifdef PEDANTIC
  25 #pragma GCC diagnostic error "-Wpedantic"
  26 #endif
  27
  28 #include <rte_malloc.h>
  29 #include <rte_ethdev_driver.h>
  30 #include <rte_ethdev_pci.h>
  31 #include <rte_pci.h>
  32 #include <rte_bus_pci.h>
  33 #include <rte_common.h>
  34 #include <rte_config.h>
  35 #include <rte_kvargs.h>
  36 #include <rte_rwlock.h>
  37 #include <rte_spinlock.h>
  38 #include <rte_string_fns.h>
  39 #include <rte_alarm.h>
  40
  41 #include "mlx5.h"
  42 #include "mlx5_utils.h"
  43 #include "mlx5_rxtx.h"
  44 #include "mlx5_autoconf.h"
  45 #include "mlx5_defs.h"
  46 #include "mlx5_glue.h"
  47 #include "mlx5_mr.h"
  48 #include "mlx5_flow.h"
  49
  50 /* Device parameter to enable RX completion queue compression. */
  51 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
  52
  53 /* Device parameter to enable RX completion entry padding to 128B. */
  54 #define MLX5_RXQ_CQE_PAD_EN "rxq_cqe_pad_en"
  55
  56 /* Device parameter to enable padding Rx packet to cacheline size. */
  57 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en"
  58
  59 /* Device parameter to enable Multi-Packet Rx queue. */
  60 #define MLX5_RX_MPRQ_EN "mprq_en"
  61
  62 /* Device parameter to configure log 2 of the number of strides for MPRQ. */
  63 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num"
  64
  65 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */
  66 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len"
  67
  68 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */
  69 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq"
  70
  71 /* Device parameter to configure inline send. Deprecated, ignored.*/
  72 #define MLX5_TXQ_INLINE "txq_inline"
  73
  74 /* Device parameter to limit packet size to inline with ordinary SEND. */
  75 #define MLX5_TXQ_INLINE_MAX "txq_inline_max"
  76
  77 /* Device parameter to configure minimal data size to inline. */
  78 #define MLX5_TXQ_INLINE_MIN "txq_inline_min"
  79
  80 /* Device parameter to limit packet size to inline with Enhanced MPW. */
  81 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw"
  82
  83 /*
  84  * Device parameter to configure the number of TX queues threshold for
  85  * enabling inline send.
  86  */
  87 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline"
  88
  89 /*
  90  * Device parameter to configure the number of TX queues threshold for
  91  * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines).
  92  */
  93 #define MLX5_TXQS_MAX_VEC "txqs_max_vec"
  94
  95 /* Device parameter to enable multi-packet send WQEs. */
  96 #define MLX5_TXQ_MPW_EN "txq_mpw_en"
  97
  98 /*
  99  * Device parameter to force doorbell register mapping
 100  * to non-cahed region eliminating the extra write memory barrier.
 101  */
 102 #define MLX5_TX_DB_NC "tx_db_nc"
 103
 104 /*
 105  * Device parameter to include 2 dsegs in the title WQEBB.
 106  * Deprecated, ignored.
 107  */
 108 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en"
 109
 110 /*
 111  * Device parameter to limit the size of inlining packet.
 112  * Deprecated, ignored.
 113  */
 114 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len"
 115
 116 /*
 117  * Device parameter to enable hardware Tx vector.
 118  * Deprecated, ignored (no vectorized Tx routines anymore).
 119  */
 120 #define MLX5_TX_VEC_EN "tx_vec_en"
 121
 122 /* Device parameter to enable hardware Rx vector. */
 123 #define MLX5_RX_VEC_EN "rx_vec_en"
 124
 125 /* Allow L3 VXLAN flow creation. */
 126 #define MLX5_L3_VXLAN_EN "l3_vxlan_en"
 127
 128 /* Activate DV E-Switch flow steering. */
 129 #define MLX5_DV_ESW_EN "dv_esw_en"
 130
 131 /* Activate DV flow steering. */
 132 #define MLX5_DV_FLOW_EN "dv_flow_en"
 133
 134 /* Enable extensive flow metadata support. */
 135 #define MLX5_DV_XMETA_EN "dv_xmeta_en"
 136
 137 /* Activate Netlink support in VF mode. */
 138 #define MLX5_VF_NL_EN "vf_nl_en"
 139
 140 /* Enable extending memsegs when creating a MR. */
 141 #define MLX5_MR_EXT_MEMSEG_EN "mr_ext_memseg_en"
 142
 143 /* Select port representors to instantiate. */
 144 #define MLX5_REPRESENTOR "representor"
 145
 146 /* Device parameter to configure the maximum number of dump files per queue. */
 147 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num"
 148
 149 /* Configure timeout of LRO session (in microseconds). */
 150 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec"
 151
 152 #ifndef HAVE_IBV_MLX5_MOD_MPW
 153 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 154 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
 155 #endif
 156
 157 #ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP
 158 #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
 159 #endif
 160
 161 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 162
 163 /* Shared memory between primary and secondary processes. */
 164 struct mlx5_shared_data *mlx5_shared_data;
 165
 166 /* Spinlock for mlx5_shared_data allocation. */
 167 static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 168
 169 /* Process local data for secondary processes. */
 170 static struct mlx5_local_data mlx5_local_data;
 171
 172 /** Driver-specific log messages type. */
 173 int mlx5_logtype;
 174
 175 /** Data associated with devices to spawn. */
 176 struct mlx5_dev_spawn_data {
 177         uint32_t ifindex; /**< Network interface index. */
 178         uint32_t max_port; /**< IB device maximal port index. */
 179         uint32_t ibv_port; /**< IB device physical port index. */
 180         int pf_bond; /**< bonding device PF index. < 0 - no bonding */
 181         struct mlx5_switch_info info; /**< Switch information. */
 182         struct ibv_device *ibv_dev; /**< Associated IB device. */
 183         struct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */
 184         struct rte_pci_device *pci_dev; /**< Backend PCI device. */
 185 };
 186
 187 static LIST_HEAD(, mlx5_ibv_shared) mlx5_ibv_list = LIST_HEAD_INITIALIZER();
 188 static pthread_mutex_t mlx5_ibv_list_mutex = PTHREAD_MUTEX_INITIALIZER;
 189
 190 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 191 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 192
 193 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
 194 #define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
 195
 196 /**
 197  * Allocate ID pool structure.
 198  *
 199  * @param[in] max_id
 200  *   The maximum id can be allocated from the pool.
 201  *
 202  * @return
 203  *   Pointer to pool object, NULL value otherwise.
 204  */
 205 struct mlx5_flow_id_pool *
 206 mlx5_flow_id_pool_alloc(uint32_t max_id)
 207 {
 208         struct mlx5_flow_id_pool *pool;
 209         void *mem;
 210
 211         pool = rte_zmalloc("id pool allocation", sizeof(*pool),
 212                            RTE_CACHE_LINE_SIZE);
 213         if (!pool) {
 214                 DRV_LOG(ERR, "can't allocate id pool");
 215                 rte_errno  = ENOMEM;
 216                 return NULL;
 217         }
 218         mem = rte_zmalloc("", MLX5_FLOW_MIN_ID_POOL_SIZE * sizeof(uint32_t),
 219                           RTE_CACHE_LINE_SIZE);
 220         if (!mem) {
 221                 DRV_LOG(ERR, "can't allocate mem for id pool");
 222                 rte_errno  = ENOMEM;
 223                 goto error;
 224         }
 225         pool->free_arr = mem;
 226         pool->curr = pool->free_arr;
 227         pool->last = pool->free_arr + MLX5_FLOW_MIN_ID_POOL_SIZE;
 228         pool->base_index = 0;
 229         pool->max_id = max_id;
 230         return pool;
 231 error:
 232         rte_free(pool);
 233         return NULL;
 234 }
 235
 236 /**
 237  * Release ID pool structure.
 238  *
 239  * @param[in] pool
 240  *   Pointer to flow id pool object to free.
 241  */
 242 void
 243 mlx5_flow_id_pool_release(struct mlx5_flow_id_pool *pool)
 244 {
 245         rte_free(pool->free_arr);
 246         rte_free(pool);
 247 }
 248
 249 /**
 250  * Generate ID.
 251  *
 252  * @param[in] pool
 253  *   Pointer to flow id pool.
 254  * @param[out] id
 255  *   The generated ID.
 256  *
 257  * @return
 258  *   0 on success, error value otherwise.
 259  */
 260 uint32_t
 261 mlx5_flow_id_get(struct mlx5_flow_id_pool *pool, uint32_t *id)
 262 {
 263         if (pool->curr == pool->free_arr) {
 264                 if (pool->base_index == pool->max_id) {
 265                         rte_errno  = ENOMEM;
 266                         DRV_LOG(ERR, "no free id");
 267                         return -rte_errno;
 268                 }
 269                 *id = ++pool->base_index;
 270                 return 0;
 271         }
 272         *id = *(--pool->curr);
 273         return 0;
 274 }
 275
 276 /**
 277  * Release ID.
 278  *
 279  * @param[in] pool
 280  *   Pointer to flow id pool.
 281  * @param[out] id
 282  *   The generated ID.
 283  *
 284  * @return
 285  *   0 on success, error value otherwise.
 286  */
 287 uint32_t
 288 mlx5_flow_id_release(struct mlx5_flow_id_pool *pool, uint32_t id)
 289 {
 290         uint32_t size;
 291         uint32_t size2;
 292         void *mem;
 293
 294         if (pool->curr == pool->last) {
 295                 size = pool->curr - pool->free_arr;
 296                 size2 = size * MLX5_ID_GENERATION_ARRAY_FACTOR;
 297                 assert(size2 > size);
 298                 mem = rte_malloc("", size2 * sizeof(uint32_t), 0);
 299                 if (!mem) {
 300                         DRV_LOG(ERR, "can't allocate mem for id pool");
 301                         rte_errno  = ENOMEM;
 302                         return -rte_errno;
 303                 }
 304                 memcpy(mem, pool->free_arr, size * sizeof(uint32_t));
 305                 rte_free(pool->free_arr);
 306                 pool->free_arr = mem;
 307                 pool->curr = pool->free_arr + size;
 308                 pool->last = pool->free_arr + size2;
 309         }
 310         *pool->curr = id;
 311         pool->curr++;
 312         return 0;
 313 }
 314
 315 /**
 316  * Initialize the counters management structure.
 317  *
 318  * @param[in] sh
 319  *   Pointer to mlx5_ibv_shared object to free
 320  */
 321 static void
 322 mlx5_flow_counters_mng_init(struct mlx5_ibv_shared *sh)
 323 {
 324         uint8_t i;
 325
 326         TAILQ_INIT(&sh->cmng.flow_counters);
 327         for (i = 0; i < RTE_DIM(sh->cmng.ccont); ++i)
 328                 TAILQ_INIT(&sh->cmng.ccont[i].pool_list);
 329 }
 330
 331 /**
 332  * Destroy all the resources allocated for a counter memory management.
 333  *
 334  * @param[in] mng
 335  *   Pointer to the memory management structure.
 336  */
 337 static void
 338 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng)
 339 {
 340         uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data;
 341
 342         LIST_REMOVE(mng, next);
 343         claim_zero(mlx5_devx_cmd_destroy(mng->dm));
 344         claim_zero(mlx5_glue->devx_umem_dereg(mng->umem));
 345         rte_free(mem);
 346 }
 347
 348 /**
 349  * Close and release all the resources of the counters management.
 350  *
 351  * @param[in] sh
 352  *   Pointer to mlx5_ibv_shared object to free.
 353  */
 354 static void
 355 mlx5_flow_counters_mng_close(struct mlx5_ibv_shared *sh)
 356 {
 357         struct mlx5_counter_stats_mem_mng *mng;
 358         uint8_t i;
 359         int j;
 360         int retries = 1024;
 361
 362         rte_errno = 0;
 363         while (--retries) {
 364                 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh);
 365                 if (rte_errno != EINPROGRESS)
 366                         break;
 367                 rte_pause();
 368         }
 369         for (i = 0; i < RTE_DIM(sh->cmng.ccont); ++i) {
 370                 struct mlx5_flow_counter_pool *pool;
 371                 uint32_t batch = !!(i % 2);
 372
 373                 if (!sh->cmng.ccont[i].pools)
 374                         continue;
 375                 pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
 376                 while (pool) {
 377                         if (batch) {
 378                                 if (pool->min_dcs)
 379                                         claim_zero
 380                                         (mlx5_devx_cmd_destroy(pool->min_dcs));
 381                         }
 382                         for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) {
 383                                 if (pool->counters_raw[j].action)
 384                                         claim_zero
 385                                         (mlx5_glue->destroy_flow_action
 386                                                (pool->counters_raw[j].action));
 387                                 if (!batch && pool->counters_raw[j].dcs)
 388                                         claim_zero(mlx5_devx_cmd_destroy
 389                                                   (pool->counters_raw[j].dcs));
 390                         }
 391                         TAILQ_REMOVE(&sh->cmng.ccont[i].pool_list, pool,
 392                                      next);
 393                         rte_free(pool);
 394                         pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
 395                 }
 396                 rte_free(sh->cmng.ccont[i].pools);
 397         }
 398         mng = LIST_FIRST(&sh->cmng.mem_mngs);
 399         while (mng) {
 400                 mlx5_flow_destroy_counter_stat_mem_mng(mng);
 401                 mng = LIST_FIRST(&sh->cmng.mem_mngs);
 402         }
 403         memset(&sh->cmng, 0, sizeof(sh->cmng));
 404 }
 405
 406 /**
 407  * Extract pdn of PD object using DV API.
 408  *
 409  * @param[in] pd
 410  *   Pointer to the verbs PD object.
 411  * @param[out] pdn
 412  *   Pointer to the PD object number variable.
 413  *
 414  * @return
 415  *   0 on success, error value otherwise.
 416  */
 417 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 418 static int
 419 mlx5_get_pdn(struct ibv_pd *pd __rte_unused, uint32_t *pdn __rte_unused)
 420 {
 421         struct mlx5dv_obj obj;
 422         struct mlx5dv_pd pd_info;
 423         int ret = 0;
 424
 425         obj.pd.in = pd;
 426         obj.pd.out = &pd_info;
 427         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 428         if (ret) {
 429                 DRV_LOG(DEBUG, "Fail to get PD object info");
 430                 return ret;
 431         }
 432         *pdn = pd_info.pdn;
 433         return 0;
 434 }
 435 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 436
 437 static int
 438 mlx5_config_doorbell_mapping_env(const struct mlx5_dev_config *config)
 439 {
 440         char *env;
 441         int value;
 442
 443         assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
 444         /* Get environment variable to store. */
 445         env = getenv(MLX5_SHUT_UP_BF);
 446         value = env ? !!strcmp(env, "0") : MLX5_ARG_UNSET;
 447         if (config->dbnc == MLX5_ARG_UNSET)
 448                 setenv(MLX5_SHUT_UP_BF, MLX5_SHUT_UP_BF_DEFAULT, 1);
 449         else
 450                 setenv(MLX5_SHUT_UP_BF,
 451                        config->dbnc == MLX5_TXDB_NCACHED ? "1" : "0", 1);
 452         return value;
 453 }
 454
 455 static void
 456 mlx5_restore_doorbell_mapping_env(int value)
 457 {
 458         assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
 459         /* Restore the original environment variable state. */
 460         if (value == MLX5_ARG_UNSET)
 461                 unsetenv(MLX5_SHUT_UP_BF);
 462         else
 463                 setenv(MLX5_SHUT_UP_BF, value ? "1" : "0", 1);
 464 }
 465
 466 /**
 467  * Allocate shared IB device context. If there is multiport device the
 468  * master and representors will share this context, if there is single
 469  * port dedicated IB device, the context will be used by only given
 470  * port due to unification.
 471  *
 472  * Routine first searches the context for the specified IB device name,
 473  * if found the shared context assumed and reference counter is incremented.
 474  * If no context found the new one is created and initialized with specified
 475  * IB device context and parameters.
 476  *
 477  * @param[in] spawn
 478  *   Pointer to the IB device attributes (name, port, etc).
 479  * @param[in] config
 480  *   Pointer to device configuration structure.
 481  *
 482  * @return
 483  *   Pointer to mlx5_ibv_shared object on success,
 484  *   otherwise NULL and rte_errno is set.
 485  */
 486 static struct mlx5_ibv_shared *
 487 mlx5_alloc_shared_ibctx(const struct mlx5_dev_spawn_data *spawn,
 488                         const struct mlx5_dev_config *config)
 489 {
 490         struct mlx5_ibv_shared *sh;
 491         int dbmap_env;
 492         int err = 0;
 493         uint32_t i;
 494 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 495         struct mlx5_devx_tis_attr tis_attr = { 0 };
 496 #endif
 497
 498         assert(spawn);
 499         /* Secondary process should not create the shared context. */
 500         assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
 501         pthread_mutex_lock(&mlx5_ibv_list_mutex);
 502         /* Search for IB context by device name. */
 503         LIST_FOREACH(sh, &mlx5_ibv_list, next) {
 504                 if (!strcmp(sh->ibdev_name, spawn->ibv_dev->name)) {
 505                         sh->refcnt++;
 506                         goto exit;
 507                 }
 508         }
 509         /* No device found, we have to create new shared context. */
 510         assert(spawn->max_port);
 511         sh = rte_zmalloc("ethdev shared ib context",
 512                          sizeof(struct mlx5_ibv_shared) +
 513                          spawn->max_port *
 514                          sizeof(struct mlx5_ibv_shared_port),
 515                          RTE_CACHE_LINE_SIZE);
 516         if (!sh) {
 517                 DRV_LOG(ERR, "shared context allocation failure");
 518                 rte_errno  = ENOMEM;
 519                 goto exit;
 520         }
 521         /*
 522          * Configure environment variable "MLX5_BF_SHUT_UP"
 523          * before the device creation. The rdma_core library
 524          * checks the variable at device creation and
 525          * stores the result internally.
 526          */
 527         dbmap_env = mlx5_config_doorbell_mapping_env(config);
 528         /* Try to open IB device with DV first, then usual Verbs. */
 529         errno = 0;
 530         sh->ctx = mlx5_glue->dv_open_device(spawn->ibv_dev);
 531         if (sh->ctx) {
 532                 sh->devx = 1;
 533                 DRV_LOG(DEBUG, "DevX is supported");
 534                 /* The device is created, no need for environment. */
 535                 mlx5_restore_doorbell_mapping_env(dbmap_env);
 536         } else {
 537                 /* The environment variable is still configured. */
 538                 sh->ctx = mlx5_glue->open_device(spawn->ibv_dev);
 539                 err = errno ? errno : ENODEV;
 540                 /*
 541                  * The environment variable is not needed anymore,
 542                  * all device creation attempts are completed.
 543                  */
 544                 mlx5_restore_doorbell_mapping_env(dbmap_env);
 545                 if (!sh->ctx)
 546                         goto error;
 547                 DRV_LOG(DEBUG, "DevX is NOT supported");
 548         }
 549         err = mlx5_glue->query_device_ex(sh->ctx, NULL, &sh->device_attr);
 550         if (err) {
 551                 DRV_LOG(DEBUG, "ibv_query_device_ex() failed");
 552                 goto error;
 553         }
 554         sh->refcnt = 1;
 555         sh->max_port = spawn->max_port;
 556         strncpy(sh->ibdev_name, sh->ctx->device->name,
 557                 sizeof(sh->ibdev_name));
 558         strncpy(sh->ibdev_path, sh->ctx->device->ibdev_path,
 559                 sizeof(sh->ibdev_path));
 560         pthread_mutex_init(&sh->intr_mutex, NULL);
 561         /*
 562          * Setting port_id to max unallowed value means
 563          * there is no interrupt subhandler installed for
 564          * the given port index i.
 565          */
 566         for (i = 0; i < sh->max_port; i++) {
 567                 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS;
 568                 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS;
 569         }
 570         sh->pd = mlx5_glue->alloc_pd(sh->ctx);
 571         if (sh->pd == NULL) {
 572                 DRV_LOG(ERR, "PD allocation failure");
 573                 err = ENOMEM;
 574                 goto error;
 575         }
 576 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 577         if (sh->devx) {
 578                 err = mlx5_get_pdn(sh->pd, &sh->pdn);
 579                 if (err) {
 580                         DRV_LOG(ERR, "Fail to extract pdn from PD");
 581                         goto error;
 582                 }
 583                 sh->td = mlx5_devx_cmd_create_td(sh->ctx);
 584                 if (!sh->td) {
 585                         DRV_LOG(ERR, "TD allocation failure");
 586                         err = ENOMEM;
 587                         goto error;
 588                 }
 589                 tis_attr.transport_domain = sh->td->id;
 590                 sh->tis = mlx5_devx_cmd_create_tis(sh->ctx, &tis_attr);
 591                 if (!sh->tis) {
 592                         DRV_LOG(ERR, "TIS allocation failure");
 593                         err = ENOMEM;
 594                         goto error;
 595                 }
 596         }
 597         sh->flow_id_pool = mlx5_flow_id_pool_alloc(UINT32_MAX);
 598         if (!sh->flow_id_pool) {
 599                 DRV_LOG(ERR, "can't create flow id pool");
 600                 err = ENOMEM;
 601                 goto error;
 602         }
 603 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 604         /*
 605          * Once the device is added to the list of memory event
 606          * callback, its global MR cache table cannot be expanded
 607          * on the fly because of deadlock. If it overflows, lookup
 608          * should be done by searching MR list linearly, which is slow.
 609          *
 610          * At this point the device is not added to the memory
 611          * event list yet, context is just being created.
 612          */
 613         err = mlx5_mr_btree_init(&sh->mr.cache,
 614                                  MLX5_MR_BTREE_CACHE_N * 2,
 615                                  spawn->pci_dev->device.numa_node);
 616         if (err) {
 617                 err = rte_errno;
 618                 goto error;
 619         }
 620         mlx5_flow_counters_mng_init(sh);
 621         /* Add device to memory callback list. */
 622         rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
 623         LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
 624                          sh, mem_event_cb);
 625         rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
 626         /* Add context to the global device list. */
 627         LIST_INSERT_HEAD(&mlx5_ibv_list, sh, next);
 628 exit:
 629         pthread_mutex_unlock(&mlx5_ibv_list_mutex);
 630         return sh;
 631 error:
 632         pthread_mutex_unlock(&mlx5_ibv_list_mutex);
 633         assert(sh);
 634         if (sh->tis)
 635                 claim_zero(mlx5_devx_cmd_destroy(sh->tis));
 636         if (sh->td)
 637                 claim_zero(mlx5_devx_cmd_destroy(sh->td));
 638         if (sh->pd)
 639                 claim_zero(mlx5_glue->dealloc_pd(sh->pd));
 640         if (sh->ctx)
 641                 claim_zero(mlx5_glue->close_device(sh->ctx));
 642         if (sh->flow_id_pool)
 643                 mlx5_flow_id_pool_release(sh->flow_id_pool);
 644         rte_free(sh);
 645         assert(err > 0);
 646         rte_errno = err;
 647         return NULL;
 648 }
 649
 650 /**
 651  * Free shared IB device context. Decrement counter and if zero free
 652  * all allocated resources and close handles.
 653  *
 654  * @param[in] sh
 655  *   Pointer to mlx5_ibv_shared object to free
 656  */
 657 static void
 658 mlx5_free_shared_ibctx(struct mlx5_ibv_shared *sh)
 659 {
 660         pthread_mutex_lock(&mlx5_ibv_list_mutex);
 661 #ifndef NDEBUG
 662         /* Check the object presence in the list. */
 663         struct mlx5_ibv_shared *lctx;
 664
 665         LIST_FOREACH(lctx, &mlx5_ibv_list, next)
 666                 if (lctx == sh)
 667                         break;
 668         assert(lctx);
 669         if (lctx != sh) {
 670                 DRV_LOG(ERR, "Freeing non-existing shared IB context");
 671                 goto exit;
 672         }
 673 #endif
 674         assert(sh);
 675         assert(sh->refcnt);
 676         /* Secondary process should not free the shared context. */
 677         assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
 678         if (--sh->refcnt)
 679                 goto exit;
 680         /* Release created Memory Regions. */
 681         mlx5_mr_release(sh);
 682         /* Remove from memory callback device list. */
 683         rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
 684         LIST_REMOVE(sh, mem_event_cb);
 685         rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
 686         /* Remove context from the global device list. */
 687         LIST_REMOVE(sh, next);
 688         /*
 689          *  Ensure there is no async event handler installed.
 690          *  Only primary process handles async device events.
 691          **/
 692         mlx5_flow_counters_mng_close(sh);
 693         assert(!sh->intr_cnt);
 694         if (sh->intr_cnt)
 695                 mlx5_intr_callback_unregister
 696                         (&sh->intr_handle, mlx5_dev_interrupt_handler, sh);
 697 #ifdef HAVE_MLX5_DEVX_ASYNC_SUPPORT
 698         if (sh->devx_intr_cnt) {
 699                 if (sh->intr_handle_devx.fd)
 700                         rte_intr_callback_unregister(&sh->intr_handle_devx,
 701                                           mlx5_dev_interrupt_handler_devx, sh);
 702                 if (sh->devx_comp)
 703                         mlx5dv_devx_destroy_cmd_comp(sh->devx_comp);
 704         }
 705 #endif
 706         pthread_mutex_destroy(&sh->intr_mutex);
 707         if (sh->pd)
 708                 claim_zero(mlx5_glue->dealloc_pd(sh->pd));
 709         if (sh->tis)
 710                 claim_zero(mlx5_devx_cmd_destroy(sh->tis));
 711         if (sh->td)
 712                 claim_zero(mlx5_devx_cmd_destroy(sh->td));
 713         if (sh->ctx)
 714                 claim_zero(mlx5_glue->close_device(sh->ctx));
 715         if (sh->flow_id_pool)
 716                 mlx5_flow_id_pool_release(sh->flow_id_pool);
 717         rte_free(sh);
 718 exit:
 719         pthread_mutex_unlock(&mlx5_ibv_list_mutex);
 720 }
 721
 722 /**
 723  * Destroy table hash list and all the root entries per domain.
 724  *
 725  * @param[in] priv
 726  *   Pointer to the private device data structure.
 727  */
 728 static void
 729 mlx5_free_table_hash_list(struct mlx5_priv *priv)
 730 {
 731         struct mlx5_ibv_shared *sh = priv->sh;
 732         struct mlx5_flow_tbl_data_entry *tbl_data;
 733         union mlx5_flow_tbl_key table_key = {
 734                 {
 735                         .table_id = 0,
 736                         .reserved = 0,
 737                         .domain = 0,
 738                         .direction = 0,
 739                 }
 740         };
 741         struct mlx5_hlist_entry *pos;
 742
 743         if (!sh->flow_tbls)
 744                 return;
 745         pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
 746         if (pos) {
 747                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 748                                         entry);
 749                 assert(tbl_data);
 750                 mlx5_hlist_remove(sh->flow_tbls, pos);
 751                 rte_free(tbl_data);
 752         }
 753         table_key.direction = 1;
 754         pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
 755         if (pos) {
 756                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 757                                         entry);
 758                 assert(tbl_data);
 759                 mlx5_hlist_remove(sh->flow_tbls, pos);
 760                 rte_free(tbl_data);
 761         }
 762         table_key.direction = 0;
 763         table_key.domain = 1;
 764         pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
 765         if (pos) {
 766                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 767                                         entry);
 768                 assert(tbl_data);
 769                 mlx5_hlist_remove(sh->flow_tbls, pos);
 770                 rte_free(tbl_data);
 771         }
 772         mlx5_hlist_destroy(sh->flow_tbls, NULL, NULL);
 773 }
 774
 775 /**
 776  * Initialize flow table hash list and create the root tables entry
 777  * for each domain.
 778  *
 779  * @param[in] priv
 780  *   Pointer to the private device data structure.
 781  *
 782  * @return
 783  *   Zero on success, positive error code otherwise.
 784  */
 785 static int
 786 mlx5_alloc_table_hash_list(struct mlx5_priv *priv)
 787 {
 788         struct mlx5_ibv_shared *sh = priv->sh;
 789         char s[MLX5_HLIST_NAMESIZE];
 790         int err = 0;
 791
 792         assert(sh);
 793         snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 794         sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE);
 795         if (!sh->flow_tbls) {
 796                 DRV_LOG(ERR, "flow tables with hash creation failed.\n");
 797                 err = ENOMEM;
 798                 return err;
 799         }
 800 #ifndef HAVE_MLX5DV_DR
 801         /*
 802          * In case we have not DR support, the zero tables should be created
 803          * because DV expect to see them even if they cannot be created by
 804          * RDMA-CORE.
 805          */
 806         union mlx5_flow_tbl_key table_key = {
 807                 {
 808                         .table_id = 0,
 809                         .reserved = 0,
 810                         .domain = 0,
 811                         .direction = 0,
 812                 }
 813         };
 814         struct mlx5_flow_tbl_data_entry *tbl_data = rte_zmalloc(NULL,
 815                                                           sizeof(*tbl_data), 0);
 816
 817         if (!tbl_data) {
 818                 err = ENOMEM;
 819                 goto error;
 820         }
 821         tbl_data->entry.key = table_key.v64;
 822         err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
 823         if (err)
 824                 goto error;
 825         rte_atomic32_init(&tbl_data->tbl.refcnt);
 826         rte_atomic32_inc(&tbl_data->tbl.refcnt);
 827         table_key.direction = 1;
 828         tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0);
 829         if (!tbl_data) {
 830                 err = ENOMEM;
 831                 goto error;
 832         }
 833         tbl_data->entry.key = table_key.v64;
 834         err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
 835         if (err)
 836                 goto error;
 837         rte_atomic32_init(&tbl_data->tbl.refcnt);
 838         rte_atomic32_inc(&tbl_data->tbl.refcnt);
 839         table_key.direction = 0;
 840         table_key.domain = 1;
 841         tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0);
 842         if (!tbl_data) {
 843                 err = ENOMEM;
 844                 goto error;
 845         }
 846         tbl_data->entry.key = table_key.v64;
 847         err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
 848         if (err)
 849                 goto error;
 850         rte_atomic32_init(&tbl_data->tbl.refcnt);
 851         rte_atomic32_inc(&tbl_data->tbl.refcnt);
 852         return err;
 853 error:
 854         mlx5_free_table_hash_list(priv);
 855 #endif /* HAVE_MLX5DV_DR */
 856         return err;
 857 }
 858
 859 /**
 860  * Initialize DR related data within private structure.
 861  * Routine checks the reference counter and does actual
 862  * resources creation/initialization only if counter is zero.
 863  *
 864  * @param[in] priv
 865  *   Pointer to the private device data structure.
 866  *
 867  * @return
 868  *   Zero on success, positive error code otherwise.
 869  */
 870 static int
 871 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 872 {
 873         struct mlx5_ibv_shared *sh = priv->sh;
 874         char s[MLX5_HLIST_NAMESIZE];
 875         int err = 0;
 876
 877         if (!sh->flow_tbls)
 878                 err = mlx5_alloc_table_hash_list(priv);
 879         else
 880                 DRV_LOG(DEBUG, "sh->flow_tbls[%p] already created, reuse\n",
 881                         (void *)sh->flow_tbls);
 882         if (err)
 883                 return err;
 884         /* Create tags hash list table. */
 885         snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 886         sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE);
 887         if (!sh->tag_table) {
 888                 DRV_LOG(ERR, "tags with hash creation failed.\n");
 889                 err = ENOMEM;
 890                 goto error;
 891         }
 892 #ifdef HAVE_MLX5DV_DR
 893         void *domain;
 894
 895         if (sh->dv_refcnt) {
 896                 /* Shared DV/DR structures is already initialized. */
 897                 sh->dv_refcnt++;
 898                 priv->dr_shared = 1;
 899                 return 0;
 900         }
 901         /* Reference counter is zero, we should initialize structures. */
 902         domain = mlx5_glue->dr_create_domain(sh->ctx,
 903                                              MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
 904         if (!domain) {
 905                 DRV_LOG(ERR, "ingress mlx5dv_dr_create_domain failed");
 906                 err = errno;
 907                 goto error;
 908         }
 909         sh->rx_domain = domain;
 910         domain = mlx5_glue->dr_create_domain(sh->ctx,
 911                                              MLX5DV_DR_DOMAIN_TYPE_NIC_TX);
 912         if (!domain) {
 913                 DRV_LOG(ERR, "egress mlx5dv_dr_create_domain failed");
 914                 err = errno;
 915                 goto error;
 916         }
 917         pthread_mutex_init(&sh->dv_mutex, NULL);
 918         sh->tx_domain = domain;
 919 #ifdef HAVE_MLX5DV_DR_ESWITCH
 920         if (priv->config.dv_esw_en) {
 921                 domain  = mlx5_glue->dr_create_domain
 922                         (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB);
 923                 if (!domain) {
 924                         DRV_LOG(ERR, "FDB mlx5dv_dr_create_domain failed");
 925                         err = errno;
 926                         goto error;
 927                 }
 928                 sh->fdb_domain = domain;
 929                 sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop();
 930         }
 931 #endif
 932         sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan();
 933 #endif /* HAVE_MLX5DV_DR */
 934         sh->dv_refcnt++;
 935         priv->dr_shared = 1;
 936         return 0;
 937 error:
 938         /* Rollback the created objects. */
 939         if (sh->rx_domain) {
 940                 mlx5_glue->dr_destroy_domain(sh->rx_domain);
 941                 sh->rx_domain = NULL;
 942         }
 943         if (sh->tx_domain) {
 944                 mlx5_glue->dr_destroy_domain(sh->tx_domain);
 945                 sh->tx_domain = NULL;
 946         }
 947         if (sh->fdb_domain) {
 948                 mlx5_glue->dr_destroy_domain(sh->fdb_domain);
 949                 sh->fdb_domain = NULL;
 950         }
 951         if (sh->esw_drop_action) {
 952                 mlx5_glue->destroy_flow_action(sh->esw_drop_action);
 953                 sh->esw_drop_action = NULL;
 954         }
 955         if (sh->pop_vlan_action) {
 956                 mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
 957                 sh->pop_vlan_action = NULL;
 958         }
 959         if (sh->tag_table) {
 960                 /* tags should be destroyed with flow before. */
 961                 mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
 962                 sh->tag_table = NULL;
 963         }
 964         mlx5_free_table_hash_list(priv);
 965         return err;
 966 }
 967
 968 /**
 969  * Destroy DR related data within private structure.
 970  *
 971  * @param[in] priv
 972  *   Pointer to the private device data structure.
 973  */
 974 static void
 975 mlx5_free_shared_dr(struct mlx5_priv *priv)
 976 {
 977         struct mlx5_ibv_shared *sh;
 978
 979         if (!priv->dr_shared)
 980                 return;
 981         priv->dr_shared = 0;
 982         sh = priv->sh;
 983         assert(sh);
 984 #ifdef HAVE_MLX5DV_DR
 985         assert(sh->dv_refcnt);
 986         if (sh->dv_refcnt && --sh->dv_refcnt)
 987                 return;
 988         if (sh->rx_domain) {
 989                 mlx5_glue->dr_destroy_domain(sh->rx_domain);
 990                 sh->rx_domain = NULL;
 991         }
 992         if (sh->tx_domain) {
 993                 mlx5_glue->dr_destroy_domain(sh->tx_domain);
 994                 sh->tx_domain = NULL;
 995         }
 996 #ifdef HAVE_MLX5DV_DR_ESWITCH
 997         if (sh->fdb_domain) {
 998                 mlx5_glue->dr_destroy_domain(sh->fdb_domain);
 999                 sh->fdb_domain = NULL;
1000         }
1001         if (sh->esw_drop_action) {
1002                 mlx5_glue->destroy_flow_action(sh->esw_drop_action);
1003                 sh->esw_drop_action = NULL;
1004         }
1005 #endif
1006         if (sh->pop_vlan_action) {
1007                 mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
1008                 sh->pop_vlan_action = NULL;
1009         }
1010         pthread_mutex_destroy(&sh->dv_mutex);
1011 #endif /* HAVE_MLX5DV_DR */
1012         if (sh->tag_table) {
1013                 /* tags should be destroyed with flow before. */
1014                 mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
1015                 sh->tag_table = NULL;
1016         }
1017         mlx5_free_table_hash_list(priv);
1018 }
1019
1020 /**
1021  * Initialize shared data between primary and secondary process.
1022  *
1023  * A memzone is reserved by primary process and secondary processes attach to
1024  * the memzone.
1025  *
1026  * @return
1027  *   0 on success, a negative errno value otherwise and rte_errno is set.
1028  */
1029 static int
1030 mlx5_init_shared_data(void)
1031 {
1032         const struct rte_memzone *mz;
1033         int ret = 0;
1034
1035         rte_spinlock_lock(&mlx5_shared_data_lock);
1036         if (mlx5_shared_data == NULL) {
1037                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
1038                         /* Allocate shared memory. */
1039                         mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
1040                                                  sizeof(*mlx5_shared_data),
1041                                                  SOCKET_ID_ANY, 0);
1042                         if (mz == NULL) {
1043                                 DRV_LOG(ERR,
1044                                         "Cannot allocate mlx5 shared data");
1045                                 ret = -rte_errno;
1046                                 goto error;
1047                         }
1048                         mlx5_shared_data = mz->addr;
1049                         memset(mlx5_shared_data, 0, sizeof(*mlx5_shared_data));
1050                         rte_spinlock_init(&mlx5_shared_data->lock);
1051                 } else {
1052                         /* Lookup allocated shared memory. */
1053                         mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
1054                         if (mz == NULL) {
1055                                 DRV_LOG(ERR,
1056                                         "Cannot attach mlx5 shared data");
1057                                 ret = -rte_errno;
1058                                 goto error;
1059                         }
1060                         mlx5_shared_data = mz->addr;
1061                         memset(&mlx5_local_data, 0, sizeof(mlx5_local_data));
1062                 }
1063         }
1064 error:
1065         rte_spinlock_unlock(&mlx5_shared_data_lock);
1066         return ret;
1067 }
1068
1069 /**
1070  * Retrieve integer value from environment variable.
1071  *
1072  * @param[in] name
1073  *   Environment variable name.
1074  *
1075  * @return
1076  *   Integer value, 0 if the variable is not set.
1077  */
1078 int
1079 mlx5_getenv_int(const char *name)
1080 {
1081         const char *val = getenv(name);
1082
1083         if (val == NULL)
1084                 return 0;
1085         return atoi(val);
1086 }
1087
1088 /**
1089  * Verbs callback to allocate a memory. This function should allocate the space
1090  * according to the size provided residing inside a huge page.
1091  * Please note that all allocation must respect the alignment from libmlx5
1092  * (i.e. currently sysconf(_SC_PAGESIZE)).
1093  *
1094  * @param[in] size
1095  *   The size in bytes of the memory to allocate.
1096  * @param[in] data
1097  *   A pointer to the callback data.
1098  *
1099  * @return
1100  *   Allocated buffer, NULL otherwise and rte_errno is set.
1101  */
1102 static void *
1103 mlx5_alloc_verbs_buf(size_t size, void *data)
1104 {
1105         struct mlx5_priv *priv = data;
1106         void *ret;
1107         size_t alignment = sysconf(_SC_PAGESIZE);
1108         unsigned int socket = SOCKET_ID_ANY;
1109
1110         if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) {
1111                 const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj;
1112
1113                 socket = ctrl->socket;
1114         } else if (priv->verbs_alloc_ctx.type ==
1115                    MLX5_VERBS_ALLOC_TYPE_RX_QUEUE) {
1116                 const struct mlx5_rxq_ctrl *ctrl = priv->verbs_alloc_ctx.obj;
1117
1118                 socket = ctrl->socket;
1119         }
1120         assert(data != NULL);
1121         ret = rte_malloc_socket(__func__, size, alignment, socket);
1122         if (!ret && size)
1123                 rte_errno = ENOMEM;
1124         return ret;
1125 }
1126
1127 /**
1128  * Verbs callback to free a memory.
1129  *
1130  * @param[in] ptr
1131  *   A pointer to the memory to free.
1132  * @param[in] data
1133  *   A pointer to the callback data.
1134  */
1135 static void
1136 mlx5_free_verbs_buf(void *ptr, void *data __rte_unused)
1137 {
1138         assert(data != NULL);
1139         rte_free(ptr);
1140 }
1141
1142 /**
1143  * DPDK callback to add udp tunnel port
1144  *
1145  * @param[in] dev
1146  *   A pointer to eth_dev
1147  * @param[in] udp_tunnel
1148  *   A pointer to udp tunnel
1149  *
1150  * @return
1151  *   0 on valid udp ports and tunnels, -ENOTSUP otherwise.
1152  */
1153 int
1154 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused,
1155                          struct rte_eth_udp_tunnel *udp_tunnel)
1156 {
1157         assert(udp_tunnel != NULL);
1158         if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN &&
1159             udp_tunnel->udp_port == 4789)
1160                 return 0;
1161         if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN_GPE &&
1162             udp_tunnel->udp_port == 4790)
1163                 return 0;
1164         return -ENOTSUP;
1165 }
1166
1167 /**
1168  * Initialize process private data structure.
1169  *
1170  * @param dev
1171  *   Pointer to Ethernet device structure.
1172  *
1173  * @return
1174  *   0 on success, a negative errno value otherwise and rte_errno is set.
1175  */
1176 int
1177 mlx5_proc_priv_init(struct rte_eth_dev *dev)
1178 {
1179         struct mlx5_priv *priv = dev->data->dev_private;
1180         struct mlx5_proc_priv *ppriv;
1181         size_t ppriv_size;
1182
1183         /*
1184          * UAR register table follows the process private structure. BlueFlame
1185          * registers for Tx queues are stored in the table.
1186          */
1187         ppriv_size =
1188                 sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *);
1189         ppriv = rte_malloc_socket("mlx5_proc_priv", ppriv_size,
1190                                   RTE_CACHE_LINE_SIZE, dev->device->numa_node);
1191         if (!ppriv) {
1192                 rte_errno = ENOMEM;
1193                 return -rte_errno;
1194         }
1195         ppriv->uar_table_sz = ppriv_size;
1196         dev->process_private = ppriv;
1197         return 0;
1198 }
1199
1200 /**
1201  * Un-initialize process private data structure.
1202  *
1203  * @param dev
1204  *   Pointer to Ethernet device structure.
1205  */
1206 static void
1207 mlx5_proc_priv_uninit(struct rte_eth_dev *dev)
1208 {
1209         if (!dev->process_private)
1210                 return;
1211         rte_free(dev->process_private);
1212         dev->process_private = NULL;
1213 }
1214
1215 /**
1216  * DPDK callback to close the device.
1217  *
1218  * Destroy all queues and objects, free memory.
1219  *
1220  * @param dev
1221  *   Pointer to Ethernet device structure.
1222  */
1223 static void
1224 mlx5_dev_close(struct rte_eth_dev *dev)
1225 {
1226         struct mlx5_priv *priv = dev->data->dev_private;
1227         unsigned int i;
1228         int ret;
1229
1230         DRV_LOG(DEBUG, "port %u closing device \"%s\"",
1231                 dev->data->port_id,
1232                 ((priv->sh->ctx != NULL) ? priv->sh->ctx->device->name : ""));
1233         /* In case mlx5_dev_stop() has not been called. */
1234         mlx5_dev_interrupt_handler_uninstall(dev);
1235         mlx5_dev_interrupt_handler_devx_uninstall(dev);
1236         mlx5_traffic_disable(dev);
1237         mlx5_flow_flush(dev, NULL);
1238         mlx5_flow_meter_flush(dev, NULL);
1239         /* Prevent crashes when queues are still in use. */
1240         dev->rx_pkt_burst = removed_rx_burst;
1241         dev->tx_pkt_burst = removed_tx_burst;
1242         rte_wmb();
1243         /* Disable datapath on secondary process. */
1244         mlx5_mp_req_stop_rxtx(dev);
1245         if (priv->rxqs != NULL) {
1246                 /* XXX race condition if mlx5_rx_burst() is still running. */
1247                 usleep(1000);
1248                 for (i = 0; (i != priv->rxqs_n); ++i)
1249                         mlx5_rxq_release(dev, i);
1250                 priv->rxqs_n = 0;
1251                 priv->rxqs = NULL;
1252         }
1253         if (priv->txqs != NULL) {
1254                 /* XXX race condition if mlx5_tx_burst() is still running. */
1255                 usleep(1000);
1256                 for (i = 0; (i != priv->txqs_n); ++i)
1257                         mlx5_txq_release(dev, i);
1258                 priv->txqs_n = 0;
1259                 priv->txqs = NULL;
1260         }
1261         mlx5_proc_priv_uninit(dev);
1262         if (priv->mreg_cp_tbl)
1263                 mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
1264         mlx5_mprq_free_mp(dev);
1265         mlx5_free_shared_dr(priv);
1266         if (priv->rss_conf.rss_key != NULL)
1267                 rte_free(priv->rss_conf.rss_key);
1268         if (priv->reta_idx != NULL)
1269                 rte_free(priv->reta_idx);
1270         if (priv->config.vf)
1271                 mlx5_nl_mac_addr_flush(dev);
1272         if (priv->nl_socket_route >= 0)
1273                 close(priv->nl_socket_route);
1274         if (priv->nl_socket_rdma >= 0)
1275                 close(priv->nl_socket_rdma);
1276         if (priv->vmwa_context)
1277                 mlx5_vlan_vmwa_exit(priv->vmwa_context);
1278         if (priv->sh) {
1279                 /*
1280                  * Free the shared context in last turn, because the cleanup
1281                  * routines above may use some shared fields, like
1282                  * mlx5_nl_mac_addr_flush() uses ibdev_path for retrieveing
1283                  * ifindex if Netlink fails.
1284                  */
1285                 mlx5_free_shared_ibctx(priv->sh);
1286                 priv->sh = NULL;
1287         }
1288         ret = mlx5_hrxq_verify(dev);
1289         if (ret)
1290                 DRV_LOG(WARNING, "port %u some hash Rx queue still remain",
1291                         dev->data->port_id);
1292         ret = mlx5_ind_table_obj_verify(dev);
1293         if (ret)
1294                 DRV_LOG(WARNING, "port %u some indirection table still remain",
1295                         dev->data->port_id);
1296         ret = mlx5_rxq_obj_verify(dev);
1297         if (ret)
1298                 DRV_LOG(WARNING, "port %u some Rx queue objects still remain",
1299                         dev->data->port_id);
1300         ret = mlx5_rxq_verify(dev);
1301         if (ret)
1302                 DRV_LOG(WARNING, "port %u some Rx queues still remain",
1303                         dev->data->port_id);
1304         ret = mlx5_txq_obj_verify(dev);
1305         if (ret)
1306                 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain",
1307                         dev->data->port_id);
1308         ret = mlx5_txq_verify(dev);
1309         if (ret)
1310                 DRV_LOG(WARNING, "port %u some Tx queues still remain",
1311                         dev->data->port_id);
1312         ret = mlx5_flow_verify(dev);
1313         if (ret)
1314                 DRV_LOG(WARNING, "port %u some flows still remain",
1315                         dev->data->port_id);
1316         if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
1317                 unsigned int c = 0;
1318                 uint16_t port_id;
1319
1320                 MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
1321                         struct mlx5_priv *opriv =
1322                                 rte_eth_devices[port_id].data->dev_private;
1323
1324                         if (!opriv ||
1325                             opriv->domain_id != priv->domain_id ||
1326                             &rte_eth_devices[port_id] == dev)
1327                                 continue;
1328                         ++c;
1329                         break;
1330                 }
1331                 if (!c)
1332                         claim_zero(rte_eth_switch_domain_free(priv->domain_id));
1333         }
1334         memset(priv, 0, sizeof(*priv));
1335         priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
1336         /*
1337          * Reset mac_addrs to NULL such that it is not freed as part of
1338          * rte_eth_dev_release_port(). mac_addrs is part of dev_private so
1339          * it is freed when dev_private is freed.
1340          */
1341         dev->data->mac_addrs = NULL;
1342 }
1343
1344 const struct eth_dev_ops mlx5_dev_ops = {
1345         .dev_configure = mlx5_dev_configure,
1346         .dev_start = mlx5_dev_start,
1347         .dev_stop = mlx5_dev_stop,
1348         .dev_set_link_down = mlx5_set_link_down,
1349         .dev_set_link_up = mlx5_set_link_up,
1350         .dev_close = mlx5_dev_close,
1351         .promiscuous_enable = mlx5_promiscuous_enable,
1352         .promiscuous_disable = mlx5_promiscuous_disable,
1353         .allmulticast_enable = mlx5_allmulticast_enable,
1354         .allmulticast_disable = mlx5_allmulticast_disable,
1355         .link_update = mlx5_link_update,
1356         .stats_get = mlx5_stats_get,
1357         .stats_reset = mlx5_stats_reset,
1358         .xstats_get = mlx5_xstats_get,
1359         .xstats_reset = mlx5_xstats_reset,
1360         .xstats_get_names = mlx5_xstats_get_names,
1361         .fw_version_get = mlx5_fw_version_get,
1362         .dev_infos_get = mlx5_dev_infos_get,
1363         .read_clock = mlx5_read_clock,
1364         .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
1365         .vlan_filter_set = mlx5_vlan_filter_set,
1366         .rx_queue_setup = mlx5_rx_queue_setup,
1367         .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup,
1368         .tx_queue_setup = mlx5_tx_queue_setup,
1369         .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup,
1370         .rx_queue_release = mlx5_rx_queue_release,
1371         .tx_queue_release = mlx5_tx_queue_release,
1372         .flow_ctrl_get = mlx5_dev_get_flow_ctrl,
1373         .flow_ctrl_set = mlx5_dev_set_flow_ctrl,
1374         .mac_addr_remove = mlx5_mac_addr_remove,
1375         .mac_addr_add = mlx5_mac_addr_add,
1376         .mac_addr_set = mlx5_mac_addr_set,
1377         .set_mc_addr_list = mlx5_set_mc_addr_list,
1378         .mtu_set = mlx5_dev_set_mtu,
1379         .vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
1380         .vlan_offload_set = mlx5_vlan_offload_set,
1381         .reta_update = mlx5_dev_rss_reta_update,
1382         .reta_query = mlx5_dev_rss_reta_query,
1383         .rss_hash_update = mlx5_rss_hash_update,
1384         .rss_hash_conf_get = mlx5_rss_hash_conf_get,
1385         .filter_ctrl = mlx5_dev_filter_ctrl,
1386         .rx_descriptor_status = mlx5_rx_descriptor_status,
1387         .tx_descriptor_status = mlx5_tx_descriptor_status,
1388         .rx_queue_count = mlx5_rx_queue_count,
1389         .rx_queue_intr_enable = mlx5_rx_intr_enable,
1390         .rx_queue_intr_disable = mlx5_rx_intr_disable,
1391         .is_removed = mlx5_is_removed,
1392         .udp_tunnel_port_add  = mlx5_udp_tunnel_port_add,
1393         .get_module_info = mlx5_get_module_info,
1394         .get_module_eeprom = mlx5_get_module_eeprom,
1395         .hairpin_cap_get = mlx5_hairpin_cap_get,
1396         .mtr_ops_get = mlx5_flow_meter_ops_get,
1397 };
1398
1399 /* Available operations from secondary process. */
1400 static const struct eth_dev_ops mlx5_dev_sec_ops = {
1401         .stats_get = mlx5_stats_get,
1402         .stats_reset = mlx5_stats_reset,
1403         .xstats_get = mlx5_xstats_get,
1404         .xstats_reset = mlx5_xstats_reset,
1405         .xstats_get_names = mlx5_xstats_get_names,
1406         .fw_version_get = mlx5_fw_version_get,
1407         .dev_infos_get = mlx5_dev_infos_get,
1408         .rx_descriptor_status = mlx5_rx_descriptor_status,
1409         .tx_descriptor_status = mlx5_tx_descriptor_status,
1410         .get_module_info = mlx5_get_module_info,
1411         .get_module_eeprom = mlx5_get_module_eeprom,
1412 };
1413
1414 /* Available operations in flow isolated mode. */
1415 const struct eth_dev_ops mlx5_dev_ops_isolate = {
1416         .dev_configure = mlx5_dev_configure,
1417         .dev_start = mlx5_dev_start,
1418         .dev_stop = mlx5_dev_stop,
1419         .dev_set_link_down = mlx5_set_link_down,
1420         .dev_set_link_up = mlx5_set_link_up,
1421         .dev_close = mlx5_dev_close,
1422         .promiscuous_enable = mlx5_promiscuous_enable,
1423         .promiscuous_disable = mlx5_promiscuous_disable,
1424         .allmulticast_enable = mlx5_allmulticast_enable,
1425         .allmulticast_disable = mlx5_allmulticast_disable,
1426         .link_update = mlx5_link_update,
1427         .stats_get = mlx5_stats_get,
1428         .stats_reset = mlx5_stats_reset,
1429         .xstats_get = mlx5_xstats_get,
1430         .xstats_reset = mlx5_xstats_reset,
1431         .xstats_get_names = mlx5_xstats_get_names,
1432         .fw_version_get = mlx5_fw_version_get,
1433         .dev_infos_get = mlx5_dev_infos_get,
1434         .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
1435         .vlan_filter_set = mlx5_vlan_filter_set,
1436         .rx_queue_setup = mlx5_rx_queue_setup,
1437         .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup,
1438         .tx_queue_setup = mlx5_tx_queue_setup,
1439         .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup,
1440         .rx_queue_release = mlx5_rx_queue_release,
1441         .tx_queue_release = mlx5_tx_queue_release,
1442         .flow_ctrl_get = mlx5_dev_get_flow_ctrl,
1443         .flow_ctrl_set = mlx5_dev_set_flow_ctrl,
1444         .mac_addr_remove = mlx5_mac_addr_remove,
1445         .mac_addr_add = mlx5_mac_addr_add,
1446         .mac_addr_set = mlx5_mac_addr_set,
1447         .set_mc_addr_list = mlx5_set_mc_addr_list,
1448         .mtu_set = mlx5_dev_set_mtu,
1449         .vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
1450         .vlan_offload_set = mlx5_vlan_offload_set,
1451         .filter_ctrl = mlx5_dev_filter_ctrl,
1452         .rx_descriptor_status = mlx5_rx_descriptor_status,
1453         .tx_descriptor_status = mlx5_tx_descriptor_status,
1454         .rx_queue_intr_enable = mlx5_rx_intr_enable,
1455         .rx_queue_intr_disable = mlx5_rx_intr_disable,
1456         .is_removed = mlx5_is_removed,
1457         .get_module_info = mlx5_get_module_info,
1458         .get_module_eeprom = mlx5_get_module_eeprom,
1459         .hairpin_cap_get = mlx5_hairpin_cap_get,
1460         .mtr_ops_get = mlx5_flow_meter_ops_get,
1461 };
1462
1463 /**
1464  * Verify and store value for device argument.
1465  *
1466  * @param[in] key
1467  *   Key argument to verify.
1468  * @param[in] val
1469  *   Value associated with key.
1470  * @param opaque
1471  *   User data.
1472  *
1473  * @return
1474  *   0 on success, a negative errno value otherwise and rte_errno is set.
1475  */
1476 static int
1477 mlx5_args_check(const char *key, const char *val, void *opaque)
1478 {
1479         struct mlx5_dev_config *config = opaque;
1480         unsigned long tmp;
1481
1482         /* No-op, port representors are processed in mlx5_dev_spawn(). */
1483         if (!strcmp(MLX5_REPRESENTOR, key))
1484                 return 0;
1485         errno = 0;
1486         tmp = strtoul(val, NULL, 0);
1487         if (errno) {
1488                 rte_errno = errno;
1489                 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val);
1490                 return -rte_errno;
1491         }
1492         if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) {
1493                 config->cqe_comp = !!tmp;
1494         } else if (strcmp(MLX5_RXQ_CQE_PAD_EN, key) == 0) {
1495                 config->cqe_pad = !!tmp;
1496         } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) {
1497                 config->hw_padding = !!tmp;
1498         } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) {
1499                 config->mprq.enabled = !!tmp;
1500         } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) {
1501                 config->mprq.stride_num_n = tmp;
1502         } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) {
1503                 config->mprq.max_memcpy_len = tmp;
1504         } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) {
1505                 config->mprq.min_rxqs_num = tmp;
1506         } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) {
1507                 DRV_LOG(WARNING, "%s: deprecated parameter,"
1508                                  " converted to txq_inline_max", key);
1509                 config->txq_inline_max = tmp;
1510         } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) {
1511                 config->txq_inline_max = tmp;
1512         } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) {
1513                 config->txq_inline_min = tmp;
1514         } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) {
1515                 config->txq_inline_mpw = tmp;
1516         } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) {
1517                 config->txqs_inline = tmp;
1518         } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) {
1519                 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key);
1520         } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) {
1521                 config->mps = !!tmp;
1522         } else if (strcmp(MLX5_TX_DB_NC, key) == 0) {
1523                 if (tmp != MLX5_TXDB_CACHED &&
1524                     tmp != MLX5_TXDB_NCACHED &&
1525                     tmp != MLX5_TXDB_HEURISTIC) {
1526                         DRV_LOG(ERR, "invalid Tx doorbell "
1527                                      "mapping parameter");
1528                         rte_errno = EINVAL;
1529                         return -rte_errno;
1530                 }
1531                 config->dbnc = tmp;
1532         } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) {
1533                 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key);
1534         } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) {
1535                 DRV_LOG(WARNING, "%s: deprecated parameter,"
1536                                  " converted to txq_inline_mpw", key);
1537                 config->txq_inline_mpw = tmp;
1538         } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) {
1539                 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key);
1540         } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) {
1541                 config->rx_vec_en = !!tmp;
1542         } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) {
1543                 config->l3_vxlan_en = !!tmp;
1544         } else if (strcmp(MLX5_VF_NL_EN, key) == 0) {
1545                 config->vf_nl_en = !!tmp;
1546         } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) {
1547                 config->dv_esw_en = !!tmp;
1548         } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) {
1549                 config->dv_flow_en = !!tmp;
1550         } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) {
1551                 if (tmp != MLX5_XMETA_MODE_LEGACY &&
1552                     tmp != MLX5_XMETA_MODE_META16 &&
1553                     tmp != MLX5_XMETA_MODE_META32) {
1554                         DRV_LOG(ERR, "invalid extensive "
1555                                      "metadata parameter");
1556                         rte_errno = EINVAL;
1557                         return -rte_errno;
1558                 }
1559                 config->dv_xmeta_en = tmp;
1560         } else if (strcmp(MLX5_MR_EXT_MEMSEG_EN, key) == 0) {
1561                 config->mr_ext_memseg_en = !!tmp;
1562         } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) {
1563                 config->max_dump_files_num = tmp;
1564         } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
1565                 config->lro.timeout = tmp;
1566         } else {
1567                 DRV_LOG(WARNING, "%s: unknown parameter", key);
1568                 rte_errno = EINVAL;
1569                 return -rte_errno;
1570         }
1571         return 0;
1572 }
1573
1574 /**
1575  * Parse device parameters.
1576  *
1577  * @param config
1578  *   Pointer to device configuration structure.
1579  * @param devargs
1580  *   Device arguments structure.
1581  *
1582  * @return
1583  *   0 on success, a negative errno value otherwise and rte_errno is set.
1584  */
1585 static int
1586 mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
1587 {
1588         const char **params = (const char *[]){
1589                 MLX5_RXQ_CQE_COMP_EN,
1590                 MLX5_RXQ_CQE_PAD_EN,
1591                 MLX5_RXQ_PKT_PAD_EN,
1592                 MLX5_RX_MPRQ_EN,
1593                 MLX5_RX_MPRQ_LOG_STRIDE_NUM,
1594                 MLX5_RX_MPRQ_MAX_MEMCPY_LEN,
1595                 MLX5_RXQS_MIN_MPRQ,
1596                 MLX5_TXQ_INLINE,
1597                 MLX5_TXQ_INLINE_MIN,
1598                 MLX5_TXQ_INLINE_MAX,
1599                 MLX5_TXQ_INLINE_MPW,
1600                 MLX5_TXQS_MIN_INLINE,
1601                 MLX5_TXQS_MAX_VEC,
1602                 MLX5_TXQ_MPW_EN,
1603                 MLX5_TXQ_MPW_HDR_DSEG_EN,
1604                 MLX5_TXQ_MAX_INLINE_LEN,
1605                 MLX5_TX_DB_NC,
1606                 MLX5_TX_VEC_EN,
1607                 MLX5_RX_VEC_EN,
1608                 MLX5_L3_VXLAN_EN,
1609                 MLX5_VF_NL_EN,
1610                 MLX5_DV_ESW_EN,
1611                 MLX5_DV_FLOW_EN,
1612                 MLX5_DV_XMETA_EN,
1613                 MLX5_MR_EXT_MEMSEG_EN,
1614                 MLX5_REPRESENTOR,
1615                 MLX5_MAX_DUMP_FILES_NUM,
1616                 MLX5_LRO_TIMEOUT_USEC,
1617                 NULL,
1618         };
1619         struct rte_kvargs *kvlist;
1620         int ret = 0;
1621         int i;
1622
1623         if (devargs == NULL)
1624                 return 0;
1625         /* Following UGLY cast is done to pass checkpatch. */
1626         kvlist = rte_kvargs_parse(devargs->args, params);
1627         if (kvlist == NULL) {
1628                 rte_errno = EINVAL;
1629                 return -rte_errno;
1630         }
1631         /* Process parameters. */
1632         for (i = 0; (params[i] != NULL); ++i) {
1633                 if (rte_kvargs_count(kvlist, params[i])) {
1634                         ret = rte_kvargs_process(kvlist, params[i],
1635                                                  mlx5_args_check, config);
1636                         if (ret) {
1637                                 rte_errno = EINVAL;
1638                                 rte_kvargs_free(kvlist);
1639                                 return -rte_errno;
1640                         }
1641                 }
1642         }
1643         rte_kvargs_free(kvlist);
1644         return 0;
1645 }
1646
1647 static struct rte_pci_driver mlx5_driver;
1648
1649 /**
1650  * PMD global initialization.
1651  *
1652  * Independent from individual device, this function initializes global
1653  * per-PMD data structures distinguishing primary and secondary processes.
1654  * Hence, each initialization is called once per a process.
1655  *
1656  * @return
1657  *   0 on success, a negative errno value otherwise and rte_errno is set.
1658  */
1659 static int
1660 mlx5_init_once(void)
1661 {
1662         struct mlx5_shared_data *sd;
1663         struct mlx5_local_data *ld = &mlx5_local_data;
1664         int ret = 0;
1665
1666         if (mlx5_init_shared_data())
1667                 return -rte_errno;
1668         sd = mlx5_shared_data;
1669         assert(sd);
1670         rte_spinlock_lock(&sd->lock);
1671         switch (rte_eal_process_type()) {
1672         case RTE_PROC_PRIMARY:
1673                 if (sd->init_done)
1674                         break;
1675                 LIST_INIT(&sd->mem_event_cb_list);
1676                 rte_rwlock_init(&sd->mem_event_rwlock);
1677                 rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
1678                                                 mlx5_mr_mem_event_cb, NULL);
1679                 ret = mlx5_mp_init_primary();
1680                 if (ret)
1681                         goto out;
1682                 sd->init_done = true;
1683                 break;
1684         case RTE_PROC_SECONDARY:
1685                 if (ld->init_done)
1686                         break;
1687                 ret = mlx5_mp_init_secondary();
1688                 if (ret)
1689                         goto out;
1690                 ++sd->secondary_cnt;
1691                 ld->init_done = true;
1692                 break;
1693         default:
1694                 break;
1695         }
1696 out:
1697         rte_spinlock_unlock(&sd->lock);
1698         return ret;
1699 }
1700
1701 /**
1702  * Configures the minimal amount of data to inline into WQE
1703  * while sending packets.
1704  *
1705  * - the txq_inline_min has the maximal priority, if this
1706  *   key is specified in devargs
1707  * - if DevX is enabled the inline mode is queried from the
1708  *   device (HCA attributes and NIC vport context if needed).
1709  * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4LX
1710  *   and none (0 bytes) for other NICs
1711  *
1712  * @param spawn
1713  *   Verbs device parameters (name, port, switch_info) to spawn.
1714  * @param config
1715  *   Device configuration parameters.
1716  */
1717 static void
1718 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
1719                     struct mlx5_dev_config *config)
1720 {
1721         if (config->txq_inline_min != MLX5_ARG_UNSET) {
1722                 /* Application defines size of inlined data explicitly. */
1723                 switch (spawn->pci_dev->id.device_id) {
1724                 case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
1725                 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
1726                         if (config->txq_inline_min <
1727                                        (int)MLX5_INLINE_HSIZE_L2) {
1728                                 DRV_LOG(DEBUG,
1729                                         "txq_inline_mix aligned to minimal"
1730                                         " ConnectX-4 required value %d",
1731                                         (int)MLX5_INLINE_HSIZE_L2);
1732                                 config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
1733                         }
1734                         break;
1735                 }
1736                 goto exit;
1737         }
1738         if (config->hca_attr.eth_net_offloads) {
1739                 /* We have DevX enabled, inline mode queried successfully. */
1740                 switch (config->hca_attr.wqe_inline_mode) {
1741                 case MLX5_CAP_INLINE_MODE_L2:
1742                         /* outer L2 header must be inlined. */
1743                         config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
1744                         goto exit;
1745                 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
1746                         /* No inline data are required by NIC. */
1747                         config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
1748                         config->hw_vlan_insert =
1749                                 config->hca_attr.wqe_vlan_insert;
1750                         DRV_LOG(DEBUG, "Tx VLAN insertion is supported");
1751                         goto exit;
1752                 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
1753                         /* inline mode is defined by NIC vport context. */
1754                         if (!config->hca_attr.eth_virt)
1755                                 break;
1756                         switch (config->hca_attr.vport_inline_mode) {
1757                         case MLX5_INLINE_MODE_NONE:
1758                                 config->txq_inline_min =
1759                                         MLX5_INLINE_HSIZE_NONE;
1760                                 goto exit;
1761                         case MLX5_INLINE_MODE_L2:
1762                                 config->txq_inline_min =
1763                                         MLX5_INLINE_HSIZE_L2;
1764                                 goto exit;
1765                         case MLX5_INLINE_MODE_IP:
1766                                 config->txq_inline_min =
1767                                         MLX5_INLINE_HSIZE_L3;
1768                                 goto exit;
1769                         case MLX5_INLINE_MODE_TCP_UDP:
1770                                 config->txq_inline_min =
1771                                         MLX5_INLINE_HSIZE_L4;
1772                                 goto exit;
1773                         case MLX5_INLINE_MODE_INNER_L2:
1774                                 config->txq_inline_min =
1775                                         MLX5_INLINE_HSIZE_INNER_L2;
1776                                 goto exit;
1777                         case MLX5_INLINE_MODE_INNER_IP:
1778                                 config->txq_inline_min =
1779                                         MLX5_INLINE_HSIZE_INNER_L3;
1780                                 goto exit;
1781                         case MLX5_INLINE_MODE_INNER_TCP_UDP:
1782                                 config->txq_inline_min =
1783                                         MLX5_INLINE_HSIZE_INNER_L4;
1784                                 goto exit;
1785                         }
1786                 }
1787         }
1788         /*
1789          * We get here if we are unable to deduce
1790          * inline data size with DevX. Try PCI ID
1791          * to determine old NICs.
1792          */
1793         switch (spawn->pci_dev->id.device_id) {
1794         case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
1795         case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
1796         case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX:
1797         case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
1798                 config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
1799                 config->hw_vlan_insert = 0;
1800                 break;
1801         case PCI_DEVICE_ID_MELLANOX_CONNECTX5:
1802         case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
1803         case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX:
1804         case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
1805                 /*
1806                  * These NICs support VLAN insertion from WQE and
1807                  * report the wqe_vlan_insert flag. But there is the bug
1808                  * and PFC control may be broken, so disable feature.
1809                  */
1810                 config->hw_vlan_insert = 0;
1811                 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
1812                 break;
1813         default:
1814                 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
1815                 break;
1816         }
1817 exit:
1818         DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min);
1819 }
1820
1821 /**
1822  * Configures the metadata mask fields in the shared context.
1823  *
1824  * @param [in] dev
1825  *   Pointer to Ethernet device.
1826  */
1827 static void
1828 mlx5_set_metadata_mask(struct rte_eth_dev *dev)
1829 {
1830         struct mlx5_priv *priv = dev->data->dev_private;
1831         struct mlx5_ibv_shared *sh = priv->sh;
1832         uint32_t meta, mark, reg_c0;
1833
1834         reg_c0 = ~priv->vport_meta_mask;
1835         switch (priv->config.dv_xmeta_en) {
1836         case MLX5_XMETA_MODE_LEGACY:
1837                 meta = UINT32_MAX;
1838                 mark = MLX5_FLOW_MARK_MASK;
1839                 break;
1840         case MLX5_XMETA_MODE_META16:
1841                 meta = reg_c0 >> rte_bsf32(reg_c0);
1842                 mark = MLX5_FLOW_MARK_MASK;
1843                 break;
1844         case MLX5_XMETA_MODE_META32:
1845                 meta = UINT32_MAX;
1846                 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK;
1847                 break;
1848         default:
1849                 meta = 0;
1850                 mark = 0;
1851                 assert(false);
1852                 break;
1853         }
1854         if (sh->dv_mark_mask && sh->dv_mark_mask != mark)
1855                 DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X",
1856                                  sh->dv_mark_mask, mark);
1857         else
1858                 sh->dv_mark_mask = mark;
1859         if (sh->dv_meta_mask && sh->dv_meta_mask != meta)
1860                 DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X",
1861                                  sh->dv_meta_mask, meta);
1862         else
1863                 sh->dv_meta_mask = meta;
1864         if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0)
1865                 DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X",
1866                                  sh->dv_meta_mask, reg_c0);
1867         else
1868                 sh->dv_regc0_mask = reg_c0;
1869         DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en);
1870         DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask);
1871         DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask);
1872         DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask);
1873 }
1874
1875 /**
1876  * Allocate page of door-bells and register it using DevX API.
1877  *
1878  * @param [in] dev
1879  *   Pointer to Ethernet device.
1880  *
1881  * @return
1882  *   Pointer to new page on success, NULL otherwise.
1883  */
1884 static struct mlx5_devx_dbr_page *
1885 mlx5_alloc_dbr_page(struct rte_eth_dev *dev)
1886 {
1887         struct mlx5_priv *priv = dev->data->dev_private;
1888         struct mlx5_devx_dbr_page *page;
1889
1890         /* Allocate space for door-bell page and management data. */
1891         page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
1892                                  RTE_CACHE_LINE_SIZE, dev->device->numa_node);
1893         if (!page) {
1894                 DRV_LOG(ERR, "port %u cannot allocate dbr page",
1895                         dev->data->port_id);
1896                 return NULL;
1897         }
1898         /* Register allocated memory. */
1899         page->umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, page->dbrs,
1900                                               MLX5_DBR_PAGE_SIZE, 0);
1901         if (!page->umem) {
1902                 DRV_LOG(ERR, "port %u cannot umem reg dbr page",
1903                         dev->data->port_id);
1904                 rte_free(page);
1905                 return NULL;
1906         }
1907         return page;
1908 }
1909
1910 /**
1911  * Find the next available door-bell, allocate new page if needed.
1912  *
1913  * @param [in] dev
1914  *   Pointer to Ethernet device.
1915  * @param [out] dbr_page
1916  *   Door-bell page containing the page data.
1917  *
1918  * @return
1919  *   Door-bell address offset on success, a negative error value otherwise.
1920  */
1921 int64_t
1922 mlx5_get_dbr(struct rte_eth_dev *dev, struct mlx5_devx_dbr_page **dbr_page)
1923 {
1924         struct mlx5_priv *priv = dev->data->dev_private;
1925         struct mlx5_devx_dbr_page *page = NULL;
1926         uint32_t i, j;
1927
1928         LIST_FOREACH(page, &priv->dbrpgs, next)
1929                 if (page->dbr_count < MLX5_DBR_PER_PAGE)
1930                         break;
1931         if (!page) { /* No page with free door-bell exists. */
1932                 page = mlx5_alloc_dbr_page(dev);
1933                 if (!page) /* Failed to allocate new page. */
1934                         return (-1);
1935                 LIST_INSERT_HEAD(&priv->dbrpgs, page, next);
1936         }
1937         /* Loop to find bitmap part with clear bit. */
1938         for (i = 0;
1939              i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
1940              i++)
1941                 ; /* Empty. */
1942         /* Find the first clear bit. */
1943         j = rte_bsf64(~page->dbr_bitmap[i]);
1944         assert(i < (MLX5_DBR_PER_PAGE / 64));
1945         page->dbr_bitmap[i] |= (1 << j);
1946         page->dbr_count++;
1947         *dbr_page = page;
1948         return (((i * 64) + j) * sizeof(uint64_t));
1949 }
1950
1951 /**
1952  * Release a door-bell record.
1953  *
1954  * @param [in] dev
1955  *   Pointer to Ethernet device.
1956  * @param [in] umem_id
1957  *   UMEM ID of page containing the door-bell record to release.
1958  * @param [in] offset
1959  *   Offset of door-bell record in page.
1960  *
1961  * @return
1962  *   0 on success, a negative error value otherwise.
1963  */
1964 int32_t
1965 mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id, uint64_t offset)
1966 {
1967         struct mlx5_priv *priv = dev->data->dev_private;
1968         struct mlx5_devx_dbr_page *page = NULL;
1969         int ret = 0;
1970
1971         LIST_FOREACH(page, &priv->dbrpgs, next)
1972                 /* Find the page this address belongs to. */
1973                 if (page->umem->umem_id == umem_id)
1974                         break;
1975         if (!page)
1976                 return -EINVAL;
1977         page->dbr_count--;
1978         if (!page->dbr_count) {
1979                 /* Page not used, free it and remove from list. */
1980                 LIST_REMOVE(page, next);
1981                 if (page->umem)
1982                         ret = -mlx5_glue->devx_umem_dereg(page->umem);
1983                 rte_free(page);
1984         } else {
1985                 /* Mark in bitmap that this door-bell is not in use. */
1986                 offset /= MLX5_DBR_SIZE;
1987                 int i = offset / 64;
1988                 int j = offset % 64;
1989
1990                 page->dbr_bitmap[i] &= ~(1 << j);
1991         }
1992         return ret;
1993 }
1994
1995 /**
1996  * Check sibling device configurations.
1997  *
1998  * Sibling devices sharing the Infiniband device context
1999  * should have compatible configurations. This regards
2000  * representors and bonding slaves.
2001  *
2002  * @param priv
2003  *   Private device descriptor.
2004  * @param config
2005  *   Configuration of the device is going to be created.
2006  *
2007  * @return
2008  *   0 on success, EINVAL otherwise
2009  */
2010 static int
2011 mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
2012                               struct mlx5_dev_config *config)
2013 {
2014         struct mlx5_ibv_shared *sh = priv->sh;
2015         struct mlx5_dev_config *sh_conf = NULL;
2016         uint16_t port_id;
2017
2018         assert(sh);
2019         /* Nothing to compare for the single/first device. */
2020         if (sh->refcnt == 1)
2021                 return 0;
2022         /* Find the device with shared context. */
2023         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
2024                 struct mlx5_priv *opriv =
2025                         rte_eth_devices[port_id].data->dev_private;
2026
2027                 if (opriv && opriv != priv && opriv->sh == sh) {
2028                         sh_conf = &opriv->config;
2029                         break;
2030                 }
2031         }
2032         if (!sh_conf)
2033                 return 0;
2034         if (sh_conf->dv_flow_en ^ config->dv_flow_en) {
2035                 DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch"
2036                              " for shared %s context", sh->ibdev_name);
2037                 rte_errno = EINVAL;
2038                 return rte_errno;
2039         }
2040         if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) {
2041                 DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch"
2042                              " for shared %s context", sh->ibdev_name);
2043                 rte_errno = EINVAL;
2044                 return rte_errno;
2045         }
2046         return 0;
2047 }
2048 /**
2049  * Spawn an Ethernet device from Verbs information.
2050  *
2051  * @param dpdk_dev
2052  *   Backing DPDK device.
2053  * @param spawn
2054  *   Verbs device parameters (name, port, switch_info) to spawn.
2055  * @param config
2056  *   Device configuration parameters.
2057  *
2058  * @return
2059  *   A valid Ethernet device object on success, NULL otherwise and rte_errno
2060  *   is set. The following errors are defined:
2061  *
2062  *   EBUSY: device is not supposed to be spawned.
2063  *   EEXIST: device is already spawned
2064  */
2065 static struct rte_eth_dev *
2066 mlx5_dev_spawn(struct rte_device *dpdk_dev,
2067                struct mlx5_dev_spawn_data *spawn,
2068                struct mlx5_dev_config config)
2069 {
2070         const struct mlx5_switch_info *switch_info = &spawn->info;
2071         struct mlx5_ibv_shared *sh = NULL;
2072         struct ibv_port_attr port_attr;
2073         struct mlx5dv_context dv_attr = { .comp_mask = 0 };
2074         struct rte_eth_dev *eth_dev = NULL;
2075         struct mlx5_priv *priv = NULL;
2076         int err = 0;
2077         unsigned int hw_padding = 0;
2078         unsigned int mps;
2079         unsigned int cqe_comp;
2080         unsigned int cqe_pad = 0;
2081         unsigned int tunnel_en = 0;
2082         unsigned int mpls_en = 0;
2083         unsigned int swp = 0;
2084         unsigned int mprq = 0;
2085         unsigned int mprq_min_stride_size_n = 0;
2086         unsigned int mprq_max_stride_size_n = 0;
2087         unsigned int mprq_min_stride_num_n = 0;
2088         unsigned int mprq_max_stride_num_n = 0;
2089         struct rte_ether_addr mac;
2090         char name[RTE_ETH_NAME_MAX_LEN];
2091         int own_domain_id = 0;
2092         uint16_t port_id;
2093         unsigned int i;
2094 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
2095         struct mlx5dv_devx_port devx_port = { .comp_mask = 0 };
2096 #endif
2097
2098         /* Determine if this port representor is supposed to be spawned. */
2099         if (switch_info->representor && dpdk_dev->devargs) {
2100                 struct rte_eth_devargs eth_da;
2101
2102                 err = rte_eth_devargs_parse(dpdk_dev->devargs->args, &eth_da);
2103                 if (err) {
2104                         rte_errno = -err;
2105                         DRV_LOG(ERR, "failed to process device arguments: %s",
2106                                 strerror(rte_errno));
2107                         return NULL;
2108                 }
2109                 for (i = 0; i < eth_da.nb_representor_ports; ++i)
2110                         if (eth_da.representor_ports[i] ==
2111                             (uint16_t)switch_info->port_name)
2112                                 break;
2113                 if (i == eth_da.nb_representor_ports) {
2114                         rte_errno = EBUSY;
2115                         return NULL;
2116                 }
2117         }
2118         /* Build device name. */
2119         if (spawn->pf_bond <  0) {
2120                 /* Single device. */
2121                 if (!switch_info->representor)
2122                         strlcpy(name, dpdk_dev->name, sizeof(name));
2123                 else
2124                         snprintf(name, sizeof(name), "%s_representor_%u",
2125                                  dpdk_dev->name, switch_info->port_name);
2126         } else {
2127                 /* Bonding device. */
2128                 if (!switch_info->representor)
2129                         snprintf(name, sizeof(name), "%s_%s",
2130                                  dpdk_dev->name, spawn->ibv_dev->name);
2131                 else
2132                         snprintf(name, sizeof(name), "%s_%s_representor_%u",
2133                                  dpdk_dev->name, spawn->ibv_dev->name,
2134                                  switch_info->port_name);
2135         }
2136         /* check if the device is already spawned */
2137         if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) {
2138                 rte_errno = EEXIST;
2139                 return NULL;
2140         }
2141         DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name);
2142         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
2143                 eth_dev = rte_eth_dev_attach_secondary(name);
2144                 if (eth_dev == NULL) {
2145                         DRV_LOG(ERR, "can not attach rte ethdev");
2146                         rte_errno = ENOMEM;
2147                         return NULL;
2148                 }
2149                 eth_dev->device = dpdk_dev;
2150                 eth_dev->dev_ops = &mlx5_dev_sec_ops;
2151                 err = mlx5_proc_priv_init(eth_dev);
2152                 if (err)
2153                         return NULL;
2154                 /* Receive command fd from primary process */
2155                 err = mlx5_mp_req_verbs_cmd_fd(eth_dev);
2156                 if (err < 0)
2157                         return NULL;
2158                 /* Remap UAR for Tx queues. */
2159                 err = mlx5_tx_uar_init_secondary(eth_dev, err);
2160                 if (err)
2161                         return NULL;
2162                 /*
2163                  * Ethdev pointer is still required as input since
2164                  * the primary device is not accessible from the
2165                  * secondary process.
2166                  */
2167                 eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev);
2168                 eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev);
2169                 return eth_dev;
2170         }
2171         /*
2172          * Some parameters ("tx_db_nc" in particularly) are needed in
2173          * advance to create dv/verbs device context. We proceed the
2174          * devargs here to get ones, and later proceed devargs again
2175          * to override some hardware settings.
2176          */
2177         err = mlx5_args(&config, dpdk_dev->devargs);
2178         if (err) {
2179                 err = rte_errno;
2180                 DRV_LOG(ERR, "failed to process device arguments: %s",
2181                         strerror(rte_errno));
2182                 goto error;
2183         }
2184         sh = mlx5_alloc_shared_ibctx(spawn, &config);
2185         if (!sh)
2186                 return NULL;
2187         config.devx = sh->devx;
2188 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
2189         config.dest_tir = 1;
2190 #endif
2191 #ifdef HAVE_IBV_MLX5_MOD_SWP
2192         dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
2193 #endif
2194         /*
2195          * Multi-packet send is supported by ConnectX-4 Lx PF as well
2196          * as all ConnectX-5 devices.
2197          */
2198 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
2199         dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
2200 #endif
2201 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
2202         dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
2203 #endif
2204         mlx5_glue->dv_query_device(sh->ctx, &dv_attr);
2205         if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
2206                 if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
2207                         DRV_LOG(DEBUG, "enhanced MPW is supported");
2208                         mps = MLX5_MPW_ENHANCED;
2209                 } else {
2210                         DRV_LOG(DEBUG, "MPW is supported");
2211                         mps = MLX5_MPW;
2212                 }
2213         } else {
2214                 DRV_LOG(DEBUG, "MPW isn't supported");
2215                 mps = MLX5_MPW_DISABLED;
2216         }
2217 #ifdef HAVE_IBV_MLX5_MOD_SWP
2218         if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
2219                 swp = dv_attr.sw_parsing_caps.sw_parsing_offloads;
2220         DRV_LOG(DEBUG, "SWP support: %u", swp);
2221 #endif
2222         config.swp = !!swp;
2223 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
2224         if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
2225                 struct mlx5dv_striding_rq_caps mprq_caps =
2226                         dv_attr.striding_rq_caps;
2227
2228                 DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d",
2229                         mprq_caps.min_single_stride_log_num_of_bytes);
2230                 DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %d",
2231                         mprq_caps.max_single_stride_log_num_of_bytes);
2232                 DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %d",
2233                         mprq_caps.min_single_wqe_log_num_of_strides);
2234                 DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %d",
2235                         mprq_caps.max_single_wqe_log_num_of_strides);
2236                 DRV_LOG(DEBUG, "\tsupported_qpts: %d",
2237                         mprq_caps.supported_qpts);
2238                 DRV_LOG(DEBUG, "device supports Multi-Packet RQ");
2239                 mprq = 1;
2240                 mprq_min_stride_size_n =
2241                         mprq_caps.min_single_stride_log_num_of_bytes;
2242                 mprq_max_stride_size_n =
2243                         mprq_caps.max_single_stride_log_num_of_bytes;
2244                 mprq_min_stride_num_n =
2245                         mprq_caps.min_single_wqe_log_num_of_strides;
2246                 mprq_max_stride_num_n =
2247                         mprq_caps.max_single_wqe_log_num_of_strides;
2248                 config.mprq.stride_num_n = RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
2249                                                    mprq_min_stride_num_n);
2250         }
2251 #endif
2252         if (RTE_CACHE_LINE_SIZE == 128 &&
2253             !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
2254                 cqe_comp = 0;
2255         else
2256                 cqe_comp = 1;
2257         config.cqe_comp = cqe_comp;
2258 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
2259         /* Whether device supports 128B Rx CQE padding. */
2260         cqe_pad = RTE_CACHE_LINE_SIZE == 128 &&
2261                   (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD);
2262 #endif
2263 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
2264         if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
2265                 tunnel_en = ((dv_attr.tunnel_offloads_caps &
2266                               MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) &&
2267                              (dv_attr.tunnel_offloads_caps &
2268                               MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE) &&
2269                              (dv_attr.tunnel_offloads_caps &
2270                               MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE));
2271         }
2272         DRV_LOG(DEBUG, "tunnel offloading is %ssupported",
2273                 tunnel_en ? "" : "not ");
2274 #else
2275         DRV_LOG(WARNING,
2276                 "tunnel offloading disabled due to old OFED/rdma-core version");
2277 #endif
2278         config.tunnel_en = tunnel_en;
2279 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2280         mpls_en = ((dv_attr.tunnel_offloads_caps &
2281                     MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) &&
2282                    (dv_attr.tunnel_offloads_caps &
2283                     MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP));
2284         DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported",
2285                 mpls_en ? "" : "not ");
2286 #else
2287         DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to"
2288                 " old OFED/rdma-core version or firmware configuration");
2289 #endif
2290         config.mpls_en = mpls_en;
2291         /* Check port status. */
2292         err = mlx5_glue->query_port(sh->ctx, spawn->ibv_port, &port_attr);
2293         if (err) {
2294                 DRV_LOG(ERR, "port query failed: %s", strerror(err));
2295                 goto error;
2296         }
2297         if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
2298                 DRV_LOG(ERR, "port is not configured in Ethernet mode");
2299                 err = EINVAL;
2300                 goto error;
2301         }
2302         if (port_attr.state != IBV_PORT_ACTIVE)
2303                 DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)",
2304                         mlx5_glue->port_state_str(port_attr.state),
2305                         port_attr.state);
2306         /* Allocate private eth device data. */
2307         priv = rte_zmalloc("ethdev private structure",
2308                            sizeof(*priv),
2309                            RTE_CACHE_LINE_SIZE);
2310         if (priv == NULL) {
2311                 DRV_LOG(ERR, "priv allocation failure");
2312                 err = ENOMEM;
2313                 goto error;
2314         }
2315         priv->sh = sh;
2316         priv->ibv_port = spawn->ibv_port;
2317         priv->pci_dev = spawn->pci_dev;
2318         priv->mtu = RTE_ETHER_MTU;
2319 #ifndef RTE_ARCH_64
2320         /* Initialize UAR access locks for 32bit implementations. */
2321         rte_spinlock_init(&priv->uar_lock_cq);
2322         for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++)
2323                 rte_spinlock_init(&priv->uar_lock[i]);
2324 #endif
2325         /* Some internal functions rely on Netlink sockets, open them now. */
2326         priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA);
2327         priv->nl_socket_route = mlx5_nl_init(NETLINK_ROUTE);
2328         priv->nl_sn = 0;
2329         priv->representor = !!switch_info->representor;
2330         priv->master = !!switch_info->master;
2331         priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
2332         priv->vport_meta_tag = 0;
2333         priv->vport_meta_mask = 0;
2334         priv->pf_bond = spawn->pf_bond;
2335 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
2336         /*
2337          * The DevX port query API is implemented. E-Switch may use
2338          * either vport or reg_c[0] metadata register to match on
2339          * vport index. The engaged part of metadata register is
2340          * defined by mask.
2341          */
2342         if (switch_info->representor || switch_info->master) {
2343                 devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT |
2344                                       MLX5DV_DEVX_PORT_MATCH_REG_C_0;
2345                 err = mlx5_glue->devx_port_query(sh->ctx, spawn->ibv_port,
2346                                                  &devx_port);
2347                 if (err) {
2348                         DRV_LOG(WARNING,
2349                                 "can't query devx port %d on device %s",
2350                                 spawn->ibv_port, spawn->ibv_dev->name);
2351                         devx_port.comp_mask = 0;
2352                 }
2353         }
2354         if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) {
2355                 priv->vport_meta_tag = devx_port.reg_c_0.value;
2356                 priv->vport_meta_mask = devx_port.reg_c_0.mask;
2357                 if (!priv->vport_meta_mask) {
2358                         DRV_LOG(ERR, "vport zero mask for port %d"
2359                                      " on bonding device %s",
2360                                      spawn->ibv_port, spawn->ibv_dev->name);
2361                         err = ENOTSUP;
2362                         goto error;
2363                 }
2364                 if (priv->vport_meta_tag & ~priv->vport_meta_mask) {
2365                         DRV_LOG(ERR, "invalid vport tag for port %d"
2366                                      " on bonding device %s",
2367                                      spawn->ibv_port, spawn->ibv_dev->name);
2368                         err = ENOTSUP;
2369                         goto error;
2370                 }
2371         }
2372         if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) {
2373                 priv->vport_id = devx_port.vport_num;
2374         } else if (spawn->pf_bond >= 0) {
2375                 DRV_LOG(ERR, "can't deduce vport index for port %d"
2376                              " on bonding device %s",
2377                              spawn->ibv_port, spawn->ibv_dev->name);
2378                 err = ENOTSUP;
2379                 goto error;
2380         } else {
2381                 /* Suppose vport index in compatible way. */
2382                 priv->vport_id = switch_info->representor ?
2383                                  switch_info->port_name + 1 : -1;
2384         }
2385 #else
2386         /*
2387          * Kernel/rdma_core support single E-Switch per PF configurations
2388          * only and vport_id field contains the vport index for
2389          * associated VF, which is deduced from representor port name.
2390          * For example, let's have the IB device port 10, it has
2391          * attached network device eth0, which has port name attribute
2392          * pf0vf2, we can deduce the VF number as 2, and set vport index
2393          * as 3 (2+1). This assigning schema should be changed if the
2394          * multiple E-Switch instances per PF configurations or/and PCI
2395          * subfunctions are added.
2396          */
2397         priv->vport_id = switch_info->representor ?
2398                          switch_info->port_name + 1 : -1;
2399 #endif
2400         /* representor_id field keeps the unmodified VF index. */
2401         priv->representor_id = switch_info->representor ?
2402                                switch_info->port_name : -1;
2403         /*
2404          * Look for sibling devices in order to reuse their switch domain
2405          * if any, otherwise allocate one.
2406          */
2407         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
2408                 const struct mlx5_priv *opriv =
2409                         rte_eth_devices[port_id].data->dev_private;
2410
2411                 if (!opriv ||
2412                     opriv->sh != priv->sh ||
2413                         opriv->domain_id ==
2414                         RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
2415                         continue;
2416                 priv->domain_id = opriv->domain_id;
2417                 break;
2418         }
2419         if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
2420                 err = rte_eth_switch_domain_alloc(&priv->domain_id);
2421                 if (err) {
2422                         err = rte_errno;
2423                         DRV_LOG(ERR, "unable to allocate switch domain: %s",
2424                                 strerror(rte_errno));
2425                         goto error;
2426                 }
2427                 own_domain_id = 1;
2428         }
2429         /* Override some values set by hardware configuration. */
2430         mlx5_args(&config, dpdk_dev->devargs);
2431         err = mlx5_dev_check_sibling_config(priv, &config);
2432         if (err)
2433                 goto error;
2434         config.hw_csum = !!(sh->device_attr.device_cap_flags_ex &
2435                             IBV_DEVICE_RAW_IP_CSUM);
2436         DRV_LOG(DEBUG, "checksum offloading is %ssupported",
2437                 (config.hw_csum ? "" : "not "));
2438 #if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
2439         !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
2440         DRV_LOG(DEBUG, "counters are not supported");
2441 #endif
2442 #if !defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_MLX5DV_DR)
2443         if (config.dv_flow_en) {
2444                 DRV_LOG(WARNING, "DV flow is not supported");
2445                 config.dv_flow_en = 0;
2446         }
2447 #endif
2448         config.ind_table_max_size =
2449                 sh->device_attr.rss_caps.max_rwq_indirection_table_size;
2450         /*
2451          * Remove this check once DPDK supports larger/variable
2452          * indirection tables.
2453          */
2454         if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512)
2455                 config.ind_table_max_size = ETH_RSS_RETA_SIZE_512;
2456         DRV_LOG(DEBUG, "maximum Rx indirection table size is %u",
2457                 config.ind_table_max_size);
2458         config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps &
2459                                   IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
2460         DRV_LOG(DEBUG, "VLAN stripping is %ssupported",
2461                 (config.hw_vlan_strip ? "" : "not "));
2462         config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps &
2463                                  IBV_RAW_PACKET_CAP_SCATTER_FCS);
2464         DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported",
2465                 (config.hw_fcs_strip ? "" : "not "));
2466 #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
2467         hw_padding = !!sh->device_attr.rx_pad_end_addr_align;
2468 #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
2469         hw_padding = !!(sh->device_attr.device_cap_flags_ex &
2470                         IBV_DEVICE_PCI_WRITE_END_PADDING);
2471 #endif
2472         if (config.hw_padding && !hw_padding) {
2473                 DRV_LOG(DEBUG, "Rx end alignment padding isn't supported");
2474                 config.hw_padding = 0;
2475         } else if (config.hw_padding) {
2476                 DRV_LOG(DEBUG, "Rx end alignment padding is enabled");
2477         }
2478         config.tso = (sh->device_attr.tso_caps.max_tso > 0 &&
2479                       (sh->device_attr.tso_caps.supported_qpts &
2480                        (1 << IBV_QPT_RAW_PACKET)));
2481         if (config.tso)
2482                 config.tso_max_payload_sz = sh->device_attr.tso_caps.max_tso;
2483         /*
2484          * MPW is disabled by default, while the Enhanced MPW is enabled
2485          * by default.
2486          */
2487         if (config.mps == MLX5_ARG_UNSET)
2488                 config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED :
2489                                                           MLX5_MPW_DISABLED;
2490         else
2491                 config.mps = config.mps ? mps : MLX5_MPW_DISABLED;
2492         DRV_LOG(INFO, "%sMPS is %s",
2493                 config.mps == MLX5_MPW_ENHANCED ? "enhanced " :
2494                 config.mps == MLX5_MPW ? "legacy " : "",
2495                 config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
2496         if (config.cqe_comp && !cqe_comp) {
2497                 DRV_LOG(WARNING, "Rx CQE compression isn't supported");
2498                 config.cqe_comp = 0;
2499         }
2500         if (config.cqe_pad && !cqe_pad) {
2501                 DRV_LOG(WARNING, "Rx CQE padding isn't supported");
2502                 config.cqe_pad = 0;
2503         } else if (config.cqe_pad) {
2504                 DRV_LOG(INFO, "Rx CQE padding is enabled");
2505         }
2506         if (config.devx) {
2507                 priv->counter_fallback = 0;
2508                 err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr);
2509                 if (err) {
2510                         err = -err;
2511                         goto error;
2512                 }
2513                 if (!config.hca_attr.flow_counters_dump)
2514                         priv->counter_fallback = 1;
2515 #ifndef HAVE_IBV_DEVX_ASYNC
2516                 priv->counter_fallback = 1;
2517 #endif
2518                 if (priv->counter_fallback)
2519                         DRV_LOG(INFO, "Use fall-back DV counter management");
2520                 /* Check for LRO support. */
2521                 if (config.dest_tir && config.hca_attr.lro_cap &&
2522                     config.dv_flow_en) {
2523                         /* TBD check tunnel lro caps. */
2524                         config.lro.supported = config.hca_attr.lro_cap;
2525                         DRV_LOG(DEBUG, "Device supports LRO");
2526                         /*
2527                          * If LRO timeout is not configured by application,
2528                          * use the minimal supported value.
2529                          */
2530                         if (!config.lro.timeout)
2531                                 config.lro.timeout =
2532                                 config.hca_attr.lro_timer_supported_periods[0];
2533                         DRV_LOG(DEBUG, "LRO session timeout set to %d usec",
2534                                 config.lro.timeout);
2535                 }
2536 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
2537                 if (config.hca_attr.qos.sup && config.hca_attr.qos.srtcm_sup &&
2538                     config.dv_flow_en) {
2539                         uint8_t reg_c_mask =
2540                                 config.hca_attr.qos.flow_meter_reg_c_ids;
2541                         /*
2542                          * Meter needs two REG_C's for color match and pre-sfx
2543                          * flow match. Here get the REG_C for color match.
2544                          * REG_C_0 and REG_C_1 is reserved for metadata feature.
2545                          */
2546                         reg_c_mask &= 0xfc;
2547                         if (__builtin_popcount(reg_c_mask) < 1) {
2548                                 priv->mtr_en = 0;
2549                                 DRV_LOG(WARNING, "No available register for"
2550                                         " meter.");
2551                         } else {
2552                                 priv->mtr_color_reg = ffs(reg_c_mask) - 1 +
2553                                                       REG_C_0;
2554                                 priv->mtr_en = 1;
2555                                 priv->mtr_reg_share =
2556                                       config.hca_attr.qos.flow_meter_reg_share;
2557                                 DRV_LOG(DEBUG, "The REG_C meter uses is %d",
2558                                         priv->mtr_color_reg);
2559                         }
2560                 }
2561 #endif
2562         }
2563         if (config.mprq.enabled && mprq) {
2564                 if (config.mprq.stride_num_n > mprq_max_stride_num_n ||
2565                     config.mprq.stride_num_n < mprq_min_stride_num_n) {
2566                         config.mprq.stride_num_n =
2567                                 RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
2568                                         mprq_min_stride_num_n);
2569                         DRV_LOG(WARNING,
2570                                 "the number of strides"
2571                                 " for Multi-Packet RQ is out of range,"
2572                                 " setting default value (%u)",
2573                                 1 << config.mprq.stride_num_n);
2574                 }
2575                 config.mprq.min_stride_size_n = mprq_min_stride_size_n;
2576                 config.mprq.max_stride_size_n = mprq_max_stride_size_n;
2577         } else if (config.mprq.enabled && !mprq) {
2578                 DRV_LOG(WARNING, "Multi-Packet RQ isn't supported");
2579                 config.mprq.enabled = 0;
2580         }
2581         if (config.max_dump_files_num == 0)
2582                 config.max_dump_files_num = 128;
2583         eth_dev = rte_eth_dev_allocate(name);
2584         if (eth_dev == NULL) {
2585                 DRV_LOG(ERR, "can not allocate rte ethdev");
2586                 err = ENOMEM;
2587                 goto error;
2588         }
2589         /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */
2590         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
2591         if (priv->representor) {
2592                 eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
2593                 eth_dev->data->representor_id = priv->representor_id;
2594         }
2595         /*
2596          * Store associated network device interface index. This index
2597          * is permanent throughout the lifetime of device. So, we may store
2598          * the ifindex here and use the cached value further.
2599          */
2600         assert(spawn->ifindex);
2601         priv->if_index = spawn->ifindex;
2602         eth_dev->data->dev_private = priv;
2603         priv->dev_data = eth_dev->data;
2604         eth_dev->data->mac_addrs = priv->mac;
2605         eth_dev->device = dpdk_dev;
2606         /* Configure the first MAC address by default. */
2607         if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
2608                 DRV_LOG(ERR,
2609                         "port %u cannot get MAC address, is mlx5_en"
2610                         " loaded? (errno: %s)",
2611                         eth_dev->data->port_id, strerror(rte_errno));
2612                 err = ENODEV;
2613                 goto error;
2614         }
2615         DRV_LOG(INFO,
2616                 "port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
2617                 eth_dev->data->port_id,
2618                 mac.addr_bytes[0], mac.addr_bytes[1],
2619                 mac.addr_bytes[2], mac.addr_bytes[3],
2620                 mac.addr_bytes[4], mac.addr_bytes[5]);
2621 #ifndef NDEBUG
2622         {
2623                 char ifname[IF_NAMESIZE];
2624
2625                 if (mlx5_get_ifname(eth_dev, &ifname) == 0)
2626                         DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
2627                                 eth_dev->data->port_id, ifname);
2628                 else
2629                         DRV_LOG(DEBUG, "port %u ifname is unknown",
2630                                 eth_dev->data->port_id);
2631         }
2632 #endif
2633         /* Get actual MTU if possible. */
2634         err = mlx5_get_mtu(eth_dev, &priv->mtu);
2635         if (err) {
2636                 err = rte_errno;
2637                 goto error;
2638         }
2639         DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
2640                 priv->mtu);
2641         /* Initialize burst functions to prevent crashes before link-up. */
2642         eth_dev->rx_pkt_burst = removed_rx_burst;
2643         eth_dev->tx_pkt_burst = removed_tx_burst;
2644         eth_dev->dev_ops = &mlx5_dev_ops;
2645         /* Register MAC address. */
2646         claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
2647         if (config.vf && config.vf_nl_en)
2648                 mlx5_nl_mac_addr_sync(eth_dev);
2649         TAILQ_INIT(&priv->flows);
2650         TAILQ_INIT(&priv->ctrl_flows);
2651         TAILQ_INIT(&priv->flow_meters);
2652         TAILQ_INIT(&priv->flow_meter_profiles);
2653         /* Hint libmlx5 to use PMD allocator for data plane resources */
2654         struct mlx5dv_ctx_allocators alctr = {
2655                 .alloc = &mlx5_alloc_verbs_buf,
2656                 .free = &mlx5_free_verbs_buf,
2657                 .data = priv,
2658         };
2659         mlx5_glue->dv_set_context_attr(sh->ctx,
2660                                        MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
2661                                        (void *)((uintptr_t)&alctr));
2662         /* Bring Ethernet device up. */
2663         DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
2664                 eth_dev->data->port_id);
2665         mlx5_set_link_up(eth_dev);
2666         /*
2667          * Even though the interrupt handler is not installed yet,
2668          * interrupts will still trigger on the async_fd from
2669          * Verbs context returned by ibv_open_device().
2670          */
2671         mlx5_link_update(eth_dev, 0);
2672 #ifdef HAVE_MLX5DV_DR_ESWITCH
2673         if (!(config.hca_attr.eswitch_manager && config.dv_flow_en &&
2674               (switch_info->representor || switch_info->master)))
2675                 config.dv_esw_en = 0;
2676 #else
2677         config.dv_esw_en = 0;
2678 #endif
2679         /* Detect minimal data bytes to inline. */
2680         mlx5_set_min_inline(spawn, &config);
2681         /* Store device configuration on private structure. */
2682         priv->config = config;
2683         /* Create context for virtual machine VLAN workaround. */
2684         priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
2685         if (config.dv_flow_en) {
2686                 err = mlx5_alloc_shared_dr(priv);
2687                 if (err)
2688                         goto error;
2689                 /*
2690                  * RSS id is shared with meter flow id. Meter flow id can only
2691                  * use the 24 MSB of the register.
2692                  */
2693                 priv->qrss_id_pool = mlx5_flow_id_pool_alloc(UINT32_MAX >>
2694                                      MLX5_MTR_COLOR_BITS);
2695                 if (!priv->qrss_id_pool) {
2696                         DRV_LOG(ERR, "can't create flow id pool");
2697                         err = ENOMEM;
2698                         goto error;
2699                 }
2700         }
2701         /* Supported Verbs flow priority number detection. */
2702         err = mlx5_flow_discover_priorities(eth_dev);
2703         if (err < 0) {
2704                 err = -err;
2705                 goto error;
2706         }
2707         priv->config.flow_prio = err;
2708         if (!priv->config.dv_esw_en &&
2709             priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2710                 DRV_LOG(WARNING, "metadata mode %u is not supported "
2711                                  "(no E-Switch)", priv->config.dv_xmeta_en);
2712                 priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY;
2713         }
2714         mlx5_set_metadata_mask(eth_dev);
2715         if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
2716             !priv->sh->dv_regc0_mask) {
2717                 DRV_LOG(ERR, "metadata mode %u is not supported "
2718                              "(no metadata reg_c[0] is available)",
2719                              priv->config.dv_xmeta_en);
2720                         err = ENOTSUP;
2721                         goto error;
2722         }
2723         /* Query availibility of metadata reg_c's. */
2724         err = mlx5_flow_discover_mreg_c(eth_dev);
2725         if (err < 0) {
2726                 err = -err;
2727                 goto error;
2728         }
2729         if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
2730                 DRV_LOG(DEBUG,
2731                         "port %u extensive metadata register is not supported",
2732                         eth_dev->data->port_id);
2733                 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2734                         DRV_LOG(ERR, "metadata mode %u is not supported "
2735                                      "(no metadata registers available)",
2736                                      priv->config.dv_xmeta_en);
2737                         err = ENOTSUP;
2738                         goto error;
2739                 }
2740         }
2741         if (priv->config.dv_flow_en &&
2742             priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
2743             mlx5_flow_ext_mreg_supported(eth_dev) &&
2744             priv->sh->dv_regc0_mask) {
2745                 priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
2746                                                       MLX5_FLOW_MREG_HTABLE_SZ);
2747                 if (!priv->mreg_cp_tbl) {
2748                         err = ENOMEM;
2749                         goto error;
2750                 }
2751         }
2752         return eth_dev;
2753 error:
2754         if (priv) {
2755                 if (priv->mreg_cp_tbl)
2756                         mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
2757                 if (priv->sh)
2758                         mlx5_free_shared_dr(priv);
2759                 if (priv->nl_socket_route >= 0)
2760                         close(priv->nl_socket_route);
2761                 if (priv->nl_socket_rdma >= 0)
2762                         close(priv->nl_socket_rdma);
2763                 if (priv->vmwa_context)
2764                         mlx5_vlan_vmwa_exit(priv->vmwa_context);
2765                 if (priv->qrss_id_pool)
2766                         mlx5_flow_id_pool_release(priv->qrss_id_pool);
2767                 if (own_domain_id)
2768                         claim_zero(rte_eth_switch_domain_free(priv->domain_id));
2769                 rte_free(priv);
2770                 if (eth_dev != NULL)
2771                         eth_dev->data->dev_private = NULL;
2772         }
2773         if (eth_dev != NULL) {
2774                 /* mac_addrs must not be freed alone because part of dev_private */
2775                 eth_dev->data->mac_addrs = NULL;
2776                 rte_eth_dev_release_port(eth_dev);
2777         }
2778         if (sh)
2779                 mlx5_free_shared_ibctx(sh);
2780         assert(err > 0);
2781         rte_errno = err;
2782         return NULL;
2783 }
2784
2785 /**
2786  * Comparison callback to sort device data.
2787  *
2788  * This is meant to be used with qsort().
2789  *
2790  * @param a[in]
2791  *   Pointer to pointer to first data object.
2792  * @param b[in]
2793  *   Pointer to pointer to second data object.
2794  *
2795  * @return
2796  *   0 if both objects are equal, less than 0 if the first argument is less
2797  *   than the second, greater than 0 otherwise.
2798  */
2799 static int
2800 mlx5_dev_spawn_data_cmp(const void *a, const void *b)
2801 {
2802         const struct mlx5_switch_info *si_a =
2803                 &((const struct mlx5_dev_spawn_data *)a)->info;
2804         const struct mlx5_switch_info *si_b =
2805                 &((const struct mlx5_dev_spawn_data *)b)->info;
2806         int ret;
2807
2808         /* Master device first. */
2809         ret = si_b->master - si_a->master;
2810         if (ret)
2811                 return ret;
2812         /* Then representor devices. */
2813         ret = si_b->representor - si_a->representor;
2814         if (ret)
2815                 return ret;
2816         /* Unidentified devices come last in no specific order. */
2817         if (!si_a->representor)
2818                 return 0;
2819         /* Order representors by name. */
2820         return si_a->port_name - si_b->port_name;
2821 }
2822
2823 /**
2824  * Match PCI information for possible slaves of bonding device.
2825  *
2826  * @param[in] ibv_dev
2827  *   Pointer to Infiniband device structure.
2828  * @param[in] pci_dev
2829  *   Pointer to PCI device structure to match PCI address.
2830  * @param[in] nl_rdma
2831  *   Netlink RDMA group socket handle.
2832  *
2833  * @return
2834  *   negative value if no bonding device found, otherwise
2835  *   positive index of slave PF in bonding.
2836  */
2837 static int
2838 mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev,
2839                            const struct rte_pci_device *pci_dev,
2840                            int nl_rdma)
2841 {
2842         char ifname[IF_NAMESIZE + 1];
2843         unsigned int ifindex;
2844         unsigned int np, i;
2845         FILE *file = NULL;
2846         int pf = -1;
2847
2848         /*
2849          * Try to get master device name. If something goes
2850          * wrong suppose the lack of kernel support and no
2851          * bonding devices.
2852          */
2853         if (nl_rdma < 0)
2854                 return -1;
2855         if (!strstr(ibv_dev->name, "bond"))
2856                 return -1;
2857         np = mlx5_nl_portnum(nl_rdma, ibv_dev->name);
2858         if (!np)
2859                 return -1;
2860         /*
2861          * The Master device might not be on the predefined
2862          * port (not on port index 1, it is not garanted),
2863          * we have to scan all Infiniband device port and
2864          * find master.
2865          */
2866         for (i = 1; i <= np; ++i) {
2867                 /* Check whether Infiniband port is populated. */
2868                 ifindex = mlx5_nl_ifindex(nl_rdma, ibv_dev->name, i);
2869                 if (!ifindex)
2870                         continue;
2871                 if (!if_indextoname(ifindex, ifname))
2872                         continue;
2873                 /* Try to read bonding slave names from sysfs. */
2874                 MKSTR(slaves,
2875                       "/sys/class/net/%s/master/bonding/slaves", ifname);
2876                 file = fopen(slaves, "r");
2877                 if (file)
2878                         break;
2879         }
2880         if (!file)
2881                 return -1;
2882         /* Use safe format to check maximal buffer length. */
2883         assert(atol(RTE_STR(IF_NAMESIZE)) == IF_NAMESIZE);
2884         while (fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", ifname) == 1) {
2885                 char tmp_str[IF_NAMESIZE + 32];
2886                 struct rte_pci_addr pci_addr;
2887                 struct mlx5_switch_info info;
2888
2889                 /* Process slave interface names in the loop. */
2890                 snprintf(tmp_str, sizeof(tmp_str),
2891                          "/sys/class/net/%s", ifname);
2892                 if (mlx5_dev_to_pci_addr(tmp_str, &pci_addr)) {
2893                         DRV_LOG(WARNING, "can not get PCI address"
2894                                          " for netdev \"%s\"", ifname);
2895                         continue;
2896                 }
2897                 if (pci_dev->addr.domain != pci_addr.domain ||
2898                     pci_dev->addr.bus != pci_addr.bus ||
2899                     pci_dev->addr.devid != pci_addr.devid ||
2900                     pci_dev->addr.function != pci_addr.function)
2901                         continue;
2902                 /* Slave interface PCI address match found. */
2903                 fclose(file);
2904                 snprintf(tmp_str, sizeof(tmp_str),
2905                          "/sys/class/net/%s/phys_port_name", ifname);
2906                 file = fopen(tmp_str, "rb");
2907                 if (!file)
2908                         break;
2909                 info.name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET;
2910                 if (fscanf(file, "%32s", tmp_str) == 1)
2911                         mlx5_translate_port_name(tmp_str, &info);
2912                 if (info.name_type == MLX5_PHYS_PORT_NAME_TYPE_LEGACY ||
2913                     info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK)
2914                         pf = info.port_name;
2915                 break;
2916         }
2917         if (file)
2918                 fclose(file);
2919         return pf;
2920 }
2921
2922 /**
2923  * DPDK callback to register a PCI device.
2924  *
2925  * This function spawns Ethernet devices out of a given PCI device.
2926  *
2927  * @param[in] pci_drv
2928  *   PCI driver structure (mlx5_driver).
2929  * @param[in] pci_dev
2930  *   PCI device information.
2931  *
2932  * @return
2933  *   0 on success, a negative errno value otherwise and rte_errno is set.
2934  */
2935 static int
2936 mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2937                struct rte_pci_device *pci_dev)
2938 {
2939         struct ibv_device **ibv_list;
2940         /*
2941          * Number of found IB Devices matching with requested PCI BDF.
2942          * nd != 1 means there are multiple IB devices over the same
2943          * PCI device and we have representors and master.
2944          */
2945         unsigned int nd = 0;
2946         /*
2947          * Number of found IB device Ports. nd = 1 and np = 1..n means
2948          * we have the single multiport IB device, and there may be
2949          * representors attached to some of found ports.
2950          */
2951         unsigned int np = 0;
2952         /*
2953          * Number of DPDK ethernet devices to Spawn - either over
2954          * multiple IB devices or multiple ports of single IB device.
2955          * Actually this is the number of iterations to spawn.
2956          */
2957         unsigned int ns = 0;
2958         /*
2959          * Bonding device
2960          *   < 0 - no bonding device (single one)
2961          *  >= 0 - bonding device (value is slave PF index)
2962          */
2963         int bd = -1;
2964         struct mlx5_dev_spawn_data *list = NULL;
2965         struct mlx5_dev_config dev_config;
2966         int ret;
2967
2968         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2969                 mlx5_pmd_socket_init();
2970         ret = mlx5_init_once();
2971         if (ret) {
2972                 DRV_LOG(ERR, "unable to init PMD global data: %s",
2973                         strerror(rte_errno));
2974                 return -rte_errno;
2975         }
2976         assert(pci_drv == &mlx5_driver);
2977         errno = 0;
2978         ibv_list = mlx5_glue->get_device_list(&ret);
2979         if (!ibv_list) {
2980                 rte_errno = errno ? errno : ENOSYS;
2981                 DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?");
2982                 return -rte_errno;
2983         }
2984         /*
2985          * First scan the list of all Infiniband devices to find
2986          * matching ones, gathering into the list.
2987          */
2988         struct ibv_device *ibv_match[ret + 1];
2989         int nl_route = mlx5_nl_init(NETLINK_ROUTE);
2990         int nl_rdma = mlx5_nl_init(NETLINK_RDMA);
2991         unsigned int i;
2992
2993         while (ret-- > 0) {
2994                 struct rte_pci_addr pci_addr;
2995
2996                 DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name);
2997                 bd = mlx5_device_bond_pci_match
2998                                 (ibv_list[ret], pci_dev, nl_rdma);
2999                 if (bd >= 0) {
3000                         /*
3001                          * Bonding device detected. Only one match is allowed,
3002                          * the bonding is supported over multi-port IB device,
3003                          * there should be no matches on representor PCI
3004                          * functions or non VF LAG bonding devices with
3005                          * specified address.
3006                          */
3007                         if (nd) {
3008                                 DRV_LOG(ERR,
3009                                         "multiple PCI match on bonding device"
3010                                         "\"%s\" found", ibv_list[ret]->name);
3011                                 rte_errno = ENOENT;
3012                                 ret = -rte_errno;
3013                                 goto exit;
3014                         }
3015                         DRV_LOG(INFO, "PCI information matches for"
3016                                       " slave %d bonding device \"%s\"",
3017                                       bd, ibv_list[ret]->name);
3018                         ibv_match[nd++] = ibv_list[ret];
3019                         break;
3020                 }
3021                 if (mlx5_dev_to_pci_addr
3022                         (ibv_list[ret]->ibdev_path, &pci_addr))
3023                         continue;
3024                 if (pci_dev->addr.domain != pci_addr.domain ||
3025                     pci_dev->addr.bus != pci_addr.bus ||
3026                     pci_dev->addr.devid != pci_addr.devid ||
3027                     pci_dev->addr.function != pci_addr.function)
3028                         continue;
3029                 DRV_LOG(INFO, "PCI information matches for device \"%s\"",
3030                         ibv_list[ret]->name);
3031                 ibv_match[nd++] = ibv_list[ret];
3032         }
3033         ibv_match[nd] = NULL;
3034         if (!nd) {
3035                 /* No device matches, just complain and bail out. */
3036                 DRV_LOG(WARNING,
3037                         "no Verbs device matches PCI device " PCI_PRI_FMT ","
3038                         " are kernel drivers loaded?",
3039                         pci_dev->addr.domain, pci_dev->addr.bus,
3040                         pci_dev->addr.devid, pci_dev->addr.function);
3041                 rte_errno = ENOENT;
3042                 ret = -rte_errno;
3043                 goto exit;
3044         }
3045         if (nd == 1) {
3046                 /*
3047                  * Found single matching device may have multiple ports.
3048                  * Each port may be representor, we have to check the port
3049                  * number and check the representors existence.
3050                  */
3051                 if (nl_rdma >= 0)
3052                         np = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name);
3053                 if (!np)
3054                         DRV_LOG(WARNING, "can not get IB device \"%s\""
3055                                          " ports number", ibv_match[0]->name);
3056                 if (bd >= 0 && !np) {
3057                         DRV_LOG(ERR, "can not get ports"
3058                                      " for bonding device");
3059                         rte_errno = ENOENT;
3060                         ret = -rte_errno;
3061                         goto exit;
3062                 }
3063         }
3064 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
3065         if (bd >= 0) {
3066                 /*
3067                  * This may happen if there is VF LAG kernel support and
3068                  * application is compiled with older rdma_core library.
3069                  */
3070                 DRV_LOG(ERR,
3071                         "No kernel/verbs support for VF LAG bonding found.");
3072                 rte_errno = ENOTSUP;
3073                 ret = -rte_errno;
3074                 goto exit;
3075         }
3076 #endif
3077         /*
3078          * Now we can determine the maximal
3079          * amount of devices to be spawned.
3080          */
3081         list = rte_zmalloc("device spawn data",
3082                          sizeof(struct mlx5_dev_spawn_data) *
3083                          (np ? np : nd),
3084                          RTE_CACHE_LINE_SIZE);
3085         if (!list) {
3086                 DRV_LOG(ERR, "spawn data array allocation failure");
3087                 rte_errno = ENOMEM;
3088                 ret = -rte_errno;
3089                 goto exit;
3090         }
3091         if (bd >= 0 || np > 1) {
3092                 /*
3093                  * Single IB device with multiple ports found,
3094                  * it may be E-Switch master device and representors.
3095                  * We have to perform identification trough the ports.
3096                  */
3097                 assert(nl_rdma >= 0);
3098                 assert(ns == 0);
3099                 assert(nd == 1);
3100                 assert(np);
3101                 for (i = 1; i <= np; ++i) {
3102                         list[ns].max_port = np;
3103                         list[ns].ibv_port = i;
3104                         list[ns].ibv_dev = ibv_match[0];
3105                         list[ns].eth_dev = NULL;
3106                         list[ns].pci_dev = pci_dev;
3107                         list[ns].pf_bond = bd;
3108                         list[ns].ifindex = mlx5_nl_ifindex
3109                                         (nl_rdma, list[ns].ibv_dev->name, i);
3110                         if (!list[ns].ifindex) {
3111                                 /*
3112                                  * No network interface index found for the
3113                                  * specified port, it means there is no
3114                                  * representor on this port. It's OK,
3115                                  * there can be disabled ports, for example
3116                                  * if sriov_numvfs < sriov_totalvfs.
3117                                  */
3118                                 continue;
3119                         }
3120                         ret = -1;
3121                         if (nl_route >= 0)
3122                                 ret = mlx5_nl_switch_info
3123                                                (nl_route,
3124                                                 list[ns].ifindex,
3125                                                 &list[ns].info);
3126                         if (ret || (!list[ns].info.representor &&
3127                                     !list[ns].info.master)) {
3128                                 /*
3129                                  * We failed to recognize representors with
3130                                  * Netlink, let's try to perform the task
3131                                  * with sysfs.
3132                                  */
3133                                 ret =  mlx5_sysfs_switch_info
3134                                                 (list[ns].ifindex,
3135                                                  &list[ns].info);
3136                         }
3137                         if (!ret && bd >= 0) {
3138                                 switch (list[ns].info.name_type) {
3139                                 case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
3140                                         if (list[ns].info.port_name == bd)
3141                                                 ns++;
3142                                         break;
3143                                 case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
3144                                         if (list[ns].info.pf_num == bd)
3145                                                 ns++;
3146                                         break;
3147                                 default:
3148                                         break;
3149                                 }
3150                                 continue;
3151                         }
3152                         if (!ret && (list[ns].info.representor ^
3153                                      list[ns].info.master))
3154                                 ns++;
3155                 }
3156                 if (!ns) {
3157                         DRV_LOG(ERR,
3158                                 "unable to recognize master/representors"
3159                                 " on the IB device with multiple ports");
3160                         rte_errno = ENOENT;
3161                         ret = -rte_errno;
3162                         goto exit;
3163                 }
3164         } else {
3165                 /*
3166                  * The existence of several matching entries (nd > 1) means
3167                  * port representors have been instantiated. No existing Verbs
3168                  * call nor sysfs entries can tell them apart, this can only
3169                  * be done through Netlink calls assuming kernel drivers are
3170                  * recent enough to support them.
3171                  *
3172                  * In the event of identification failure through Netlink,
3173                  * try again through sysfs, then:
3174                  *
3175                  * 1. A single IB device matches (nd == 1) with single
3176                  *    port (np=0/1) and is not a representor, assume
3177                  *    no switch support.
3178                  *
3179                  * 2. Otherwise no safe assumptions can be made;
3180                  *    complain louder and bail out.
3181                  */
3182                 np = 1;
3183                 for (i = 0; i != nd; ++i) {
3184                         memset(&list[ns].info, 0, sizeof(list[ns].info));
3185                         list[ns].max_port = 1;
3186                         list[ns].ibv_port = 1;
3187                         list[ns].ibv_dev = ibv_match[i];
3188                         list[ns].eth_dev = NULL;
3189                         list[ns].pci_dev = pci_dev;
3190                         list[ns].pf_bond = -1;
3191                         list[ns].ifindex = 0;
3192                         if (nl_rdma >= 0)
3193                                 list[ns].ifindex = mlx5_nl_ifindex
3194                                         (nl_rdma, list[ns].ibv_dev->name, 1);
3195                         if (!list[ns].ifindex) {
3196                                 char ifname[IF_NAMESIZE];
3197
3198                                 /*
3199                                  * Netlink failed, it may happen with old
3200                                  * ib_core kernel driver (before 4.16).
3201                                  * We can assume there is old driver because
3202                                  * here we are processing single ports IB
3203                                  * devices. Let's try sysfs to retrieve
3204                                  * the ifindex. The method works for
3205                                  * master device only.
3206                                  */
3207                                 if (nd > 1) {
3208                                         /*
3209                                          * Multiple devices found, assume
3210                                          * representors, can not distinguish
3211                                          * master/representor and retrieve
3212                                          * ifindex via sysfs.
3213                                          */
3214                                         continue;
3215                                 }
3216                                 ret = mlx5_get_master_ifname
3217                                         (ibv_match[i]->ibdev_path, &ifname);
3218                                 if (!ret)
3219                                         list[ns].ifindex =
3220                                                 if_nametoindex(ifname);
3221                                 if (!list[ns].ifindex) {
3222                                         /*
3223                                          * No network interface index found
3224                                          * for the specified device, it means
3225                                          * there it is neither representor
3226                                          * nor master.
3227                                          */
3228                                         continue;
3229                                 }
3230                         }
3231                         ret = -1;
3232                         if (nl_route >= 0)
3233                                 ret = mlx5_nl_switch_info
3234                                                (nl_route,
3235                                                 list[ns].ifindex,
3236                                                 &list[ns].info);
3237                         if (ret || (!list[ns].info.representor &&
3238                                     !list[ns].info.master)) {
3239                                 /*
3240                                  * We failed to recognize representors with
3241                                  * Netlink, let's try to perform the task
3242                                  * with sysfs.
3243                                  */
3244                                 ret =  mlx5_sysfs_switch_info
3245                                                 (list[ns].ifindex,
3246                                                  &list[ns].info);
3247                         }
3248                         if (!ret && (list[ns].info.representor ^
3249                                      list[ns].info.master)) {
3250                                 ns++;
3251                         } else if ((nd == 1) &&
3252                                    !list[ns].info.representor &&
3253                                    !list[ns].info.master) {
3254                                 /*
3255                                  * Single IB device with
3256                                  * one physical port and
3257                                  * attached network device.
3258                                  * May be SRIOV is not enabled
3259                                  * or there is no representors.
3260                                  */
3261                                 DRV_LOG(INFO, "no E-Switch support detected");
3262                                 ns++;
3263                                 break;
3264                         }
3265                 }
3266                 if (!ns) {
3267                         DRV_LOG(ERR,
3268                                 "unable to recognize master/representors"
3269                                 " on the multiple IB devices");
3270                         rte_errno = ENOENT;
3271                         ret = -rte_errno;
3272                         goto exit;
3273                 }
3274         }
3275         assert(ns);
3276         /*
3277          * Sort list to probe devices in natural order for users convenience
3278          * (i.e. master first, then representors from lowest to highest ID).
3279          */
3280         qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp);
3281         /* Default configuration. */
3282         dev_config = (struct mlx5_dev_config){
3283                 .hw_padding = 0,
3284                 .mps = MLX5_ARG_UNSET,
3285                 .dbnc = MLX5_ARG_UNSET,
3286                 .rx_vec_en = 1,
3287                 .txq_inline_max = MLX5_ARG_UNSET,
3288                 .txq_inline_min = MLX5_ARG_UNSET,
3289                 .txq_inline_mpw = MLX5_ARG_UNSET,
3290                 .txqs_inline = MLX5_ARG_UNSET,
3291                 .vf_nl_en = 1,
3292                 .mr_ext_memseg_en = 1,
3293                 .mprq = {
3294                         .enabled = 0, /* Disabled by default. */
3295                         .stride_num_n = MLX5_MPRQ_STRIDE_NUM_N,
3296                         .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN,
3297                         .min_rxqs_num = MLX5_MPRQ_MIN_RXQS,
3298                 },
3299                 .dv_esw_en = 1,
3300                 .dv_flow_en = 1,
3301         };
3302         /* Device specific configuration. */
3303         switch (pci_dev->id.device_id) {
3304         case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
3305         case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
3306         case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
3307         case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
3308         case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF:
3309         case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF:
3310         case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF:
3311                 dev_config.vf = 1;
3312                 break;
3313         default:
3314                 break;
3315         }
3316         for (i = 0; i != ns; ++i) {
3317                 uint32_t restore;
3318
3319                 list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device,
3320                                                  &list[i],
3321                                                  dev_config);
3322                 if (!list[i].eth_dev) {
3323                         if (rte_errno != EBUSY && rte_errno != EEXIST)
3324                                 break;
3325                         /* Device is disabled or already spawned. Ignore it. */
3326                         continue;
3327                 }
3328                 restore = list[i].eth_dev->data->dev_flags;
3329                 rte_eth_copy_pci_info(list[i].eth_dev, pci_dev);
3330                 /* Restore non-PCI flags cleared by the above call. */
3331                 list[i].eth_dev->data->dev_flags |= restore;
3332                 mlx5_dev_interrupt_handler_devx_install(list[i].eth_dev);
3333                 rte_eth_dev_probing_finish(list[i].eth_dev);
3334         }
3335         if (i != ns) {
3336                 DRV_LOG(ERR,
3337                         "probe of PCI device " PCI_PRI_FMT " aborted after"
3338                         " encountering an error: %s",
3339                         pci_dev->addr.domain, pci_dev->addr.bus,
3340                         pci_dev->addr.devid, pci_dev->addr.function,
3341                         strerror(rte_errno));
3342                 ret = -rte_errno;
3343                 /* Roll back. */
3344                 while (i--) {
3345                         if (!list[i].eth_dev)
3346                                 continue;
3347                         mlx5_dev_close(list[i].eth_dev);
3348                         /* mac_addrs must not be freed because in dev_private */
3349                         list[i].eth_dev->data->mac_addrs = NULL;
3350                         claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
3351                 }
3352                 /* Restore original error. */
3353                 rte_errno = -ret;
3354         } else {
3355                 ret = 0;
3356         }
3357 exit:
3358         /*
3359          * Do the routine cleanup:
3360          * - close opened Netlink sockets
3361          * - free allocated spawn data array
3362          * - free the Infiniband device list
3363          */
3364         if (nl_rdma >= 0)
3365                 close(nl_rdma);
3366         if (nl_route >= 0)
3367                 close(nl_route);
3368         if (list)
3369                 rte_free(list);
3370         assert(ibv_list);
3371         mlx5_glue->free_device_list(ibv_list);
3372         return ret;
3373 }
3374
3375 /**
3376  * Look for the ethernet device belonging to mlx5 driver.
3377  *
3378  * @param[in] port_id
3379  *   port_id to start looking for device.
3380  * @param[in] pci_dev
3381  *   Pointer to the hint PCI device. When device is being probed
3382  *   the its siblings (master and preceding representors might
3383  *   not have assigned driver yet (because the mlx5_pci_probe()
3384  *   is not completed yet, for this case match on hint PCI
3385  *   device may be used to detect sibling device.
3386  *
3387  * @return
3388  *   port_id of found device, RTE_MAX_ETHPORT if not found.
3389  */
3390 uint16_t
3391 mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev)
3392 {
3393         while (port_id < RTE_MAX_ETHPORTS) {
3394                 struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3395
3396                 if (dev->state != RTE_ETH_DEV_UNUSED &&
3397                     dev->device &&
3398                     (dev->device == &pci_dev->device ||
3399                      (dev->device->driver &&
3400                      dev->device->driver->name &&
3401                      !strcmp(dev->device->driver->name, MLX5_DRIVER_NAME))))
3402                         break;
3403                 port_id++;
3404         }
3405         if (port_id >= RTE_MAX_ETHPORTS)
3406                 return RTE_MAX_ETHPORTS;
3407         return port_id;
3408 }
3409
3410 /**
3411  * DPDK callback to remove a PCI device.
3412  *
3413  * This function removes all Ethernet devices belong to a given PCI device.
3414  *
3415  * @param[in] pci_dev
3416  *   Pointer to the PCI device.
3417  *
3418  * @return
3419  *   0 on success, the function cannot fail.
3420  */
3421 static int
3422 mlx5_pci_remove(struct rte_pci_device *pci_dev)
3423 {
3424         uint16_t port_id;
3425
3426         RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
3427                 rte_eth_dev_close(port_id);
3428         return 0;
3429 }
3430
3431 static const struct rte_pci_id mlx5_pci_id_map[] = {
3432         {
3433                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3434                                PCI_DEVICE_ID_MELLANOX_CONNECTX4)
3435         },
3436         {
3437                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3438                                PCI_DEVICE_ID_MELLANOX_CONNECTX4VF)
3439         },
3440         {
3441                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3442                                PCI_DEVICE_ID_MELLANOX_CONNECTX4LX)
3443         },
3444         {
3445                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3446                                PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF)
3447         },
3448         {
3449                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3450                                PCI_DEVICE_ID_MELLANOX_CONNECTX5)
3451         },
3452         {
3453                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3454                                PCI_DEVICE_ID_MELLANOX_CONNECTX5VF)
3455         },
3456         {
3457                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3458                                PCI_DEVICE_ID_MELLANOX_CONNECTX5EX)
3459         },
3460         {
3461                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3462                                PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF)
3463         },
3464         {
3465                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3466                                PCI_DEVICE_ID_MELLANOX_CONNECTX5BF)
3467         },
3468         {
3469                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3470                                PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF)
3471         },
3472         {
3473                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3474                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
3475         },
3476         {
3477                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3478                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
3479         },
3480         {
3481                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3482                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
3483         },
3484         {
3485                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3486                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
3487         },
3488         {
3489                 .vendor_id = 0
3490         }
3491 };
3492
3493 static struct rte_pci_driver mlx5_driver = {
3494         .driver = {
3495                 .name = MLX5_DRIVER_NAME
3496         },
3497         .id_table = mlx5_pci_id_map,
3498         .probe = mlx5_pci_probe,
3499         .remove = mlx5_pci_remove,
3500         .dma_map = mlx5_dma_map,
3501         .dma_unmap = mlx5_dma_unmap,
3502         .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
3503                      RTE_PCI_DRV_PROBE_AGAIN,
3504 };
3505
3506 #ifdef RTE_IBVERBS_LINK_DLOPEN
3507
3508 /**
3509  * Suffix RTE_EAL_PMD_PATH with "-glue".
3510  *
3511  * This function performs a sanity check on RTE_EAL_PMD_PATH before
3512  * suffixing its last component.
3513  *
3514  * @param buf[out]
3515  *   Output buffer, should be large enough otherwise NULL is returned.
3516  * @param size
3517  *   Size of @p out.
3518  *
3519  * @return
3520  *   Pointer to @p buf or @p NULL in case suffix cannot be appended.
3521  */
3522 static char *
3523 mlx5_glue_path(char *buf, size_t size)
3524 {
3525         static const char *const bad[] = { "/", ".", "..", NULL };
3526         const char *path = RTE_EAL_PMD_PATH;
3527         size_t len = strlen(path);
3528         size_t off;
3529         int i;
3530
3531         while (len && path[len - 1] == '/')
3532                 --len;
3533         for (off = len; off && path[off - 1] != '/'; --off)
3534                 ;
3535         for (i = 0; bad[i]; ++i)
3536                 if (!strncmp(path + off, bad[i], (int)(len - off)))
3537                         goto error;
3538         i = snprintf(buf, size, "%.*s-glue", (int)len, path);
3539         if (i == -1 || (size_t)i >= size)
3540                 goto error;
3541         return buf;
3542 error:
3543         DRV_LOG(ERR,
3544                 "unable to append \"-glue\" to last component of"
3545                 " RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\"),"
3546                 " please re-configure DPDK");
3547         return NULL;
3548 }
3549
3550 /**
3551  * Initialization routine for run-time dependency on rdma-core.
3552  */
3553 static int
3554 mlx5_glue_init(void)
3555 {
3556         char glue_path[sizeof(RTE_EAL_PMD_PATH) - 1 + sizeof("-glue")];
3557         const char *path[] = {
3558                 /*
3559                  * A basic security check is necessary before trusting
3560                  * MLX5_GLUE_PATH, which may override RTE_EAL_PMD_PATH.
3561                  */
3562                 (geteuid() == getuid() && getegid() == getgid() ?
3563                  getenv("MLX5_GLUE_PATH") : NULL),
3564                 /*
3565                  * When RTE_EAL_PMD_PATH is set, use its glue-suffixed
3566                  * variant, otherwise let dlopen() look up libraries on its
3567                  * own.
3568                  */
3569                 (*RTE_EAL_PMD_PATH ?
3570                  mlx5_glue_path(glue_path, sizeof(glue_path)) : ""),
3571         };
3572         unsigned int i = 0;
3573         void *handle = NULL;
3574         void **sym;
3575         const char *dlmsg;
3576
3577         while (!handle && i != RTE_DIM(path)) {
3578                 const char *end;
3579                 size_t len;
3580                 int ret;
3581
3582                 if (!path[i]) {
3583                         ++i;
3584                         continue;
3585                 }
3586                 end = strpbrk(path[i], ":;");
3587                 if (!end)
3588                         end = path[i] + strlen(path[i]);
3589                 len = end - path[i];
3590                 ret = 0;
3591                 do {
3592                         char name[ret + 1];
3593
3594                         ret = snprintf(name, sizeof(name), "%.*s%s" MLX5_GLUE,
3595                                        (int)len, path[i],
3596                                        (!len || *(end - 1) == '/') ? "" : "/");
3597                         if (ret == -1)
3598                                 break;
3599                         if (sizeof(name) != (size_t)ret + 1)
3600                                 continue;
3601                         DRV_LOG(DEBUG, "looking for rdma-core glue as \"%s\"",
3602                                 name);
3603                         handle = dlopen(name, RTLD_LAZY);
3604                         break;
3605                 } while (1);
3606                 path[i] = end + 1;
3607                 if (!*end)
3608                         ++i;
3609         }
3610         if (!handle) {
3611                 rte_errno = EINVAL;
3612                 dlmsg = dlerror();
3613                 if (dlmsg)
3614                         DRV_LOG(WARNING, "cannot load glue library: %s", dlmsg);
3615                 goto glue_error;
3616         }
3617         sym = dlsym(handle, "mlx5_glue");
3618         if (!sym || !*sym) {
3619                 rte_errno = EINVAL;
3620                 dlmsg = dlerror();
3621                 if (dlmsg)
3622                         DRV_LOG(ERR, "cannot resolve glue symbol: %s", dlmsg);
3623                 goto glue_error;
3624         }
3625         mlx5_glue = *sym;
3626         return 0;
3627 glue_error:
3628         if (handle)
3629                 dlclose(handle);
3630         DRV_LOG(WARNING,
3631                 "cannot initialize PMD due to missing run-time dependency on"
3632                 " rdma-core libraries (libibverbs, libmlx5)");
3633         return -rte_errno;
3634 }
3635
3636 #endif
3637
3638 /**
3639  * Driver initialization routine.
3640  */
3641 RTE_INIT(rte_mlx5_pmd_init)
3642 {
3643         /* Initialize driver log type. */
3644         mlx5_logtype = rte_log_register("pmd.net.mlx5");
3645         if (mlx5_logtype >= 0)
3646                 rte_log_set_level(mlx5_logtype, RTE_LOG_NOTICE);
3647
3648         /* Build the static tables for Verbs conversion. */
3649         mlx5_set_ptype_table();
3650         mlx5_set_cksum_table();
3651         mlx5_set_swp_types_table();
3652         /*
3653          * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
3654          * huge pages. Calling ibv_fork_init() during init allows
3655          * applications to use fork() safely for purposes other than
3656          * using this PMD, which is not supported in forked processes.
3657          */
3658         setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
3659         /* Match the size of Rx completion entry to the size of a cacheline. */
3660         if (RTE_CACHE_LINE_SIZE == 128)
3661                 setenv("MLX5_CQE_SIZE", "128", 0);
3662         /*
3663          * MLX5_DEVICE_FATAL_CLEANUP tells ibv_destroy functions to
3664          * cleanup all the Verbs resources even when the device was removed.
3665          */
3666         setenv("MLX5_DEVICE_FATAL_CLEANUP", "1", 1);
3667 #ifdef RTE_IBVERBS_LINK_DLOPEN
3668         if (mlx5_glue_init())
3669                 return;
3670         assert(mlx5_glue);
3671 #endif
3672 #ifndef NDEBUG
3673         /* Glue structure must not contain any NULL pointers. */
3674         {
3675                 unsigned int i;
3676
3677                 for (i = 0; i != sizeof(*mlx5_glue) / sizeof(void *); ++i)
3678                         assert(((const void *const *)mlx5_glue)[i]);
3679         }
3680 #endif
3681         if (strcmp(mlx5_glue->version, MLX5_GLUE_VERSION)) {
3682                 DRV_LOG(ERR,
3683                         "rdma-core glue \"%s\" mismatch: \"%s\" is required",
3684                         mlx5_glue->version, MLX5_GLUE_VERSION);
3685                 return;
3686         }
3687         mlx5_glue->fork_init();
3688         rte_pci_register(&mlx5_driver);
3689 }
3690
3691 RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
3692 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map);
3693 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib");