drivers/net/mlx5/mlx5.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <unistd.h>
   8 #include <string.h>
   9 #include <assert.h>
  10 #include <dlfcn.h>
  11 #include <stdint.h>
  12 #include <stdlib.h>
  13 #include <errno.h>
  14 #include <net/if.h>
  15 #include <sys/mman.h>
  16 #include <linux/rtnetlink.h>
  17
  18 /* Verbs header. */
  19 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  20 #ifdef PEDANTIC
  21 #pragma GCC diagnostic ignored "-Wpedantic"
  22 #endif
  23 #include <infiniband/verbs.h>
  24 #ifdef PEDANTIC
  25 #pragma GCC diagnostic error "-Wpedantic"
  26 #endif
  27
  28 #include <rte_malloc.h>
  29 #include <rte_ethdev_driver.h>
  30 #include <rte_ethdev_pci.h>
  31 #include <rte_pci.h>
  32 #include <rte_bus_pci.h>
  33 #include <rte_common.h>
  34 #include <rte_config.h>
  35 #include <rte_kvargs.h>
  36 #include <rte_rwlock.h>
  37 #include <rte_spinlock.h>
  38 #include <rte_string_fns.h>
  39 #include <rte_alarm.h>
  40
  41 #include "mlx5.h"
  42 #include "mlx5_utils.h"
  43 #include "mlx5_rxtx.h"
  44 #include "mlx5_autoconf.h"
  45 #include "mlx5_defs.h"
  46 #include "mlx5_glue.h"
  47 #include "mlx5_mr.h"
  48 #include "mlx5_flow.h"
  49
  50 /* Device parameter to enable RX completion queue compression. */
  51 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
  52
  53 /* Device parameter to enable RX completion entry padding to 128B. */
  54 #define MLX5_RXQ_CQE_PAD_EN "rxq_cqe_pad_en"
  55
  56 /* Device parameter to enable padding Rx packet to cacheline size. */
  57 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en"
  58
  59 /* Device parameter to enable Multi-Packet Rx queue. */
  60 #define MLX5_RX_MPRQ_EN "mprq_en"
  61
  62 /* Device parameter to configure log 2 of the number of strides for MPRQ. */
  63 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num"
  64
  65 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */
  66 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len"
  67
  68 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */
  69 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq"
  70
  71 /* Device parameter to configure inline send. Deprecated, ignored.*/
  72 #define MLX5_TXQ_INLINE "txq_inline"
  73
  74 /* Device parameter to limit packet size to inline with ordinary SEND. */
  75 #define MLX5_TXQ_INLINE_MAX "txq_inline_max"
  76
  77 /* Device parameter to configure minimal data size to inline. */
  78 #define MLX5_TXQ_INLINE_MIN "txq_inline_min"
  79
  80 /* Device parameter to limit packet size to inline with Enhanced MPW. */
  81 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw"
  82
  83 /*
  84  * Device parameter to configure the number of TX queues threshold for
  85  * enabling inline send.
  86  */
  87 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline"
  88
  89 /*
  90  * Device parameter to configure the number of TX queues threshold for
  91  * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines).
  92  */
  93 #define MLX5_TXQS_MAX_VEC "txqs_max_vec"
  94
  95 /* Device parameter to enable multi-packet send WQEs. */
  96 #define MLX5_TXQ_MPW_EN "txq_mpw_en"
  97
  98 /*
  99  * Device parameter to include 2 dsegs in the title WQEBB.
 100  * Deprecated, ignored.
 101  */
 102 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en"
 103
 104 /*
 105  * Device parameter to limit the size of inlining packet.
 106  * Deprecated, ignored.
 107  */
 108 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len"
 109
 110 /*
 111  * Device parameter to enable hardware Tx vector.
 112  * Deprecated, ignored (no vectorized Tx routines anymore).
 113  */
 114 #define MLX5_TX_VEC_EN "tx_vec_en"
 115
 116 /* Device parameter to enable hardware Rx vector. */
 117 #define MLX5_RX_VEC_EN "rx_vec_en"
 118
 119 /* Allow L3 VXLAN flow creation. */
 120 #define MLX5_L3_VXLAN_EN "l3_vxlan_en"
 121
 122 /* Activate DV E-Switch flow steering. */
 123 #define MLX5_DV_ESW_EN "dv_esw_en"
 124
 125 /* Activate DV flow steering. */
 126 #define MLX5_DV_FLOW_EN "dv_flow_en"
 127
 128 /* Activate Netlink support in VF mode. */
 129 #define MLX5_VF_NL_EN "vf_nl_en"
 130
 131 /* Enable extending memsegs when creating a MR. */
 132 #define MLX5_MR_EXT_MEMSEG_EN "mr_ext_memseg_en"
 133
 134 /* Select port representors to instantiate. */
 135 #define MLX5_REPRESENTOR "representor"
 136
 137 /* Device parameter to configure the maximum number of dump files per queue. */
 138 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num"
 139
 140 /* Configure timeout of LRO session (in microseconds). */
 141 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec"
 142
 143 #ifndef HAVE_IBV_MLX5_MOD_MPW
 144 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 145 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
 146 #endif
 147
 148 #ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP
 149 #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
 150 #endif
 151
 152 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 153
 154 /* Shared memory between primary and secondary processes. */
 155 struct mlx5_shared_data *mlx5_shared_data;
 156
 157 /* Spinlock for mlx5_shared_data allocation. */
 158 static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 159
 160 /* Process local data for secondary processes. */
 161 static struct mlx5_local_data mlx5_local_data;
 162
 163 /** Driver-specific log messages type. */
 164 int mlx5_logtype;
 165
 166 /** Data associated with devices to spawn. */
 167 struct mlx5_dev_spawn_data {
 168         uint32_t ifindex; /**< Network interface index. */
 169         uint32_t max_port; /**< IB device maximal port index. */
 170         uint32_t ibv_port; /**< IB device physical port index. */
 171         int pf_bond; /**< bonding device PF index. < 0 - no bonding */
 172         struct mlx5_switch_info info; /**< Switch information. */
 173         struct ibv_device *ibv_dev; /**< Associated IB device. */
 174         struct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */
 175         struct rte_pci_device *pci_dev; /**< Backend PCI device. */
 176 };
 177
 178 static LIST_HEAD(, mlx5_ibv_shared) mlx5_ibv_list = LIST_HEAD_INITIALIZER();
 179 static pthread_mutex_t mlx5_ibv_list_mutex = PTHREAD_MUTEX_INITIALIZER;
 180
 181 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 182 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 183
 184 /**
 185  * Allocate ID pool structure.
 186  *
 187  * @return
 188  *   Pointer to pool object, NULL value otherwise.
 189  */
 190 struct mlx5_flow_id_pool *
 191 mlx5_flow_id_pool_alloc(void)
 192 {
 193         struct mlx5_flow_id_pool *pool;
 194         void *mem;
 195
 196         pool = rte_zmalloc("id pool allocation", sizeof(*pool),
 197                            RTE_CACHE_LINE_SIZE);
 198         if (!pool) {
 199                 DRV_LOG(ERR, "can't allocate id pool");
 200                 rte_errno  = ENOMEM;
 201                 return NULL;
 202         }
 203         mem = rte_zmalloc("", MLX5_FLOW_MIN_ID_POOL_SIZE * sizeof(uint32_t),
 204                           RTE_CACHE_LINE_SIZE);
 205         if (!mem) {
 206                 DRV_LOG(ERR, "can't allocate mem for id pool");
 207                 rte_errno  = ENOMEM;
 208                 goto error;
 209         }
 210         pool->free_arr = mem;
 211         pool->curr = pool->free_arr;
 212         pool->last = pool->free_arr + MLX5_FLOW_MIN_ID_POOL_SIZE;
 213         pool->base_index = 0;
 214         return pool;
 215 error:
 216         rte_free(pool);
 217         return NULL;
 218 }
 219
 220 /**
 221  * Release ID pool structure.
 222  *
 223  * @param[in] pool
 224  *   Pointer to flow id pool object to free.
 225  */
 226 void
 227 mlx5_flow_id_pool_release(struct mlx5_flow_id_pool *pool)
 228 {
 229         rte_free(pool->free_arr);
 230         rte_free(pool);
 231 }
 232
 233 /**
 234  * Generate ID.
 235  *
 236  * @param[in] pool
 237  *   Pointer to flow id pool.
 238  * @param[out] id
 239  *   The generated ID.
 240  *
 241  * @return
 242  *   0 on success, error value otherwise.
 243  */
 244 uint32_t
 245 mlx5_flow_id_get(struct mlx5_flow_id_pool *pool, uint32_t *id)
 246 {
 247         if (pool->curr == pool->free_arr) {
 248                 if (pool->base_index == UINT32_MAX) {
 249                         rte_errno  = ENOMEM;
 250                         DRV_LOG(ERR, "no free id");
 251                         return -rte_errno;
 252                 }
 253                 *id = ++pool->base_index;
 254                 return 0;
 255         }
 256         *id = *(--pool->curr);
 257         return 0;
 258 }
 259
 260 /**
 261  * Release ID.
 262  *
 263  * @param[in] pool
 264  *   Pointer to flow id pool.
 265  * @param[out] id
 266  *   The generated ID.
 267  *
 268  * @return
 269  *   0 on success, error value otherwise.
 270  */
 271 uint32_t
 272 mlx5_flow_id_release(struct mlx5_flow_id_pool *pool, uint32_t id)
 273 {
 274         uint32_t size;
 275         uint32_t size2;
 276         void *mem;
 277
 278         if (pool->curr == pool->last) {
 279                 size = pool->curr - pool->free_arr;
 280                 size2 = size * MLX5_ID_GENERATION_ARRAY_FACTOR;
 281                 assert(size2 > size);
 282                 mem = rte_malloc("", size2 * sizeof(uint32_t), 0);
 283                 if (!mem) {
 284                         DRV_LOG(ERR, "can't allocate mem for id pool");
 285                         rte_errno  = ENOMEM;
 286                         return -rte_errno;
 287                 }
 288                 memcpy(mem, pool->free_arr, size * sizeof(uint32_t));
 289                 rte_free(pool->free_arr);
 290                 pool->free_arr = mem;
 291                 pool->curr = pool->free_arr + size;
 292                 pool->last = pool->free_arr + size2;
 293         }
 294         *pool->curr = id;
 295         pool->curr++;
 296         return 0;
 297 }
 298
 299 /**
 300  * Initialize the counters management structure.
 301  *
 302  * @param[in] sh
 303  *   Pointer to mlx5_ibv_shared object to free
 304  */
 305 static void
 306 mlx5_flow_counters_mng_init(struct mlx5_ibv_shared *sh)
 307 {
 308         uint8_t i;
 309
 310         TAILQ_INIT(&sh->cmng.flow_counters);
 311         for (i = 0; i < RTE_DIM(sh->cmng.ccont); ++i)
 312                 TAILQ_INIT(&sh->cmng.ccont[i].pool_list);
 313 }
 314
 315 /**
 316  * Destroy all the resources allocated for a counter memory management.
 317  *
 318  * @param[in] mng
 319  *   Pointer to the memory management structure.
 320  */
 321 static void
 322 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng)
 323 {
 324         uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data;
 325
 326         LIST_REMOVE(mng, next);
 327         claim_zero(mlx5_devx_cmd_destroy(mng->dm));
 328         claim_zero(mlx5_glue->devx_umem_dereg(mng->umem));
 329         rte_free(mem);
 330 }
 331
 332 /**
 333  * Close and release all the resources of the counters management.
 334  *
 335  * @param[in] sh
 336  *   Pointer to mlx5_ibv_shared object to free.
 337  */
 338 static void
 339 mlx5_flow_counters_mng_close(struct mlx5_ibv_shared *sh)
 340 {
 341         struct mlx5_counter_stats_mem_mng *mng;
 342         uint8_t i;
 343         int j;
 344         int retries = 1024;
 345
 346         rte_errno = 0;
 347         while (--retries) {
 348                 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh);
 349                 if (rte_errno != EINPROGRESS)
 350                         break;
 351                 rte_pause();
 352         }
 353         for (i = 0; i < RTE_DIM(sh->cmng.ccont); ++i) {
 354                 struct mlx5_flow_counter_pool *pool;
 355                 uint32_t batch = !!(i % 2);
 356
 357                 if (!sh->cmng.ccont[i].pools)
 358                         continue;
 359                 pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
 360                 while (pool) {
 361                         if (batch) {
 362                                 if (pool->min_dcs)
 363                                         claim_zero
 364                                         (mlx5_devx_cmd_destroy(pool->min_dcs));
 365                         }
 366                         for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) {
 367                                 if (pool->counters_raw[j].action)
 368                                         claim_zero
 369                                         (mlx5_glue->destroy_flow_action
 370                                                (pool->counters_raw[j].action));
 371                                 if (!batch && pool->counters_raw[j].dcs)
 372                                         claim_zero(mlx5_devx_cmd_destroy
 373                                                   (pool->counters_raw[j].dcs));
 374                         }
 375                         TAILQ_REMOVE(&sh->cmng.ccont[i].pool_list, pool,
 376                                      next);
 377                         rte_free(pool);
 378                         pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
 379                 }
 380                 rte_free(sh->cmng.ccont[i].pools);
 381         }
 382         mng = LIST_FIRST(&sh->cmng.mem_mngs);
 383         while (mng) {
 384                 mlx5_flow_destroy_counter_stat_mem_mng(mng);
 385                 mng = LIST_FIRST(&sh->cmng.mem_mngs);
 386         }
 387         memset(&sh->cmng, 0, sizeof(sh->cmng));
 388 }
 389
 390 /**
 391  * Extract pdn of PD object using DV API.
 392  *
 393  * @param[in] pd
 394  *   Pointer to the verbs PD object.
 395  * @param[out] pdn
 396  *   Pointer to the PD object number variable.
 397  *
 398  * @return
 399  *   0 on success, error value otherwise.
 400  */
 401 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 402 static int
 403 mlx5_get_pdn(struct ibv_pd *pd __rte_unused, uint32_t *pdn __rte_unused)
 404 {
 405         struct mlx5dv_obj obj;
 406         struct mlx5dv_pd pd_info;
 407         int ret = 0;
 408
 409         obj.pd.in = pd;
 410         obj.pd.out = &pd_info;
 411         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 412         if (ret) {
 413                 DRV_LOG(DEBUG, "Fail to get PD object info");
 414                 return ret;
 415         }
 416         *pdn = pd_info.pdn;
 417         return 0;
 418 }
 419 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 420
 421 /**
 422  * Allocate shared IB device context. If there is multiport device the
 423  * master and representors will share this context, if there is single
 424  * port dedicated IB device, the context will be used by only given
 425  * port due to unification.
 426  *
 427  * Routine first searches the context for the specified IB device name,
 428  * if found the shared context assumed and reference counter is incremented.
 429  * If no context found the new one is created and initialized with specified
 430  * IB device context and parameters.
 431  *
 432  * @param[in] spawn
 433  *   Pointer to the IB device attributes (name, port, etc).
 434  *
 435  * @return
 436  *   Pointer to mlx5_ibv_shared object on success,
 437  *   otherwise NULL and rte_errno is set.
 438  */
 439 static struct mlx5_ibv_shared *
 440 mlx5_alloc_shared_ibctx(const struct mlx5_dev_spawn_data *spawn)
 441 {
 442         struct mlx5_ibv_shared *sh;
 443         int err = 0;
 444         uint32_t i;
 445 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 446         struct mlx5_devx_tis_attr tis_attr = { 0 };
 447 #endif
 448
 449 assert(spawn);
 450         /* Secondary process should not create the shared context. */
 451         assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
 452         pthread_mutex_lock(&mlx5_ibv_list_mutex);
 453         /* Search for IB context by device name. */
 454         LIST_FOREACH(sh, &mlx5_ibv_list, next) {
 455                 if (!strcmp(sh->ibdev_name, spawn->ibv_dev->name)) {
 456                         sh->refcnt++;
 457                         goto exit;
 458                 }
 459         }
 460         /* No device found, we have to create new shared context. */
 461         assert(spawn->max_port);
 462         sh = rte_zmalloc("ethdev shared ib context",
 463                          sizeof(struct mlx5_ibv_shared) +
 464                          spawn->max_port *
 465                          sizeof(struct mlx5_ibv_shared_port),
 466                          RTE_CACHE_LINE_SIZE);
 467         if (!sh) {
 468                 DRV_LOG(ERR, "shared context allocation failure");
 469                 rte_errno  = ENOMEM;
 470                 goto exit;
 471         }
 472         /* Try to open IB device with DV first, then usual Verbs. */
 473         errno = 0;
 474         sh->ctx = mlx5_glue->dv_open_device(spawn->ibv_dev);
 475         if (sh->ctx) {
 476                 sh->devx = 1;
 477                 DRV_LOG(DEBUG, "DevX is supported");
 478         } else {
 479                 sh->ctx = mlx5_glue->open_device(spawn->ibv_dev);
 480                 if (!sh->ctx) {
 481                         err = errno ? errno : ENODEV;
 482                         goto error;
 483                 }
 484                 DRV_LOG(DEBUG, "DevX is NOT supported");
 485         }
 486         err = mlx5_glue->query_device_ex(sh->ctx, NULL, &sh->device_attr);
 487         if (err) {
 488                 DRV_LOG(DEBUG, "ibv_query_device_ex() failed");
 489                 goto error;
 490         }
 491         sh->refcnt = 1;
 492         sh->max_port = spawn->max_port;
 493         strncpy(sh->ibdev_name, sh->ctx->device->name,
 494                 sizeof(sh->ibdev_name));
 495         strncpy(sh->ibdev_path, sh->ctx->device->ibdev_path,
 496                 sizeof(sh->ibdev_path));
 497         pthread_mutex_init(&sh->intr_mutex, NULL);
 498         /*
 499          * Setting port_id to max unallowed value means
 500          * there is no interrupt subhandler installed for
 501          * the given port index i.
 502          */
 503         for (i = 0; i < sh->max_port; i++) {
 504                 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS;
 505                 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS;
 506         }
 507         sh->pd = mlx5_glue->alloc_pd(sh->ctx);
 508         if (sh->pd == NULL) {
 509                 DRV_LOG(ERR, "PD allocation failure");
 510                 err = ENOMEM;
 511                 goto error;
 512         }
 513 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 514         if (sh->devx) {
 515                 err = mlx5_get_pdn(sh->pd, &sh->pdn);
 516                 if (err) {
 517                         DRV_LOG(ERR, "Fail to extract pdn from PD");
 518                         goto error;
 519                 }
 520                 sh->td = mlx5_devx_cmd_create_td(sh->ctx);
 521                 if (!sh->td) {
 522                         DRV_LOG(ERR, "TD allocation failure");
 523                         err = ENOMEM;
 524                         goto error;
 525                 }
 526                 tis_attr.transport_domain = sh->td->id;
 527                 sh->tis = mlx5_devx_cmd_create_tis(sh->ctx, &tis_attr);
 528                 if (!sh->tis) {
 529                         DRV_LOG(ERR, "TIS allocation failure");
 530                         err = ENOMEM;
 531                         goto error;
 532                 }
 533         }
 534         sh->flow_id_pool = mlx5_flow_id_pool_alloc();
 535         if (!sh->flow_id_pool) {
 536                 DRV_LOG(ERR, "can't create flow id pool");
 537                 err = ENOMEM;
 538                 goto error;
 539         }
 540 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 541         /*
 542          * Once the device is added to the list of memory event
 543          * callback, its global MR cache table cannot be expanded
 544          * on the fly because of deadlock. If it overflows, lookup
 545          * should be done by searching MR list linearly, which is slow.
 546          *
 547          * At this point the device is not added to the memory
 548          * event list yet, context is just being created.
 549          */
 550         err = mlx5_mr_btree_init(&sh->mr.cache,
 551                                  MLX5_MR_BTREE_CACHE_N * 2,
 552                                  spawn->pci_dev->device.numa_node);
 553         if (err) {
 554                 err = rte_errno;
 555                 goto error;
 556         }
 557         mlx5_flow_counters_mng_init(sh);
 558         /* Add device to memory callback list. */
 559         rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
 560         LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
 561                          sh, mem_event_cb);
 562         rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
 563         /* Add context to the global device list. */
 564         LIST_INSERT_HEAD(&mlx5_ibv_list, sh, next);
 565 exit:
 566         pthread_mutex_unlock(&mlx5_ibv_list_mutex);
 567         return sh;
 568 error:
 569         pthread_mutex_unlock(&mlx5_ibv_list_mutex);
 570         assert(sh);
 571         if (sh->tis)
 572                 claim_zero(mlx5_devx_cmd_destroy(sh->tis));
 573         if (sh->td)
 574                 claim_zero(mlx5_devx_cmd_destroy(sh->td));
 575         if (sh->pd)
 576                 claim_zero(mlx5_glue->dealloc_pd(sh->pd));
 577         if (sh->ctx)
 578                 claim_zero(mlx5_glue->close_device(sh->ctx));
 579         if (sh->flow_id_pool)
 580                 mlx5_flow_id_pool_release(sh->flow_id_pool);
 581         rte_free(sh);
 582         assert(err > 0);
 583         rte_errno = err;
 584         return NULL;
 585 }
 586
 587 /**
 588  * Free shared IB device context. Decrement counter and if zero free
 589  * all allocated resources and close handles.
 590  *
 591  * @param[in] sh
 592  *   Pointer to mlx5_ibv_shared object to free
 593  */
 594 static void
 595 mlx5_free_shared_ibctx(struct mlx5_ibv_shared *sh)
 596 {
 597         pthread_mutex_lock(&mlx5_ibv_list_mutex);
 598 #ifndef NDEBUG
 599         /* Check the object presence in the list. */
 600         struct mlx5_ibv_shared *lctx;
 601
 602         LIST_FOREACH(lctx, &mlx5_ibv_list, next)
 603                 if (lctx == sh)
 604                         break;
 605         assert(lctx);
 606         if (lctx != sh) {
 607                 DRV_LOG(ERR, "Freeing non-existing shared IB context");
 608                 goto exit;
 609         }
 610 #endif
 611         assert(sh);
 612         assert(sh->refcnt);
 613         /* Secondary process should not free the shared context. */
 614         assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
 615         if (--sh->refcnt)
 616                 goto exit;
 617         /* Release created Memory Regions. */
 618         mlx5_mr_release(sh);
 619         /* Remove from memory callback device list. */
 620         rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
 621         LIST_REMOVE(sh, mem_event_cb);
 622         rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
 623         /* Remove context from the global device list. */
 624         LIST_REMOVE(sh, next);
 625         /*
 626          *  Ensure there is no async event handler installed.
 627          *  Only primary process handles async device events.
 628          **/
 629         mlx5_flow_counters_mng_close(sh);
 630         assert(!sh->intr_cnt);
 631         if (sh->intr_cnt)
 632                 mlx5_intr_callback_unregister
 633                         (&sh->intr_handle, mlx5_dev_interrupt_handler, sh);
 634 #ifdef HAVE_MLX5_DEVX_ASYNC_SUPPORT
 635         if (sh->devx_intr_cnt) {
 636                 if (sh->intr_handle_devx.fd)
 637                         rte_intr_callback_unregister(&sh->intr_handle_devx,
 638                                           mlx5_dev_interrupt_handler_devx, sh);
 639                 if (sh->devx_comp)
 640                         mlx5dv_devx_destroy_cmd_comp(sh->devx_comp);
 641         }
 642 #endif
 643         pthread_mutex_destroy(&sh->intr_mutex);
 644         if (sh->pd)
 645                 claim_zero(mlx5_glue->dealloc_pd(sh->pd));
 646         if (sh->tis)
 647                 claim_zero(mlx5_devx_cmd_destroy(sh->tis));
 648         if (sh->td)
 649                 claim_zero(mlx5_devx_cmd_destroy(sh->td));
 650         if (sh->ctx)
 651                 claim_zero(mlx5_glue->close_device(sh->ctx));
 652         if (sh->flow_id_pool)
 653                 mlx5_flow_id_pool_release(sh->flow_id_pool);
 654         rte_free(sh);
 655 exit:
 656         pthread_mutex_unlock(&mlx5_ibv_list_mutex);
 657 }
 658
 659 /**
 660  * Initialize DR related data within private structure.
 661  * Routine checks the reference counter and does actual
 662  * resources creation/initialization only if counter is zero.
 663  *
 664  * @param[in] priv
 665  *   Pointer to the private device data structure.
 666  *
 667  * @return
 668  *   Zero on success, positive error code otherwise.
 669  */
 670 static int
 671 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 672 {
 673 #ifdef HAVE_MLX5DV_DR
 674         struct mlx5_ibv_shared *sh = priv->sh;
 675         int err = 0;
 676         void *domain;
 677
 678         assert(sh);
 679         if (sh->dv_refcnt) {
 680                 /* Shared DV/DR structures is already initialized. */
 681                 sh->dv_refcnt++;
 682                 priv->dr_shared = 1;
 683                 return 0;
 684         }
 685         /* Reference counter is zero, we should initialize structures. */
 686         domain = mlx5_glue->dr_create_domain(sh->ctx,
 687                                              MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
 688         if (!domain) {
 689                 DRV_LOG(ERR, "ingress mlx5dv_dr_create_domain failed");
 690                 err = errno;
 691                 goto error;
 692         }
 693         sh->rx_domain = domain;
 694         domain = mlx5_glue->dr_create_domain(sh->ctx,
 695                                              MLX5DV_DR_DOMAIN_TYPE_NIC_TX);
 696         if (!domain) {
 697                 DRV_LOG(ERR, "egress mlx5dv_dr_create_domain failed");
 698                 err = errno;
 699                 goto error;
 700         }
 701         pthread_mutex_init(&sh->dv_mutex, NULL);
 702         sh->tx_domain = domain;
 703 #ifdef HAVE_MLX5DV_DR_ESWITCH
 704         if (priv->config.dv_esw_en) {
 705                 domain  = mlx5_glue->dr_create_domain
 706                         (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB);
 707                 if (!domain) {
 708                         DRV_LOG(ERR, "FDB mlx5dv_dr_create_domain failed");
 709                         err = errno;
 710                         goto error;
 711                 }
 712                 sh->fdb_domain = domain;
 713                 sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop();
 714         }
 715 #endif
 716         sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan();
 717         sh->dv_refcnt++;
 718         priv->dr_shared = 1;
 719         return 0;
 720
 721 error:
 722        /* Rollback the created objects. */
 723         if (sh->rx_domain) {
 724                 mlx5_glue->dr_destroy_domain(sh->rx_domain);
 725                 sh->rx_domain = NULL;
 726         }
 727         if (sh->tx_domain) {
 728                 mlx5_glue->dr_destroy_domain(sh->tx_domain);
 729                 sh->tx_domain = NULL;
 730         }
 731         if (sh->fdb_domain) {
 732                 mlx5_glue->dr_destroy_domain(sh->fdb_domain);
 733                 sh->fdb_domain = NULL;
 734         }
 735         if (sh->esw_drop_action) {
 736                 mlx5_glue->destroy_flow_action(sh->esw_drop_action);
 737                 sh->esw_drop_action = NULL;
 738         }
 739         if (sh->pop_vlan_action) {
 740                 mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
 741                 sh->pop_vlan_action = NULL;
 742         }
 743         return err;
 744 #else
 745         (void)priv;
 746         return 0;
 747 #endif
 748 }
 749
 750 /**
 751  * Destroy DR related data within private structure.
 752  *
 753  * @param[in] priv
 754  *   Pointer to the private device data structure.
 755  */
 756 static void
 757 mlx5_free_shared_dr(struct mlx5_priv *priv)
 758 {
 759 #ifdef HAVE_MLX5DV_DR
 760         struct mlx5_ibv_shared *sh;
 761
 762         if (!priv->dr_shared)
 763                 return;
 764         priv->dr_shared = 0;
 765         sh = priv->sh;
 766         assert(sh);
 767         assert(sh->dv_refcnt);
 768         if (sh->dv_refcnt && --sh->dv_refcnt)
 769                 return;
 770         if (sh->rx_domain) {
 771                 mlx5_glue->dr_destroy_domain(sh->rx_domain);
 772                 sh->rx_domain = NULL;
 773         }
 774         if (sh->tx_domain) {
 775                 mlx5_glue->dr_destroy_domain(sh->tx_domain);
 776                 sh->tx_domain = NULL;
 777         }
 778 #ifdef HAVE_MLX5DV_DR_ESWITCH
 779         if (sh->fdb_domain) {
 780                 mlx5_glue->dr_destroy_domain(sh->fdb_domain);
 781                 sh->fdb_domain = NULL;
 782         }
 783         if (sh->esw_drop_action) {
 784                 mlx5_glue->destroy_flow_action(sh->esw_drop_action);
 785                 sh->esw_drop_action = NULL;
 786         }
 787 #endif
 788         if (sh->pop_vlan_action) {
 789                 mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
 790                 sh->pop_vlan_action = NULL;
 791         }
 792         pthread_mutex_destroy(&sh->dv_mutex);
 793 #else
 794         (void)priv;
 795 #endif
 796 }
 797
 798 /**
 799  * Initialize shared data between primary and secondary process.
 800  *
 801  * A memzone is reserved by primary process and secondary processes attach to
 802  * the memzone.
 803  *
 804  * @return
 805  *   0 on success, a negative errno value otherwise and rte_errno is set.
 806  */
 807 static int
 808 mlx5_init_shared_data(void)
 809 {
 810         const struct rte_memzone *mz;
 811         int ret = 0;
 812
 813         rte_spinlock_lock(&mlx5_shared_data_lock);
 814         if (mlx5_shared_data == NULL) {
 815                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 816                         /* Allocate shared memory. */
 817                         mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
 818                                                  sizeof(*mlx5_shared_data),
 819                                                  SOCKET_ID_ANY, 0);
 820                         if (mz == NULL) {
 821                                 DRV_LOG(ERR,
 822                                         "Cannot allocate mlx5 shared data");
 823                                 ret = -rte_errno;
 824                                 goto error;
 825                         }
 826                         mlx5_shared_data = mz->addr;
 827                         memset(mlx5_shared_data, 0, sizeof(*mlx5_shared_data));
 828                         rte_spinlock_init(&mlx5_shared_data->lock);
 829                 } else {
 830                         /* Lookup allocated shared memory. */
 831                         mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
 832                         if (mz == NULL) {
 833                                 DRV_LOG(ERR,
 834                                         "Cannot attach mlx5 shared data");
 835                                 ret = -rte_errno;
 836                                 goto error;
 837                         }
 838                         mlx5_shared_data = mz->addr;
 839                         memset(&mlx5_local_data, 0, sizeof(mlx5_local_data));
 840                 }
 841         }
 842 error:
 843         rte_spinlock_unlock(&mlx5_shared_data_lock);
 844         return ret;
 845 }
 846
 847 /**
 848  * Retrieve integer value from environment variable.
 849  *
 850  * @param[in] name
 851  *   Environment variable name.
 852  *
 853  * @return
 854  *   Integer value, 0 if the variable is not set.
 855  */
 856 int
 857 mlx5_getenv_int(const char *name)
 858 {
 859         const char *val = getenv(name);
 860
 861         if (val == NULL)
 862                 return 0;
 863         return atoi(val);
 864 }
 865
 866 /**
 867  * Verbs callback to allocate a memory. This function should allocate the space
 868  * according to the size provided residing inside a huge page.
 869  * Please note that all allocation must respect the alignment from libmlx5
 870  * (i.e. currently sysconf(_SC_PAGESIZE)).
 871  *
 872  * @param[in] size
 873  *   The size in bytes of the memory to allocate.
 874  * @param[in] data
 875  *   A pointer to the callback data.
 876  *
 877  * @return
 878  *   Allocated buffer, NULL otherwise and rte_errno is set.
 879  */
 880 static void *
 881 mlx5_alloc_verbs_buf(size_t size, void *data)
 882 {
 883         struct mlx5_priv *priv = data;
 884         void *ret;
 885         size_t alignment = sysconf(_SC_PAGESIZE);
 886         unsigned int socket = SOCKET_ID_ANY;
 887
 888         if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) {
 889                 const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj;
 890
 891                 socket = ctrl->socket;
 892         } else if (priv->verbs_alloc_ctx.type ==
 893                    MLX5_VERBS_ALLOC_TYPE_RX_QUEUE) {
 894                 const struct mlx5_rxq_ctrl *ctrl = priv->verbs_alloc_ctx.obj;
 895
 896                 socket = ctrl->socket;
 897         }
 898         assert(data != NULL);
 899         ret = rte_malloc_socket(__func__, size, alignment, socket);
 900         if (!ret && size)
 901                 rte_errno = ENOMEM;
 902         return ret;
 903 }
 904
 905 /**
 906  * Verbs callback to free a memory.
 907  *
 908  * @param[in] ptr
 909  *   A pointer to the memory to free.
 910  * @param[in] data
 911  *   A pointer to the callback data.
 912  */
 913 static void
 914 mlx5_free_verbs_buf(void *ptr, void *data __rte_unused)
 915 {
 916         assert(data != NULL);
 917         rte_free(ptr);
 918 }
 919
 920 /**
 921  * DPDK callback to add udp tunnel port
 922  *
 923  * @param[in] dev
 924  *   A pointer to eth_dev
 925  * @param[in] udp_tunnel
 926  *   A pointer to udp tunnel
 927  *
 928  * @return
 929  *   0 on valid udp ports and tunnels, -ENOTSUP otherwise.
 930  */
 931 int
 932 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused,
 933                          struct rte_eth_udp_tunnel *udp_tunnel)
 934 {
 935         assert(udp_tunnel != NULL);
 936         if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN &&
 937             udp_tunnel->udp_port == 4789)
 938                 return 0;
 939         if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN_GPE &&
 940             udp_tunnel->udp_port == 4790)
 941                 return 0;
 942         return -ENOTSUP;
 943 }
 944
 945 /**
 946  * Initialize process private data structure.
 947  *
 948  * @param dev
 949  *   Pointer to Ethernet device structure.
 950  *
 951  * @return
 952  *   0 on success, a negative errno value otherwise and rte_errno is set.
 953  */
 954 int
 955 mlx5_proc_priv_init(struct rte_eth_dev *dev)
 956 {
 957         struct mlx5_priv *priv = dev->data->dev_private;
 958         struct mlx5_proc_priv *ppriv;
 959         size_t ppriv_size;
 960
 961         /*
 962          * UAR register table follows the process private structure. BlueFlame
 963          * registers for Tx queues are stored in the table.
 964          */
 965         ppriv_size =
 966                 sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *);
 967         ppriv = rte_malloc_socket("mlx5_proc_priv", ppriv_size,
 968                                   RTE_CACHE_LINE_SIZE, dev->device->numa_node);
 969         if (!ppriv) {
 970                 rte_errno = ENOMEM;
 971                 return -rte_errno;
 972         }
 973         ppriv->uar_table_sz = ppriv_size;
 974         dev->process_private = ppriv;
 975         return 0;
 976 }
 977
 978 /**
 979  * Un-initialize process private data structure.
 980  *
 981  * @param dev
 982  *   Pointer to Ethernet device structure.
 983  */
 984 static void
 985 mlx5_proc_priv_uninit(struct rte_eth_dev *dev)
 986 {
 987         if (!dev->process_private)
 988                 return;
 989         rte_free(dev->process_private);
 990         dev->process_private = NULL;
 991 }
 992
 993 /**
 994  * DPDK callback to close the device.
 995  *
 996  * Destroy all queues and objects, free memory.
 997  *
 998  * @param dev
 999  *   Pointer to Ethernet device structure.
1000  */
1001 static void
1002 mlx5_dev_close(struct rte_eth_dev *dev)
1003 {
1004         struct mlx5_priv *priv = dev->data->dev_private;
1005         unsigned int i;
1006         int ret;
1007
1008         DRV_LOG(DEBUG, "port %u closing device \"%s\"",
1009                 dev->data->port_id,
1010                 ((priv->sh->ctx != NULL) ? priv->sh->ctx->device->name : ""));
1011         /* In case mlx5_dev_stop() has not been called. */
1012         mlx5_dev_interrupt_handler_uninstall(dev);
1013         mlx5_dev_interrupt_handler_devx_uninstall(dev);
1014         mlx5_traffic_disable(dev);
1015         mlx5_flow_flush(dev, NULL);
1016         /* Prevent crashes when queues are still in use. */
1017         dev->rx_pkt_burst = removed_rx_burst;
1018         dev->tx_pkt_burst = removed_tx_burst;
1019         rte_wmb();
1020         /* Disable datapath on secondary process. */
1021         mlx5_mp_req_stop_rxtx(dev);
1022         if (priv->rxqs != NULL) {
1023                 /* XXX race condition if mlx5_rx_burst() is still running. */
1024                 usleep(1000);
1025                 for (i = 0; (i != priv->rxqs_n); ++i)
1026                         mlx5_rxq_release(dev, i);
1027                 priv->rxqs_n = 0;
1028                 priv->rxqs = NULL;
1029         }
1030         if (priv->txqs != NULL) {
1031                 /* XXX race condition if mlx5_tx_burst() is still running. */
1032                 usleep(1000);
1033                 for (i = 0; (i != priv->txqs_n); ++i)
1034                         mlx5_txq_release(dev, i);
1035                 priv->txqs_n = 0;
1036                 priv->txqs = NULL;
1037         }
1038         mlx5_proc_priv_uninit(dev);
1039         mlx5_mprq_free_mp(dev);
1040         mlx5_free_shared_dr(priv);
1041         if (priv->rss_conf.rss_key != NULL)
1042                 rte_free(priv->rss_conf.rss_key);
1043         if (priv->reta_idx != NULL)
1044                 rte_free(priv->reta_idx);
1045         if (priv->config.vf)
1046                 mlx5_nl_mac_addr_flush(dev);
1047         if (priv->nl_socket_route >= 0)
1048                 close(priv->nl_socket_route);
1049         if (priv->nl_socket_rdma >= 0)
1050                 close(priv->nl_socket_rdma);
1051         if (priv->vmwa_context)
1052                 mlx5_vlan_vmwa_exit(priv->vmwa_context);
1053         if (priv->sh) {
1054                 /*
1055                  * Free the shared context in last turn, because the cleanup
1056                  * routines above may use some shared fields, like
1057                  * mlx5_nl_mac_addr_flush() uses ibdev_path for retrieveing
1058                  * ifindex if Netlink fails.
1059                  */
1060                 mlx5_free_shared_ibctx(priv->sh);
1061                 priv->sh = NULL;
1062         }
1063         ret = mlx5_hrxq_verify(dev);
1064         if (ret)
1065                 DRV_LOG(WARNING, "port %u some hash Rx queue still remain",
1066                         dev->data->port_id);
1067         ret = mlx5_ind_table_obj_verify(dev);
1068         if (ret)
1069                 DRV_LOG(WARNING, "port %u some indirection table still remain",
1070                         dev->data->port_id);
1071         ret = mlx5_rxq_obj_verify(dev);
1072         if (ret)
1073                 DRV_LOG(WARNING, "port %u some Rx queue objects still remain",
1074                         dev->data->port_id);
1075         ret = mlx5_rxq_verify(dev);
1076         if (ret)
1077                 DRV_LOG(WARNING, "port %u some Rx queues still remain",
1078                         dev->data->port_id);
1079         ret = mlx5_txq_obj_verify(dev);
1080         if (ret)
1081                 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain",
1082                         dev->data->port_id);
1083         ret = mlx5_txq_verify(dev);
1084         if (ret)
1085                 DRV_LOG(WARNING, "port %u some Tx queues still remain",
1086                         dev->data->port_id);
1087         ret = mlx5_flow_verify(dev);
1088         if (ret)
1089                 DRV_LOG(WARNING, "port %u some flows still remain",
1090                         dev->data->port_id);
1091         if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
1092                 unsigned int c = 0;
1093                 uint16_t port_id;
1094
1095                 MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
1096                         struct mlx5_priv *opriv =
1097                                 rte_eth_devices[port_id].data->dev_private;
1098
1099                         if (!opriv ||
1100                             opriv->domain_id != priv->domain_id ||
1101                             &rte_eth_devices[port_id] == dev)
1102                                 continue;
1103                         ++c;
1104                         break;
1105                 }
1106                 if (!c)
1107                         claim_zero(rte_eth_switch_domain_free(priv->domain_id));
1108         }
1109         memset(priv, 0, sizeof(*priv));
1110         priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
1111         /*
1112          * Reset mac_addrs to NULL such that it is not freed as part of
1113          * rte_eth_dev_release_port(). mac_addrs is part of dev_private so
1114          * it is freed when dev_private is freed.
1115          */
1116         dev->data->mac_addrs = NULL;
1117 }
1118
1119 const struct eth_dev_ops mlx5_dev_ops = {
1120         .dev_configure = mlx5_dev_configure,
1121         .dev_start = mlx5_dev_start,
1122         .dev_stop = mlx5_dev_stop,
1123         .dev_set_link_down = mlx5_set_link_down,
1124         .dev_set_link_up = mlx5_set_link_up,
1125         .dev_close = mlx5_dev_close,
1126         .promiscuous_enable = mlx5_promiscuous_enable,
1127         .promiscuous_disable = mlx5_promiscuous_disable,
1128         .allmulticast_enable = mlx5_allmulticast_enable,
1129         .allmulticast_disable = mlx5_allmulticast_disable,
1130         .link_update = mlx5_link_update,
1131         .stats_get = mlx5_stats_get,
1132         .stats_reset = mlx5_stats_reset,
1133         .xstats_get = mlx5_xstats_get,
1134         .xstats_reset = mlx5_xstats_reset,
1135         .xstats_get_names = mlx5_xstats_get_names,
1136         .fw_version_get = mlx5_fw_version_get,
1137         .dev_infos_get = mlx5_dev_infos_get,
1138         .read_clock = mlx5_read_clock,
1139         .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
1140         .vlan_filter_set = mlx5_vlan_filter_set,
1141         .rx_queue_setup = mlx5_rx_queue_setup,
1142         .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup,
1143         .tx_queue_setup = mlx5_tx_queue_setup,
1144         .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup,
1145         .rx_queue_release = mlx5_rx_queue_release,
1146         .tx_queue_release = mlx5_tx_queue_release,
1147         .flow_ctrl_get = mlx5_dev_get_flow_ctrl,
1148         .flow_ctrl_set = mlx5_dev_set_flow_ctrl,
1149         .mac_addr_remove = mlx5_mac_addr_remove,
1150         .mac_addr_add = mlx5_mac_addr_add,
1151         .mac_addr_set = mlx5_mac_addr_set,
1152         .set_mc_addr_list = mlx5_set_mc_addr_list,
1153         .mtu_set = mlx5_dev_set_mtu,
1154         .vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
1155         .vlan_offload_set = mlx5_vlan_offload_set,
1156         .reta_update = mlx5_dev_rss_reta_update,
1157         .reta_query = mlx5_dev_rss_reta_query,
1158         .rss_hash_update = mlx5_rss_hash_update,
1159         .rss_hash_conf_get = mlx5_rss_hash_conf_get,
1160         .filter_ctrl = mlx5_dev_filter_ctrl,
1161         .rx_descriptor_status = mlx5_rx_descriptor_status,
1162         .tx_descriptor_status = mlx5_tx_descriptor_status,
1163         .rx_queue_count = mlx5_rx_queue_count,
1164         .rx_queue_intr_enable = mlx5_rx_intr_enable,
1165         .rx_queue_intr_disable = mlx5_rx_intr_disable,
1166         .is_removed = mlx5_is_removed,
1167         .udp_tunnel_port_add  = mlx5_udp_tunnel_port_add,
1168         .get_module_info = mlx5_get_module_info,
1169         .get_module_eeprom = mlx5_get_module_eeprom,
1170         .hairpin_cap_get = mlx5_hairpin_cap_get,
1171 };
1172
1173 /* Available operations from secondary process. */
1174 static const struct eth_dev_ops mlx5_dev_sec_ops = {
1175         .stats_get = mlx5_stats_get,
1176         .stats_reset = mlx5_stats_reset,
1177         .xstats_get = mlx5_xstats_get,
1178         .xstats_reset = mlx5_xstats_reset,
1179         .xstats_get_names = mlx5_xstats_get_names,
1180         .fw_version_get = mlx5_fw_version_get,
1181         .dev_infos_get = mlx5_dev_infos_get,
1182         .rx_descriptor_status = mlx5_rx_descriptor_status,
1183         .tx_descriptor_status = mlx5_tx_descriptor_status,
1184         .get_module_info = mlx5_get_module_info,
1185         .get_module_eeprom = mlx5_get_module_eeprom,
1186 };
1187
1188 /* Available operations in flow isolated mode. */
1189 const struct eth_dev_ops mlx5_dev_ops_isolate = {
1190         .dev_configure = mlx5_dev_configure,
1191         .dev_start = mlx5_dev_start,
1192         .dev_stop = mlx5_dev_stop,
1193         .dev_set_link_down = mlx5_set_link_down,
1194         .dev_set_link_up = mlx5_set_link_up,
1195         .dev_close = mlx5_dev_close,
1196         .promiscuous_enable = mlx5_promiscuous_enable,
1197         .promiscuous_disable = mlx5_promiscuous_disable,
1198         .allmulticast_enable = mlx5_allmulticast_enable,
1199         .allmulticast_disable = mlx5_allmulticast_disable,
1200         .link_update = mlx5_link_update,
1201         .stats_get = mlx5_stats_get,
1202         .stats_reset = mlx5_stats_reset,
1203         .xstats_get = mlx5_xstats_get,
1204         .xstats_reset = mlx5_xstats_reset,
1205         .xstats_get_names = mlx5_xstats_get_names,
1206         .fw_version_get = mlx5_fw_version_get,
1207         .dev_infos_get = mlx5_dev_infos_get,
1208         .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
1209         .vlan_filter_set = mlx5_vlan_filter_set,
1210         .rx_queue_setup = mlx5_rx_queue_setup,
1211         .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup,
1212         .tx_queue_setup = mlx5_tx_queue_setup,
1213         .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup,
1214         .rx_queue_release = mlx5_rx_queue_release,
1215         .tx_queue_release = mlx5_tx_queue_release,
1216         .flow_ctrl_get = mlx5_dev_get_flow_ctrl,
1217         .flow_ctrl_set = mlx5_dev_set_flow_ctrl,
1218         .mac_addr_remove = mlx5_mac_addr_remove,
1219         .mac_addr_add = mlx5_mac_addr_add,
1220         .mac_addr_set = mlx5_mac_addr_set,
1221         .set_mc_addr_list = mlx5_set_mc_addr_list,
1222         .mtu_set = mlx5_dev_set_mtu,
1223         .vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
1224         .vlan_offload_set = mlx5_vlan_offload_set,
1225         .filter_ctrl = mlx5_dev_filter_ctrl,
1226         .rx_descriptor_status = mlx5_rx_descriptor_status,
1227         .tx_descriptor_status = mlx5_tx_descriptor_status,
1228         .rx_queue_intr_enable = mlx5_rx_intr_enable,
1229         .rx_queue_intr_disable = mlx5_rx_intr_disable,
1230         .is_removed = mlx5_is_removed,
1231         .get_module_info = mlx5_get_module_info,
1232         .get_module_eeprom = mlx5_get_module_eeprom,
1233         .hairpin_cap_get = mlx5_hairpin_cap_get,
1234 };
1235
1236 /**
1237  * Verify and store value for device argument.
1238  *
1239  * @param[in] key
1240  *   Key argument to verify.
1241  * @param[in] val
1242  *   Value associated with key.
1243  * @param opaque
1244  *   User data.
1245  *
1246  * @return
1247  *   0 on success, a negative errno value otherwise and rte_errno is set.
1248  */
1249 static int
1250 mlx5_args_check(const char *key, const char *val, void *opaque)
1251 {
1252         struct mlx5_dev_config *config = opaque;
1253         unsigned long tmp;
1254
1255         /* No-op, port representors are processed in mlx5_dev_spawn(). */
1256         if (!strcmp(MLX5_REPRESENTOR, key))
1257                 return 0;
1258         errno = 0;
1259         tmp = strtoul(val, NULL, 0);
1260         if (errno) {
1261                 rte_errno = errno;
1262                 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val);
1263                 return -rte_errno;
1264         }
1265         if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) {
1266                 config->cqe_comp = !!tmp;
1267         } else if (strcmp(MLX5_RXQ_CQE_PAD_EN, key) == 0) {
1268                 config->cqe_pad = !!tmp;
1269         } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) {
1270                 config->hw_padding = !!tmp;
1271         } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) {
1272                 config->mprq.enabled = !!tmp;
1273         } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) {
1274                 config->mprq.stride_num_n = tmp;
1275         } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) {
1276                 config->mprq.max_memcpy_len = tmp;
1277         } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) {
1278                 config->mprq.min_rxqs_num = tmp;
1279         } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) {
1280                 DRV_LOG(WARNING, "%s: deprecated parameter,"
1281                                  " converted to txq_inline_max", key);
1282                 config->txq_inline_max = tmp;
1283         } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) {
1284                 config->txq_inline_max = tmp;
1285         } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) {
1286                 config->txq_inline_min = tmp;
1287         } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) {
1288                 config->txq_inline_mpw = tmp;
1289         } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) {
1290                 config->txqs_inline = tmp;
1291         } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) {
1292                 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key);
1293         } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) {
1294                 config->mps = !!tmp;
1295         } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) {
1296                 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key);
1297         } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) {
1298                 DRV_LOG(WARNING, "%s: deprecated parameter,"
1299                                  " converted to txq_inline_mpw", key);
1300                 config->txq_inline_mpw = tmp;
1301         } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) {
1302                 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key);
1303         } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) {
1304                 config->rx_vec_en = !!tmp;
1305         } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) {
1306                 config->l3_vxlan_en = !!tmp;
1307         } else if (strcmp(MLX5_VF_NL_EN, key) == 0) {
1308                 config->vf_nl_en = !!tmp;
1309         } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) {
1310                 config->dv_esw_en = !!tmp;
1311         } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) {
1312                 config->dv_flow_en = !!tmp;
1313         } else if (strcmp(MLX5_MR_EXT_MEMSEG_EN, key) == 0) {
1314                 config->mr_ext_memseg_en = !!tmp;
1315         } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) {
1316                 config->max_dump_files_num = tmp;
1317         } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
1318                 config->lro.timeout = tmp;
1319         } else {
1320                 DRV_LOG(WARNING, "%s: unknown parameter", key);
1321                 rte_errno = EINVAL;
1322                 return -rte_errno;
1323         }
1324         return 0;
1325 }
1326
1327 /**
1328  * Parse device parameters.
1329  *
1330  * @param config
1331  *   Pointer to device configuration structure.
1332  * @param devargs
1333  *   Device arguments structure.
1334  *
1335  * @return
1336  *   0 on success, a negative errno value otherwise and rte_errno is set.
1337  */
1338 static int
1339 mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
1340 {
1341         const char **params = (const char *[]){
1342                 MLX5_RXQ_CQE_COMP_EN,
1343                 MLX5_RXQ_CQE_PAD_EN,
1344                 MLX5_RXQ_PKT_PAD_EN,
1345                 MLX5_RX_MPRQ_EN,
1346                 MLX5_RX_MPRQ_LOG_STRIDE_NUM,
1347                 MLX5_RX_MPRQ_MAX_MEMCPY_LEN,
1348                 MLX5_RXQS_MIN_MPRQ,
1349                 MLX5_TXQ_INLINE,
1350                 MLX5_TXQ_INLINE_MIN,
1351                 MLX5_TXQ_INLINE_MAX,
1352                 MLX5_TXQ_INLINE_MPW,
1353                 MLX5_TXQS_MIN_INLINE,
1354                 MLX5_TXQS_MAX_VEC,
1355                 MLX5_TXQ_MPW_EN,
1356                 MLX5_TXQ_MPW_HDR_DSEG_EN,
1357                 MLX5_TXQ_MAX_INLINE_LEN,
1358                 MLX5_TX_VEC_EN,
1359                 MLX5_RX_VEC_EN,
1360                 MLX5_L3_VXLAN_EN,
1361                 MLX5_VF_NL_EN,
1362                 MLX5_DV_ESW_EN,
1363                 MLX5_DV_FLOW_EN,
1364                 MLX5_MR_EXT_MEMSEG_EN,
1365                 MLX5_REPRESENTOR,
1366                 MLX5_MAX_DUMP_FILES_NUM,
1367                 MLX5_LRO_TIMEOUT_USEC,
1368                 NULL,
1369         };
1370         struct rte_kvargs *kvlist;
1371         int ret = 0;
1372         int i;
1373
1374         if (devargs == NULL)
1375                 return 0;
1376         /* Following UGLY cast is done to pass checkpatch. */
1377         kvlist = rte_kvargs_parse(devargs->args, params);
1378         if (kvlist == NULL) {
1379                 rte_errno = EINVAL;
1380                 return -rte_errno;
1381         }
1382         /* Process parameters. */
1383         for (i = 0; (params[i] != NULL); ++i) {
1384                 if (rte_kvargs_count(kvlist, params[i])) {
1385                         ret = rte_kvargs_process(kvlist, params[i],
1386                                                  mlx5_args_check, config);
1387                         if (ret) {
1388                                 rte_errno = EINVAL;
1389                                 rte_kvargs_free(kvlist);
1390                                 return -rte_errno;
1391                         }
1392                 }
1393         }
1394         rte_kvargs_free(kvlist);
1395         return 0;
1396 }
1397
1398 static struct rte_pci_driver mlx5_driver;
1399
1400 /**
1401  * PMD global initialization.
1402  *
1403  * Independent from individual device, this function initializes global
1404  * per-PMD data structures distinguishing primary and secondary processes.
1405  * Hence, each initialization is called once per a process.
1406  *
1407  * @return
1408  *   0 on success, a negative errno value otherwise and rte_errno is set.
1409  */
1410 static int
1411 mlx5_init_once(void)
1412 {
1413         struct mlx5_shared_data *sd;
1414         struct mlx5_local_data *ld = &mlx5_local_data;
1415         int ret = 0;
1416
1417         if (mlx5_init_shared_data())
1418                 return -rte_errno;
1419         sd = mlx5_shared_data;
1420         assert(sd);
1421         rte_spinlock_lock(&sd->lock);
1422         switch (rte_eal_process_type()) {
1423         case RTE_PROC_PRIMARY:
1424                 if (sd->init_done)
1425                         break;
1426                 LIST_INIT(&sd->mem_event_cb_list);
1427                 rte_rwlock_init(&sd->mem_event_rwlock);
1428                 rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
1429                                                 mlx5_mr_mem_event_cb, NULL);
1430                 ret = mlx5_mp_init_primary();
1431                 if (ret)
1432                         goto out;
1433                 sd->init_done = true;
1434                 break;
1435         case RTE_PROC_SECONDARY:
1436                 if (ld->init_done)
1437                         break;
1438                 ret = mlx5_mp_init_secondary();
1439                 if (ret)
1440                         goto out;
1441                 ++sd->secondary_cnt;
1442                 ld->init_done = true;
1443                 break;
1444         default:
1445                 break;
1446         }
1447 out:
1448         rte_spinlock_unlock(&sd->lock);
1449         return ret;
1450 }
1451
1452 /**
1453  * Configures the minimal amount of data to inline into WQE
1454  * while sending packets.
1455  *
1456  * - the txq_inline_min has the maximal priority, if this
1457  *   key is specified in devargs
1458  * - if DevX is enabled the inline mode is queried from the
1459  *   device (HCA attributes and NIC vport context if needed).
1460  * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4LX
1461  *   and none (0 bytes) for other NICs
1462  *
1463  * @param spawn
1464  *   Verbs device parameters (name, port, switch_info) to spawn.
1465  * @param config
1466  *   Device configuration parameters.
1467  */
1468 static void
1469 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
1470                     struct mlx5_dev_config *config)
1471 {
1472         if (config->txq_inline_min != MLX5_ARG_UNSET) {
1473                 /* Application defines size of inlined data explicitly. */
1474                 switch (spawn->pci_dev->id.device_id) {
1475                 case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
1476                 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
1477                         if (config->txq_inline_min <
1478                                        (int)MLX5_INLINE_HSIZE_L2) {
1479                                 DRV_LOG(DEBUG,
1480                                         "txq_inline_mix aligned to minimal"
1481                                         " ConnectX-4 required value %d",
1482                                         (int)MLX5_INLINE_HSIZE_L2);
1483                                 config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
1484                         }
1485                         break;
1486                 }
1487                 goto exit;
1488         }
1489         if (config->hca_attr.eth_net_offloads) {
1490                 /* We have DevX enabled, inline mode queried successfully. */
1491                 switch (config->hca_attr.wqe_inline_mode) {
1492                 case MLX5_CAP_INLINE_MODE_L2:
1493                         /* outer L2 header must be inlined. */
1494                         config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
1495                         goto exit;
1496                 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
1497                         /* No inline data are required by NIC. */
1498                         config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
1499                         config->hw_vlan_insert =
1500                                 config->hca_attr.wqe_vlan_insert;
1501                         DRV_LOG(DEBUG, "Tx VLAN insertion is supported");
1502                         goto exit;
1503                 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
1504                         /* inline mode is defined by NIC vport context. */
1505                         if (!config->hca_attr.eth_virt)
1506                                 break;
1507                         switch (config->hca_attr.vport_inline_mode) {
1508                         case MLX5_INLINE_MODE_NONE:
1509                                 config->txq_inline_min =
1510                                         MLX5_INLINE_HSIZE_NONE;
1511                                 goto exit;
1512                         case MLX5_INLINE_MODE_L2:
1513                                 config->txq_inline_min =
1514                                         MLX5_INLINE_HSIZE_L2;
1515                                 goto exit;
1516                         case MLX5_INLINE_MODE_IP:
1517                                 config->txq_inline_min =
1518                                         MLX5_INLINE_HSIZE_L3;
1519                                 goto exit;
1520                         case MLX5_INLINE_MODE_TCP_UDP:
1521                                 config->txq_inline_min =
1522                                         MLX5_INLINE_HSIZE_L4;
1523                                 goto exit;
1524                         case MLX5_INLINE_MODE_INNER_L2:
1525                                 config->txq_inline_min =
1526                                         MLX5_INLINE_HSIZE_INNER_L2;
1527                                 goto exit;
1528                         case MLX5_INLINE_MODE_INNER_IP:
1529                                 config->txq_inline_min =
1530                                         MLX5_INLINE_HSIZE_INNER_L3;
1531                                 goto exit;
1532                         case MLX5_INLINE_MODE_INNER_TCP_UDP:
1533                                 config->txq_inline_min =
1534                                         MLX5_INLINE_HSIZE_INNER_L4;
1535                                 goto exit;
1536                         }
1537                 }
1538         }
1539         /*
1540          * We get here if we are unable to deduce
1541          * inline data size with DevX. Try PCI ID
1542          * to determine old NICs.
1543          */
1544         switch (spawn->pci_dev->id.device_id) {
1545         case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
1546         case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
1547         case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX:
1548         case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
1549                 config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
1550                 config->hw_vlan_insert = 0;
1551                 break;
1552         case PCI_DEVICE_ID_MELLANOX_CONNECTX5:
1553         case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
1554         case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX:
1555         case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
1556                 /*
1557                  * These NICs support VLAN insertion from WQE and
1558                  * report the wqe_vlan_insert flag. But there is the bug
1559                  * and PFC control may be broken, so disable feature.
1560                  */
1561                 config->hw_vlan_insert = 0;
1562                 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
1563                 break;
1564         default:
1565                 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
1566                 break;
1567         }
1568 exit:
1569         DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min);
1570 }
1571
1572 /**
1573  * Allocate page of door-bells and register it using DevX API.
1574  *
1575  * @param [in] dev
1576  *   Pointer to Ethernet device.
1577  *
1578  * @return
1579  *   Pointer to new page on success, NULL otherwise.
1580  */
1581 static struct mlx5_devx_dbr_page *
1582 mlx5_alloc_dbr_page(struct rte_eth_dev *dev)
1583 {
1584         struct mlx5_priv *priv = dev->data->dev_private;
1585         struct mlx5_devx_dbr_page *page;
1586
1587         /* Allocate space for door-bell page and management data. */
1588         page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
1589                                  RTE_CACHE_LINE_SIZE, dev->device->numa_node);
1590         if (!page) {
1591                 DRV_LOG(ERR, "port %u cannot allocate dbr page",
1592                         dev->data->port_id);
1593                 return NULL;
1594         }
1595         /* Register allocated memory. */
1596         page->umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, page->dbrs,
1597                                               MLX5_DBR_PAGE_SIZE, 0);
1598         if (!page->umem) {
1599                 DRV_LOG(ERR, "port %u cannot umem reg dbr page",
1600                         dev->data->port_id);
1601                 rte_free(page);
1602                 return NULL;
1603         }
1604         return page;
1605 }
1606
1607 /**
1608  * Find the next available door-bell, allocate new page if needed.
1609  *
1610  * @param [in] dev
1611  *   Pointer to Ethernet device.
1612  * @param [out] dbr_page
1613  *   Door-bell page containing the page data.
1614  *
1615  * @return
1616  *   Door-bell address offset on success, a negative error value otherwise.
1617  */
1618 int64_t
1619 mlx5_get_dbr(struct rte_eth_dev *dev, struct mlx5_devx_dbr_page **dbr_page)
1620 {
1621         struct mlx5_priv *priv = dev->data->dev_private;
1622         struct mlx5_devx_dbr_page *page = NULL;
1623         uint32_t i, j;
1624
1625         LIST_FOREACH(page, &priv->dbrpgs, next)
1626                 if (page->dbr_count < MLX5_DBR_PER_PAGE)
1627                         break;
1628         if (!page) { /* No page with free door-bell exists. */
1629                 page = mlx5_alloc_dbr_page(dev);
1630                 if (!page) /* Failed to allocate new page. */
1631                         return (-1);
1632                 LIST_INSERT_HEAD(&priv->dbrpgs, page, next);
1633         }
1634         /* Loop to find bitmap part with clear bit. */
1635         for (i = 0;
1636              i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
1637              i++)
1638                 ; /* Empty. */
1639         /* Find the first clear bit. */
1640         j = rte_bsf64(~page->dbr_bitmap[i]);
1641         assert(i < (MLX5_DBR_PER_PAGE / 64));
1642         page->dbr_bitmap[i] |= (1 << j);
1643         page->dbr_count++;
1644         *dbr_page = page;
1645         return (((i * 64) + j) * sizeof(uint64_t));
1646 }
1647
1648 /**
1649  * Release a door-bell record.
1650  *
1651  * @param [in] dev
1652  *   Pointer to Ethernet device.
1653  * @param [in] umem_id
1654  *   UMEM ID of page containing the door-bell record to release.
1655  * @param [in] offset
1656  *   Offset of door-bell record in page.
1657  *
1658  * @return
1659  *   0 on success, a negative error value otherwise.
1660  */
1661 int32_t
1662 mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id, uint64_t offset)
1663 {
1664         struct mlx5_priv *priv = dev->data->dev_private;
1665         struct mlx5_devx_dbr_page *page = NULL;
1666         int ret = 0;
1667
1668         LIST_FOREACH(page, &priv->dbrpgs, next)
1669                 /* Find the page this address belongs to. */
1670                 if (page->umem->umem_id == umem_id)
1671                         break;
1672         if (!page)
1673                 return -EINVAL;
1674         page->dbr_count--;
1675         if (!page->dbr_count) {
1676                 /* Page not used, free it and remove from list. */
1677                 LIST_REMOVE(page, next);
1678                 if (page->umem)
1679                         ret = -mlx5_glue->devx_umem_dereg(page->umem);
1680                 rte_free(page);
1681         } else {
1682                 /* Mark in bitmap that this door-bell is not in use. */
1683                 offset /= MLX5_DBR_SIZE;
1684                 int i = offset / 64;
1685                 int j = offset % 64;
1686
1687                 page->dbr_bitmap[i] &= ~(1 << j);
1688         }
1689         return ret;
1690 }
1691
1692 /**
1693  * Check sibling device configurations.
1694  *
1695  * Sibling devices sharing the Infiniband device context
1696  * should have compatible configurations. This regards
1697  * representors and bonding slaves.
1698  *
1699  * @param priv
1700  *   Private device descriptor.
1701  * @param config
1702  *   Configuration of the device is going to be created.
1703  *
1704  * @return
1705  *   0 on success, EINVAL otherwise
1706  */
1707 static int
1708 mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
1709                               struct mlx5_dev_config *config)
1710 {
1711         struct mlx5_ibv_shared *sh = priv->sh;
1712         struct mlx5_dev_config *sh_conf = NULL;
1713         uint16_t port_id;
1714
1715         assert(sh);
1716         /* Nothing to compare for the single/first device. */
1717         if (sh->refcnt == 1)
1718                 return 0;
1719         /* Find the device with shared context. */
1720         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
1721                 struct mlx5_priv *opriv =
1722                         rte_eth_devices[port_id].data->dev_private;
1723
1724                 if (opriv && opriv != priv && opriv->sh == sh) {
1725                         sh_conf = &opriv->config;
1726                         break;
1727                 }
1728         }
1729         if (!sh_conf)
1730                 return 0;
1731         if (sh_conf->dv_flow_en ^ config->dv_flow_en) {
1732                 DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch"
1733                              " for shared %s context", sh->ibdev_name);
1734                 rte_errno = EINVAL;
1735                 return rte_errno;
1736         }
1737         return 0;
1738 }
1739 /**
1740  * Spawn an Ethernet device from Verbs information.
1741  *
1742  * @param dpdk_dev
1743  *   Backing DPDK device.
1744  * @param spawn
1745  *   Verbs device parameters (name, port, switch_info) to spawn.
1746  * @param config
1747  *   Device configuration parameters.
1748  *
1749  * @return
1750  *   A valid Ethernet device object on success, NULL otherwise and rte_errno
1751  *   is set. The following errors are defined:
1752  *
1753  *   EBUSY: device is not supposed to be spawned.
1754  *   EEXIST: device is already spawned
1755  */
1756 static struct rte_eth_dev *
1757 mlx5_dev_spawn(struct rte_device *dpdk_dev,
1758                struct mlx5_dev_spawn_data *spawn,
1759                struct mlx5_dev_config config)
1760 {
1761         const struct mlx5_switch_info *switch_info = &spawn->info;
1762         struct mlx5_ibv_shared *sh = NULL;
1763         struct ibv_port_attr port_attr;
1764         struct mlx5dv_context dv_attr = { .comp_mask = 0 };
1765         struct rte_eth_dev *eth_dev = NULL;
1766         struct mlx5_priv *priv = NULL;
1767         int err = 0;
1768         unsigned int hw_padding = 0;
1769         unsigned int mps;
1770         unsigned int cqe_comp;
1771         unsigned int cqe_pad = 0;
1772         unsigned int tunnel_en = 0;
1773         unsigned int mpls_en = 0;
1774         unsigned int swp = 0;
1775         unsigned int mprq = 0;
1776         unsigned int mprq_min_stride_size_n = 0;
1777         unsigned int mprq_max_stride_size_n = 0;
1778         unsigned int mprq_min_stride_num_n = 0;
1779         unsigned int mprq_max_stride_num_n = 0;
1780         struct rte_ether_addr mac;
1781         char name[RTE_ETH_NAME_MAX_LEN];
1782         int own_domain_id = 0;
1783         uint16_t port_id;
1784         unsigned int i;
1785 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
1786         struct mlx5dv_devx_port devx_port;
1787 #endif
1788
1789         /* Determine if this port representor is supposed to be spawned. */
1790         if (switch_info->representor && dpdk_dev->devargs) {
1791                 struct rte_eth_devargs eth_da;
1792
1793                 err = rte_eth_devargs_parse(dpdk_dev->devargs->args, &eth_da);
1794                 if (err) {
1795                         rte_errno = -err;
1796                         DRV_LOG(ERR, "failed to process device arguments: %s",
1797                                 strerror(rte_errno));
1798                         return NULL;
1799                 }
1800                 for (i = 0; i < eth_da.nb_representor_ports; ++i)
1801                         if (eth_da.representor_ports[i] ==
1802                             (uint16_t)switch_info->port_name)
1803                                 break;
1804                 if (i == eth_da.nb_representor_ports) {
1805                         rte_errno = EBUSY;
1806                         return NULL;
1807                 }
1808         }
1809         /* Build device name. */
1810         if (spawn->pf_bond <  0) {
1811                 /* Single device. */
1812                 if (!switch_info->representor)
1813                         strlcpy(name, dpdk_dev->name, sizeof(name));
1814                 else
1815                         snprintf(name, sizeof(name), "%s_representor_%u",
1816                                  dpdk_dev->name, switch_info->port_name);
1817         } else {
1818                 /* Bonding device. */
1819                 if (!switch_info->representor)
1820                         snprintf(name, sizeof(name), "%s_%s",
1821                                  dpdk_dev->name, spawn->ibv_dev->name);
1822                 else
1823                         snprintf(name, sizeof(name), "%s_%s_representor_%u",
1824                                  dpdk_dev->name, spawn->ibv_dev->name,
1825                                  switch_info->port_name);
1826         }
1827         /* check if the device is already spawned */
1828         if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) {
1829                 rte_errno = EEXIST;
1830                 return NULL;
1831         }
1832         DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name);
1833         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1834                 eth_dev = rte_eth_dev_attach_secondary(name);
1835                 if (eth_dev == NULL) {
1836                         DRV_LOG(ERR, "can not attach rte ethdev");
1837                         rte_errno = ENOMEM;
1838                         return NULL;
1839                 }
1840                 eth_dev->device = dpdk_dev;
1841                 eth_dev->dev_ops = &mlx5_dev_sec_ops;
1842                 err = mlx5_proc_priv_init(eth_dev);
1843                 if (err)
1844                         return NULL;
1845                 /* Receive command fd from primary process */
1846                 err = mlx5_mp_req_verbs_cmd_fd(eth_dev);
1847                 if (err < 0)
1848                         return NULL;
1849                 /* Remap UAR for Tx queues. */
1850                 err = mlx5_tx_uar_init_secondary(eth_dev, err);
1851                 if (err)
1852                         return NULL;
1853                 /*
1854                  * Ethdev pointer is still required as input since
1855                  * the primary device is not accessible from the
1856                  * secondary process.
1857                  */
1858                 eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev);
1859                 eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev);
1860                 return eth_dev;
1861         }
1862         sh = mlx5_alloc_shared_ibctx(spawn);
1863         if (!sh)
1864                 return NULL;
1865         config.devx = sh->devx;
1866 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
1867         config.dest_tir = 1;
1868 #endif
1869 #ifdef HAVE_IBV_MLX5_MOD_SWP
1870         dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
1871 #endif
1872         /*
1873          * Multi-packet send is supported by ConnectX-4 Lx PF as well
1874          * as all ConnectX-5 devices.
1875          */
1876 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1877         dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
1878 #endif
1879 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
1880         dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
1881 #endif
1882         mlx5_glue->dv_query_device(sh->ctx, &dv_attr);
1883         if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
1884                 if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
1885                         DRV_LOG(DEBUG, "enhanced MPW is supported");
1886                         mps = MLX5_MPW_ENHANCED;
1887                 } else {
1888                         DRV_LOG(DEBUG, "MPW is supported");
1889                         mps = MLX5_MPW;
1890                 }
1891         } else {
1892                 DRV_LOG(DEBUG, "MPW isn't supported");
1893                 mps = MLX5_MPW_DISABLED;
1894         }
1895 #ifdef HAVE_IBV_MLX5_MOD_SWP
1896         if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
1897                 swp = dv_attr.sw_parsing_caps.sw_parsing_offloads;
1898         DRV_LOG(DEBUG, "SWP support: %u", swp);
1899 #endif
1900         config.swp = !!swp;
1901 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
1902         if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
1903                 struct mlx5dv_striding_rq_caps mprq_caps =
1904                         dv_attr.striding_rq_caps;
1905
1906                 DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d",
1907                         mprq_caps.min_single_stride_log_num_of_bytes);
1908                 DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %d",
1909                         mprq_caps.max_single_stride_log_num_of_bytes);
1910                 DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %d",
1911                         mprq_caps.min_single_wqe_log_num_of_strides);
1912                 DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %d",
1913                         mprq_caps.max_single_wqe_log_num_of_strides);
1914                 DRV_LOG(DEBUG, "\tsupported_qpts: %d",
1915                         mprq_caps.supported_qpts);
1916                 DRV_LOG(DEBUG, "device supports Multi-Packet RQ");
1917                 mprq = 1;
1918                 mprq_min_stride_size_n =
1919                         mprq_caps.min_single_stride_log_num_of_bytes;
1920                 mprq_max_stride_size_n =
1921                         mprq_caps.max_single_stride_log_num_of_bytes;
1922                 mprq_min_stride_num_n =
1923                         mprq_caps.min_single_wqe_log_num_of_strides;
1924                 mprq_max_stride_num_n =
1925                         mprq_caps.max_single_wqe_log_num_of_strides;
1926                 config.mprq.stride_num_n = RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
1927                                                    mprq_min_stride_num_n);
1928         }
1929 #endif
1930         if (RTE_CACHE_LINE_SIZE == 128 &&
1931             !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
1932                 cqe_comp = 0;
1933         else
1934                 cqe_comp = 1;
1935         config.cqe_comp = cqe_comp;
1936 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
1937         /* Whether device supports 128B Rx CQE padding. */
1938         cqe_pad = RTE_CACHE_LINE_SIZE == 128 &&
1939                   (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD);
1940 #endif
1941 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1942         if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
1943                 tunnel_en = ((dv_attr.tunnel_offloads_caps &
1944                               MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) &&
1945                              (dv_attr.tunnel_offloads_caps &
1946                               MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE));
1947         }
1948         DRV_LOG(DEBUG, "tunnel offloading is %ssupported",
1949                 tunnel_en ? "" : "not ");
1950 #else
1951         DRV_LOG(WARNING,
1952                 "tunnel offloading disabled due to old OFED/rdma-core version");
1953 #endif
1954         config.tunnel_en = tunnel_en;
1955 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1956         mpls_en = ((dv_attr.tunnel_offloads_caps &
1957                     MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) &&
1958                    (dv_attr.tunnel_offloads_caps &
1959                     MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP));
1960         DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported",
1961                 mpls_en ? "" : "not ");
1962 #else
1963         DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to"
1964                 " old OFED/rdma-core version or firmware configuration");
1965 #endif
1966         config.mpls_en = mpls_en;
1967         /* Check port status. */
1968         err = mlx5_glue->query_port(sh->ctx, spawn->ibv_port, &port_attr);
1969         if (err) {
1970                 DRV_LOG(ERR, "port query failed: %s", strerror(err));
1971                 goto error;
1972         }
1973         if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
1974                 DRV_LOG(ERR, "port is not configured in Ethernet mode");
1975                 err = EINVAL;
1976                 goto error;
1977         }
1978         if (port_attr.state != IBV_PORT_ACTIVE)
1979                 DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)",
1980                         mlx5_glue->port_state_str(port_attr.state),
1981                         port_attr.state);
1982         /* Allocate private eth device data. */
1983         priv = rte_zmalloc("ethdev private structure",
1984                            sizeof(*priv),
1985                            RTE_CACHE_LINE_SIZE);
1986         if (priv == NULL) {
1987                 DRV_LOG(ERR, "priv allocation failure");
1988                 err = ENOMEM;
1989                 goto error;
1990         }
1991         priv->sh = sh;
1992         priv->ibv_port = spawn->ibv_port;
1993         priv->pci_dev = spawn->pci_dev;
1994         priv->mtu = RTE_ETHER_MTU;
1995 #ifndef RTE_ARCH_64
1996         /* Initialize UAR access locks for 32bit implementations. */
1997         rte_spinlock_init(&priv->uar_lock_cq);
1998         for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++)
1999                 rte_spinlock_init(&priv->uar_lock[i]);
2000 #endif
2001         /* Some internal functions rely on Netlink sockets, open them now. */
2002         priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA);
2003         priv->nl_socket_route = mlx5_nl_init(NETLINK_ROUTE);
2004         priv->nl_sn = 0;
2005         priv->representor = !!switch_info->representor;
2006         priv->master = !!switch_info->master;
2007         priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
2008         priv->vport_meta_tag = 0;
2009         priv->vport_meta_mask = 0;
2010         priv->pf_bond = spawn->pf_bond;
2011 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
2012         /*
2013          * The DevX port query API is implemented. E-Switch may use
2014          * either vport or reg_c[0] metadata register to match on
2015          * vport index. The engaged part of metadata register is
2016          * defined by mask.
2017          */
2018         devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT |
2019                               MLX5DV_DEVX_PORT_MATCH_REG_C_0;
2020         err = mlx5_glue->devx_port_query(sh->ctx, spawn->ibv_port, &devx_port);
2021         if (err) {
2022                 DRV_LOG(WARNING, "can't query devx port %d on device %s",
2023                         spawn->ibv_port, spawn->ibv_dev->name);
2024                 devx_port.comp_mask = 0;
2025         }
2026         if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) {
2027                 priv->vport_meta_tag = devx_port.reg_c_0.value;
2028                 priv->vport_meta_mask = devx_port.reg_c_0.mask;
2029                 if (!priv->vport_meta_mask) {
2030                         DRV_LOG(ERR, "vport zero mask for port %d"
2031                                      " on bonding device %s",
2032                                      spawn->ibv_port, spawn->ibv_dev->name);
2033                         err = ENOTSUP;
2034                         goto error;
2035                 }
2036                 if (priv->vport_meta_tag & ~priv->vport_meta_mask) {
2037                         DRV_LOG(ERR, "invalid vport tag for port %d"
2038                                      " on bonding device %s",
2039                                      spawn->ibv_port, spawn->ibv_dev->name);
2040                         err = ENOTSUP;
2041                         goto error;
2042                 }
2043         } else if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) {
2044                 priv->vport_id = devx_port.vport_num;
2045         } else if (spawn->pf_bond >= 0) {
2046                 DRV_LOG(ERR, "can't deduce vport index for port %d"
2047                              " on bonding device %s",
2048                              spawn->ibv_port, spawn->ibv_dev->name);
2049                 err = ENOTSUP;
2050                 goto error;
2051         } else {
2052                 /* Suppose vport index in compatible way. */
2053                 priv->vport_id = switch_info->representor ?
2054                                  switch_info->port_name + 1 : -1;
2055         }
2056 #else
2057         /*
2058          * Kernel/rdma_core support single E-Switch per PF configurations
2059          * only and vport_id field contains the vport index for
2060          * associated VF, which is deduced from representor port name.
2061          * For example, let's have the IB device port 10, it has
2062          * attached network device eth0, which has port name attribute
2063          * pf0vf2, we can deduce the VF number as 2, and set vport index
2064          * as 3 (2+1). This assigning schema should be changed if the
2065          * multiple E-Switch instances per PF configurations or/and PCI
2066          * subfunctions are added.
2067          */
2068         priv->vport_id = switch_info->representor ?
2069                          switch_info->port_name + 1 : -1;
2070 #endif
2071         /* representor_id field keeps the unmodified VF index. */
2072         priv->representor_id = switch_info->representor ?
2073                                switch_info->port_name : -1;
2074         /*
2075          * Look for sibling devices in order to reuse their switch domain
2076          * if any, otherwise allocate one.
2077          */
2078         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
2079                 const struct mlx5_priv *opriv =
2080                         rte_eth_devices[port_id].data->dev_private;
2081
2082                 if (!opriv ||
2083                     opriv->sh != priv->sh ||
2084                         opriv->domain_id ==
2085                         RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
2086                         continue;
2087                 priv->domain_id = opriv->domain_id;
2088                 break;
2089         }
2090         if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
2091                 err = rte_eth_switch_domain_alloc(&priv->domain_id);
2092                 if (err) {
2093                         err = rte_errno;
2094                         DRV_LOG(ERR, "unable to allocate switch domain: %s",
2095                                 strerror(rte_errno));
2096                         goto error;
2097                 }
2098                 own_domain_id = 1;
2099         }
2100         err = mlx5_args(&config, dpdk_dev->devargs);
2101         if (err) {
2102                 err = rte_errno;
2103                 DRV_LOG(ERR, "failed to process device arguments: %s",
2104                         strerror(rte_errno));
2105                 goto error;
2106         }
2107         err = mlx5_dev_check_sibling_config(priv, &config);
2108         if (err)
2109                 goto error;
2110         config.hw_csum = !!(sh->device_attr.device_cap_flags_ex &
2111                             IBV_DEVICE_RAW_IP_CSUM);
2112         DRV_LOG(DEBUG, "checksum offloading is %ssupported",
2113                 (config.hw_csum ? "" : "not "));
2114 #if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
2115         !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
2116         DRV_LOG(DEBUG, "counters are not supported");
2117 #endif
2118 #ifndef HAVE_IBV_FLOW_DV_SUPPORT
2119         if (config.dv_flow_en) {
2120                 DRV_LOG(WARNING, "DV flow is not supported");
2121                 config.dv_flow_en = 0;
2122         }
2123 #endif
2124         config.ind_table_max_size =
2125                 sh->device_attr.rss_caps.max_rwq_indirection_table_size;
2126         /*
2127          * Remove this check once DPDK supports larger/variable
2128          * indirection tables.
2129          */
2130         if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512)
2131                 config.ind_table_max_size = ETH_RSS_RETA_SIZE_512;
2132         DRV_LOG(DEBUG, "maximum Rx indirection table size is %u",
2133                 config.ind_table_max_size);
2134         config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps &
2135                                   IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
2136         DRV_LOG(DEBUG, "VLAN stripping is %ssupported",
2137                 (config.hw_vlan_strip ? "" : "not "));
2138         config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps &
2139                                  IBV_RAW_PACKET_CAP_SCATTER_FCS);
2140         DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported",
2141                 (config.hw_fcs_strip ? "" : "not "));
2142 #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
2143         hw_padding = !!sh->device_attr.rx_pad_end_addr_align;
2144 #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
2145         hw_padding = !!(sh->device_attr.device_cap_flags_ex &
2146                         IBV_DEVICE_PCI_WRITE_END_PADDING);
2147 #endif
2148         if (config.hw_padding && !hw_padding) {
2149                 DRV_LOG(DEBUG, "Rx end alignment padding isn't supported");
2150                 config.hw_padding = 0;
2151         } else if (config.hw_padding) {
2152                 DRV_LOG(DEBUG, "Rx end alignment padding is enabled");
2153         }
2154         config.tso = (sh->device_attr.tso_caps.max_tso > 0 &&
2155                       (sh->device_attr.tso_caps.supported_qpts &
2156                        (1 << IBV_QPT_RAW_PACKET)));
2157         if (config.tso)
2158                 config.tso_max_payload_sz = sh->device_attr.tso_caps.max_tso;
2159         /*
2160          * MPW is disabled by default, while the Enhanced MPW is enabled
2161          * by default.
2162          */
2163         if (config.mps == MLX5_ARG_UNSET)
2164                 config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED :
2165                                                           MLX5_MPW_DISABLED;
2166         else
2167                 config.mps = config.mps ? mps : MLX5_MPW_DISABLED;
2168         DRV_LOG(INFO, "%sMPS is %s",
2169                 config.mps == MLX5_MPW_ENHANCED ? "enhanced " : "",
2170                 config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
2171         if (config.cqe_comp && !cqe_comp) {
2172                 DRV_LOG(WARNING, "Rx CQE compression isn't supported");
2173                 config.cqe_comp = 0;
2174         }
2175         if (config.cqe_pad && !cqe_pad) {
2176                 DRV_LOG(WARNING, "Rx CQE padding isn't supported");
2177                 config.cqe_pad = 0;
2178         } else if (config.cqe_pad) {
2179                 DRV_LOG(INFO, "Rx CQE padding is enabled");
2180         }
2181         if (config.devx) {
2182                 priv->counter_fallback = 0;
2183                 err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr);
2184                 if (err) {
2185                         err = -err;
2186                         goto error;
2187                 }
2188                 if (!config.hca_attr.flow_counters_dump)
2189                         priv->counter_fallback = 1;
2190 #ifndef HAVE_IBV_DEVX_ASYNC
2191                 priv->counter_fallback = 1;
2192 #endif
2193                 if (priv->counter_fallback)
2194                         DRV_LOG(INFO, "Use fall-back DV counter management");
2195                 /* Check for LRO support. */
2196                 if (config.dest_tir && config.hca_attr.lro_cap &&
2197                     config.dv_flow_en) {
2198                         /* TBD check tunnel lro caps. */
2199                         config.lro.supported = config.hca_attr.lro_cap;
2200                         DRV_LOG(DEBUG, "Device supports LRO");
2201                         /*
2202                          * If LRO timeout is not configured by application,
2203                          * use the minimal supported value.
2204                          */
2205                         if (!config.lro.timeout)
2206                                 config.lro.timeout =
2207                                 config.hca_attr.lro_timer_supported_periods[0];
2208                         DRV_LOG(DEBUG, "LRO session timeout set to %d usec",
2209                                 config.lro.timeout);
2210                 }
2211         }
2212         if (config.mprq.enabled && mprq) {
2213                 if (config.mprq.stride_num_n > mprq_max_stride_num_n ||
2214                     config.mprq.stride_num_n < mprq_min_stride_num_n) {
2215                         config.mprq.stride_num_n =
2216                                 RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
2217                                         mprq_min_stride_num_n);
2218                         DRV_LOG(WARNING,
2219                                 "the number of strides"
2220                                 " for Multi-Packet RQ is out of range,"
2221                                 " setting default value (%u)",
2222                                 1 << config.mprq.stride_num_n);
2223                 }
2224                 config.mprq.min_stride_size_n = mprq_min_stride_size_n;
2225                 config.mprq.max_stride_size_n = mprq_max_stride_size_n;
2226         } else if (config.mprq.enabled && !mprq) {
2227                 DRV_LOG(WARNING, "Multi-Packet RQ isn't supported");
2228                 config.mprq.enabled = 0;
2229         }
2230         if (config.max_dump_files_num == 0)
2231                 config.max_dump_files_num = 128;
2232         eth_dev = rte_eth_dev_allocate(name);
2233         if (eth_dev == NULL) {
2234                 DRV_LOG(ERR, "can not allocate rte ethdev");
2235                 err = ENOMEM;
2236                 goto error;
2237         }
2238         /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */
2239         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
2240         if (priv->representor) {
2241                 eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
2242                 eth_dev->data->representor_id = priv->representor_id;
2243         }
2244         /*
2245          * Store associated network device interface index. This index
2246          * is permanent throughout the lifetime of device. So, we may store
2247          * the ifindex here and use the cached value further.
2248          */
2249         assert(spawn->ifindex);
2250         priv->if_index = spawn->ifindex;
2251         eth_dev->data->dev_private = priv;
2252         priv->dev_data = eth_dev->data;
2253         eth_dev->data->mac_addrs = priv->mac;
2254         eth_dev->device = dpdk_dev;
2255         /* Configure the first MAC address by default. */
2256         if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
2257                 DRV_LOG(ERR,
2258                         "port %u cannot get MAC address, is mlx5_en"
2259                         " loaded? (errno: %s)",
2260                         eth_dev->data->port_id, strerror(rte_errno));
2261                 err = ENODEV;
2262                 goto error;
2263         }
2264         DRV_LOG(INFO,
2265                 "port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
2266                 eth_dev->data->port_id,
2267                 mac.addr_bytes[0], mac.addr_bytes[1],
2268                 mac.addr_bytes[2], mac.addr_bytes[3],
2269                 mac.addr_bytes[4], mac.addr_bytes[5]);
2270 #ifndef NDEBUG
2271         {
2272                 char ifname[IF_NAMESIZE];
2273
2274                 if (mlx5_get_ifname(eth_dev, &ifname) == 0)
2275                         DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
2276                                 eth_dev->data->port_id, ifname);
2277                 else
2278                         DRV_LOG(DEBUG, "port %u ifname is unknown",
2279                                 eth_dev->data->port_id);
2280         }
2281 #endif
2282         /* Get actual MTU if possible. */
2283         err = mlx5_get_mtu(eth_dev, &priv->mtu);
2284         if (err) {
2285                 err = rte_errno;
2286                 goto error;
2287         }
2288         DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
2289                 priv->mtu);
2290         /* Initialize burst functions to prevent crashes before link-up. */
2291         eth_dev->rx_pkt_burst = removed_rx_burst;
2292         eth_dev->tx_pkt_burst = removed_tx_burst;
2293         eth_dev->dev_ops = &mlx5_dev_ops;
2294         /* Register MAC address. */
2295         claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
2296         if (config.vf && config.vf_nl_en)
2297                 mlx5_nl_mac_addr_sync(eth_dev);
2298         TAILQ_INIT(&priv->flows);
2299         TAILQ_INIT(&priv->ctrl_flows);
2300         /* Hint libmlx5 to use PMD allocator for data plane resources */
2301         struct mlx5dv_ctx_allocators alctr = {
2302                 .alloc = &mlx5_alloc_verbs_buf,
2303                 .free = &mlx5_free_verbs_buf,
2304                 .data = priv,
2305         };
2306         mlx5_glue->dv_set_context_attr(sh->ctx,
2307                                        MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
2308                                        (void *)((uintptr_t)&alctr));
2309         /* Bring Ethernet device up. */
2310         DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
2311                 eth_dev->data->port_id);
2312         mlx5_set_link_up(eth_dev);
2313         /*
2314          * Even though the interrupt handler is not installed yet,
2315          * interrupts will still trigger on the async_fd from
2316          * Verbs context returned by ibv_open_device().
2317          */
2318         mlx5_link_update(eth_dev, 0);
2319 #ifdef HAVE_MLX5DV_DR_ESWITCH
2320         if (!(config.hca_attr.eswitch_manager && config.dv_flow_en &&
2321               (switch_info->representor || switch_info->master)))
2322                 config.dv_esw_en = 0;
2323 #else
2324         config.dv_esw_en = 0;
2325 #endif
2326         /* Detect minimal data bytes to inline. */
2327         mlx5_set_min_inline(spawn, &config);
2328         /* Store device configuration on private structure. */
2329         priv->config = config;
2330         /* Create context for virtual machine VLAN workaround. */
2331         priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
2332         if (config.dv_flow_en) {
2333                 err = mlx5_alloc_shared_dr(priv);
2334                 if (err)
2335                         goto error;
2336         }
2337         /* Supported Verbs flow priority number detection. */
2338         err = mlx5_flow_discover_priorities(eth_dev);
2339         if (err < 0) {
2340                 err = -err;
2341                 goto error;
2342         }
2343         priv->config.flow_prio = err;
2344         return eth_dev;
2345 error:
2346         if (priv) {
2347                 if (priv->sh)
2348                         mlx5_free_shared_dr(priv);
2349                 if (priv->nl_socket_route >= 0)
2350                         close(priv->nl_socket_route);
2351                 if (priv->nl_socket_rdma >= 0)
2352                         close(priv->nl_socket_rdma);
2353                 if (priv->vmwa_context)
2354                         mlx5_vlan_vmwa_exit(priv->vmwa_context);
2355                 if (own_domain_id)
2356                         claim_zero(rte_eth_switch_domain_free(priv->domain_id));
2357                 rte_free(priv);
2358                 if (eth_dev != NULL)
2359                         eth_dev->data->dev_private = NULL;
2360         }
2361         if (eth_dev != NULL) {
2362                 /* mac_addrs must not be freed alone because part of dev_private */
2363                 eth_dev->data->mac_addrs = NULL;
2364                 rte_eth_dev_release_port(eth_dev);
2365         }
2366         if (sh)
2367                 mlx5_free_shared_ibctx(sh);
2368         assert(err > 0);
2369         rte_errno = err;
2370         return NULL;
2371 }
2372
2373 /**
2374  * Comparison callback to sort device data.
2375  *
2376  * This is meant to be used with qsort().
2377  *
2378  * @param a[in]
2379  *   Pointer to pointer to first data object.
2380  * @param b[in]
2381  *   Pointer to pointer to second data object.
2382  *
2383  * @return
2384  *   0 if both objects are equal, less than 0 if the first argument is less
2385  *   than the second, greater than 0 otherwise.
2386  */
2387 static int
2388 mlx5_dev_spawn_data_cmp(const void *a, const void *b)
2389 {
2390         const struct mlx5_switch_info *si_a =
2391                 &((const struct mlx5_dev_spawn_data *)a)->info;
2392         const struct mlx5_switch_info *si_b =
2393                 &((const struct mlx5_dev_spawn_data *)b)->info;
2394         int ret;
2395
2396         /* Master device first. */
2397         ret = si_b->master - si_a->master;
2398         if (ret)
2399                 return ret;
2400         /* Then representor devices. */
2401         ret = si_b->representor - si_a->representor;
2402         if (ret)
2403                 return ret;
2404         /* Unidentified devices come last in no specific order. */
2405         if (!si_a->representor)
2406                 return 0;
2407         /* Order representors by name. */
2408         return si_a->port_name - si_b->port_name;
2409 }
2410
2411 /**
2412  * Match PCI information for possible slaves of bonding device.
2413  *
2414  * @param[in] ibv_dev
2415  *   Pointer to Infiniband device structure.
2416  * @param[in] pci_dev
2417  *   Pointer to PCI device structure to match PCI address.
2418  * @param[in] nl_rdma
2419  *   Netlink RDMA group socket handle.
2420  *
2421  * @return
2422  *   negative value if no bonding device found, otherwise
2423  *   positive index of slave PF in bonding.
2424  */
2425 static int
2426 mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev,
2427                            const struct rte_pci_device *pci_dev,
2428                            int nl_rdma)
2429 {
2430         char ifname[IF_NAMESIZE + 1];
2431         unsigned int ifindex;
2432         unsigned int np, i;
2433         FILE *file = NULL;
2434         int pf = -1;
2435
2436         /*
2437          * Try to get master device name. If something goes
2438          * wrong suppose the lack of kernel support and no
2439          * bonding devices.
2440          */
2441         if (nl_rdma < 0)
2442                 return -1;
2443         if (!strstr(ibv_dev->name, "bond"))
2444                 return -1;
2445         np = mlx5_nl_portnum(nl_rdma, ibv_dev->name);
2446         if (!np)
2447                 return -1;
2448         /*
2449          * The Master device might not be on the predefined
2450          * port (not on port index 1, it is not garanted),
2451          * we have to scan all Infiniband device port and
2452          * find master.
2453          */
2454         for (i = 1; i <= np; ++i) {
2455                 /* Check whether Infiniband port is populated. */
2456                 ifindex = mlx5_nl_ifindex(nl_rdma, ibv_dev->name, i);
2457                 if (!ifindex)
2458                         continue;
2459                 if (!if_indextoname(ifindex, ifname))
2460                         continue;
2461                 /* Try to read bonding slave names from sysfs. */
2462                 MKSTR(slaves,
2463                       "/sys/class/net/%s/master/bonding/slaves", ifname);
2464                 file = fopen(slaves, "r");
2465                 if (file)
2466                         break;
2467         }
2468         if (!file)
2469                 return -1;
2470         /* Use safe format to check maximal buffer length. */
2471         assert(atol(RTE_STR(IF_NAMESIZE)) == IF_NAMESIZE);
2472         while (fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", ifname) == 1) {
2473                 char tmp_str[IF_NAMESIZE + 32];
2474                 struct rte_pci_addr pci_addr;
2475                 struct mlx5_switch_info info;
2476
2477                 /* Process slave interface names in the loop. */
2478                 snprintf(tmp_str, sizeof(tmp_str),
2479                          "/sys/class/net/%s", ifname);
2480                 if (mlx5_dev_to_pci_addr(tmp_str, &pci_addr)) {
2481                         DRV_LOG(WARNING, "can not get PCI address"
2482                                          " for netdev \"%s\"", ifname);
2483                         continue;
2484                 }
2485                 if (pci_dev->addr.domain != pci_addr.domain ||
2486                     pci_dev->addr.bus != pci_addr.bus ||
2487                     pci_dev->addr.devid != pci_addr.devid ||
2488                     pci_dev->addr.function != pci_addr.function)
2489                         continue;
2490                 /* Slave interface PCI address match found. */
2491                 fclose(file);
2492                 snprintf(tmp_str, sizeof(tmp_str),
2493                          "/sys/class/net/%s/phys_port_name", ifname);
2494                 file = fopen(tmp_str, "rb");
2495                 if (!file)
2496                         break;
2497                 info.name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET;
2498                 if (fscanf(file, "%32s", tmp_str) == 1)
2499                         mlx5_translate_port_name(tmp_str, &info);
2500                 if (info.name_type == MLX5_PHYS_PORT_NAME_TYPE_LEGACY ||
2501                     info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK)
2502                         pf = info.port_name;
2503                 break;
2504         }
2505         if (file)
2506                 fclose(file);
2507         return pf;
2508 }
2509
2510 /**
2511  * DPDK callback to register a PCI device.
2512  *
2513  * This function spawns Ethernet devices out of a given PCI device.
2514  *
2515  * @param[in] pci_drv
2516  *   PCI driver structure (mlx5_driver).
2517  * @param[in] pci_dev
2518  *   PCI device information.
2519  *
2520  * @return
2521  *   0 on success, a negative errno value otherwise and rte_errno is set.
2522  */
2523 static int
2524 mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2525                struct rte_pci_device *pci_dev)
2526 {
2527         struct ibv_device **ibv_list;
2528         /*
2529          * Number of found IB Devices matching with requested PCI BDF.
2530          * nd != 1 means there are multiple IB devices over the same
2531          * PCI device and we have representors and master.
2532          */
2533         unsigned int nd = 0;
2534         /*
2535          * Number of found IB device Ports. nd = 1 and np = 1..n means
2536          * we have the single multiport IB device, and there may be
2537          * representors attached to some of found ports.
2538          */
2539         unsigned int np = 0;
2540         /*
2541          * Number of DPDK ethernet devices to Spawn - either over
2542          * multiple IB devices or multiple ports of single IB device.
2543          * Actually this is the number of iterations to spawn.
2544          */
2545         unsigned int ns = 0;
2546         /*
2547          * Bonding device
2548          *   < 0 - no bonding device (single one)
2549          *  >= 0 - bonding device (value is slave PF index)
2550          */
2551         int bd = -1;
2552         struct mlx5_dev_spawn_data *list = NULL;
2553         struct mlx5_dev_config dev_config;
2554         int ret;
2555
2556         ret = mlx5_init_once();
2557         if (ret) {
2558                 DRV_LOG(ERR, "unable to init PMD global data: %s",
2559                         strerror(rte_errno));
2560                 return -rte_errno;
2561         }
2562         assert(pci_drv == &mlx5_driver);
2563         errno = 0;
2564         ibv_list = mlx5_glue->get_device_list(&ret);
2565         if (!ibv_list) {
2566                 rte_errno = errno ? errno : ENOSYS;
2567                 DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?");
2568                 return -rte_errno;
2569         }
2570         /*
2571          * First scan the list of all Infiniband devices to find
2572          * matching ones, gathering into the list.
2573          */
2574         struct ibv_device *ibv_match[ret + 1];
2575         int nl_route = mlx5_nl_init(NETLINK_ROUTE);
2576         int nl_rdma = mlx5_nl_init(NETLINK_RDMA);
2577         unsigned int i;
2578
2579         while (ret-- > 0) {
2580                 struct rte_pci_addr pci_addr;
2581
2582                 DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name);
2583                 bd = mlx5_device_bond_pci_match
2584                                 (ibv_list[ret], pci_dev, nl_rdma);
2585                 if (bd >= 0) {
2586                         /*
2587                          * Bonding device detected. Only one match is allowed,
2588                          * the bonding is supported over multi-port IB device,
2589                          * there should be no matches on representor PCI
2590                          * functions or non VF LAG bonding devices with
2591                          * specified address.
2592                          */
2593                         if (nd) {
2594                                 DRV_LOG(ERR,
2595                                         "multiple PCI match on bonding device"
2596                                         "\"%s\" found", ibv_list[ret]->name);
2597                                 rte_errno = ENOENT;
2598                                 ret = -rte_errno;
2599                                 goto exit;
2600                         }
2601                         DRV_LOG(INFO, "PCI information matches for"
2602                                       " slave %d bonding device \"%s\"",
2603                                       bd, ibv_list[ret]->name);
2604                         ibv_match[nd++] = ibv_list[ret];
2605                         break;
2606                 }
2607                 if (mlx5_dev_to_pci_addr
2608                         (ibv_list[ret]->ibdev_path, &pci_addr))
2609                         continue;
2610                 if (pci_dev->addr.domain != pci_addr.domain ||
2611                     pci_dev->addr.bus != pci_addr.bus ||
2612                     pci_dev->addr.devid != pci_addr.devid ||
2613                     pci_dev->addr.function != pci_addr.function)
2614                         continue;
2615                 DRV_LOG(INFO, "PCI information matches for device \"%s\"",
2616                         ibv_list[ret]->name);
2617                 ibv_match[nd++] = ibv_list[ret];
2618         }
2619         ibv_match[nd] = NULL;
2620         if (!nd) {
2621                 /* No device matches, just complain and bail out. */
2622                 DRV_LOG(WARNING,
2623                         "no Verbs device matches PCI device " PCI_PRI_FMT ","
2624                         " are kernel drivers loaded?",
2625                         pci_dev->addr.domain, pci_dev->addr.bus,
2626                         pci_dev->addr.devid, pci_dev->addr.function);
2627                 rte_errno = ENOENT;
2628                 ret = -rte_errno;
2629                 goto exit;
2630         }
2631         if (nd == 1) {
2632                 /*
2633                  * Found single matching device may have multiple ports.
2634                  * Each port may be representor, we have to check the port
2635                  * number and check the representors existence.
2636                  */
2637                 if (nl_rdma >= 0)
2638                         np = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name);
2639                 if (!np)
2640                         DRV_LOG(WARNING, "can not get IB device \"%s\""
2641                                          " ports number", ibv_match[0]->name);
2642                 if (bd >= 0 && !np) {
2643                         DRV_LOG(ERR, "can not get ports"
2644                                      " for bonding device");
2645                         rte_errno = ENOENT;
2646                         ret = -rte_errno;
2647                         goto exit;
2648                 }
2649         }
2650 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
2651         if (bd >= 0) {
2652                 /*
2653                  * This may happen if there is VF LAG kernel support and
2654                  * application is compiled with older rdma_core library.
2655                  */
2656                 DRV_LOG(ERR,
2657                         "No kernel/verbs support for VF LAG bonding found.");
2658                 rte_errno = ENOTSUP;
2659                 ret = -rte_errno;
2660                 goto exit;
2661         }
2662 #endif
2663         /*
2664          * Now we can determine the maximal
2665          * amount of devices to be spawned.
2666          */
2667         list = rte_zmalloc("device spawn data",
2668                          sizeof(struct mlx5_dev_spawn_data) *
2669                          (np ? np : nd),
2670                          RTE_CACHE_LINE_SIZE);
2671         if (!list) {
2672                 DRV_LOG(ERR, "spawn data array allocation failure");
2673                 rte_errno = ENOMEM;
2674                 ret = -rte_errno;
2675                 goto exit;
2676         }
2677         if (bd >= 0 || np > 1) {
2678                 /*
2679                  * Single IB device with multiple ports found,
2680                  * it may be E-Switch master device and representors.
2681                  * We have to perform identification trough the ports.
2682                  */
2683                 assert(nl_rdma >= 0);
2684                 assert(ns == 0);
2685                 assert(nd == 1);
2686                 assert(np);
2687                 for (i = 1; i <= np; ++i) {
2688                         list[ns].max_port = np;
2689                         list[ns].ibv_port = i;
2690                         list[ns].ibv_dev = ibv_match[0];
2691                         list[ns].eth_dev = NULL;
2692                         list[ns].pci_dev = pci_dev;
2693                         list[ns].pf_bond = bd;
2694                         list[ns].ifindex = mlx5_nl_ifindex
2695                                         (nl_rdma, list[ns].ibv_dev->name, i);
2696                         if (!list[ns].ifindex) {
2697                                 /*
2698                                  * No network interface index found for the
2699                                  * specified port, it means there is no
2700                                  * representor on this port. It's OK,
2701                                  * there can be disabled ports, for example
2702                                  * if sriov_numvfs < sriov_totalvfs.
2703                                  */
2704                                 continue;
2705                         }
2706                         ret = -1;
2707                         if (nl_route >= 0)
2708                                 ret = mlx5_nl_switch_info
2709                                                (nl_route,
2710                                                 list[ns].ifindex,
2711                                                 &list[ns].info);
2712                         if (ret || (!list[ns].info.representor &&
2713                                     !list[ns].info.master)) {
2714                                 /*
2715                                  * We failed to recognize representors with
2716                                  * Netlink, let's try to perform the task
2717                                  * with sysfs.
2718                                  */
2719                                 ret =  mlx5_sysfs_switch_info
2720                                                 (list[ns].ifindex,
2721                                                  &list[ns].info);
2722                         }
2723                         if (!ret && bd >= 0) {
2724                                 switch (list[ns].info.name_type) {
2725                                 case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
2726                                         if (list[ns].info.port_name == bd)
2727                                                 ns++;
2728                                         break;
2729                                 case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
2730                                         if (list[ns].info.pf_num == bd)
2731                                                 ns++;
2732                                         break;
2733                                 default:
2734                                         break;
2735                                 }
2736                                 continue;
2737                         }
2738                         if (!ret && (list[ns].info.representor ^
2739                                      list[ns].info.master))
2740                                 ns++;
2741                 }
2742                 if (!ns) {
2743                         DRV_LOG(ERR,
2744                                 "unable to recognize master/representors"
2745                                 " on the IB device with multiple ports");
2746                         rte_errno = ENOENT;
2747                         ret = -rte_errno;
2748                         goto exit;
2749                 }
2750         } else {
2751                 /*
2752                  * The existence of several matching entries (nd > 1) means
2753                  * port representors have been instantiated. No existing Verbs
2754                  * call nor sysfs entries can tell them apart, this can only
2755                  * be done through Netlink calls assuming kernel drivers are
2756                  * recent enough to support them.
2757                  *
2758                  * In the event of identification failure through Netlink,
2759                  * try again through sysfs, then:
2760                  *
2761                  * 1. A single IB device matches (nd == 1) with single
2762                  *    port (np=0/1) and is not a representor, assume
2763                  *    no switch support.
2764                  *
2765                  * 2. Otherwise no safe assumptions can be made;
2766                  *    complain louder and bail out.
2767                  */
2768                 np = 1;
2769                 for (i = 0; i != nd; ++i) {
2770                         memset(&list[ns].info, 0, sizeof(list[ns].info));
2771                         list[ns].max_port = 1;
2772                         list[ns].ibv_port = 1;
2773                         list[ns].ibv_dev = ibv_match[i];
2774                         list[ns].eth_dev = NULL;
2775                         list[ns].pci_dev = pci_dev;
2776                         list[ns].pf_bond = -1;
2777                         list[ns].ifindex = 0;
2778                         if (nl_rdma >= 0)
2779                                 list[ns].ifindex = mlx5_nl_ifindex
2780                                         (nl_rdma, list[ns].ibv_dev->name, 1);
2781                         if (!list[ns].ifindex) {
2782                                 char ifname[IF_NAMESIZE];
2783
2784                                 /*
2785                                  * Netlink failed, it may happen with old
2786                                  * ib_core kernel driver (before 4.16).
2787                                  * We can assume there is old driver because
2788                                  * here we are processing single ports IB
2789                                  * devices. Let's try sysfs to retrieve
2790                                  * the ifindex. The method works for
2791                                  * master device only.
2792                                  */
2793                                 if (nd > 1) {
2794                                         /*
2795                                          * Multiple devices found, assume
2796                                          * representors, can not distinguish
2797                                          * master/representor and retrieve
2798                                          * ifindex via sysfs.
2799                                          */
2800                                         continue;
2801                                 }
2802                                 ret = mlx5_get_master_ifname
2803                                         (ibv_match[i]->ibdev_path, &ifname);
2804                                 if (!ret)
2805                                         list[ns].ifindex =
2806                                                 if_nametoindex(ifname);
2807                                 if (!list[ns].ifindex) {
2808                                         /*
2809                                          * No network interface index found
2810                                          * for the specified device, it means
2811                                          * there it is neither representor
2812                                          * nor master.
2813                                          */
2814                                         continue;
2815                                 }
2816                         }
2817                         ret = -1;
2818                         if (nl_route >= 0)
2819                                 ret = mlx5_nl_switch_info
2820                                                (nl_route,
2821                                                 list[ns].ifindex,
2822                                                 &list[ns].info);
2823                         if (ret || (!list[ns].info.representor &&
2824                                     !list[ns].info.master)) {
2825                                 /*
2826                                  * We failed to recognize representors with
2827                                  * Netlink, let's try to perform the task
2828                                  * with sysfs.
2829                                  */
2830                                 ret =  mlx5_sysfs_switch_info
2831                                                 (list[ns].ifindex,
2832                                                  &list[ns].info);
2833                         }
2834                         if (!ret && (list[ns].info.representor ^
2835                                      list[ns].info.master)) {
2836                                 ns++;
2837                         } else if ((nd == 1) &&
2838                                    !list[ns].info.representor &&
2839                                    !list[ns].info.master) {
2840                                 /*
2841                                  * Single IB device with
2842                                  * one physical port and
2843                                  * attached network device.
2844                                  * May be SRIOV is not enabled
2845                                  * or there is no representors.
2846                                  */
2847                                 DRV_LOG(INFO, "no E-Switch support detected");
2848                                 ns++;
2849                                 break;
2850                         }
2851                 }
2852                 if (!ns) {
2853                         DRV_LOG(ERR,
2854                                 "unable to recognize master/representors"
2855                                 " on the multiple IB devices");
2856                         rte_errno = ENOENT;
2857                         ret = -rte_errno;
2858                         goto exit;
2859                 }
2860         }
2861         assert(ns);
2862         /*
2863          * Sort list to probe devices in natural order for users convenience
2864          * (i.e. master first, then representors from lowest to highest ID).
2865          */
2866         qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp);
2867         /* Default configuration. */
2868         dev_config = (struct mlx5_dev_config){
2869                 .hw_padding = 0,
2870                 .mps = MLX5_ARG_UNSET,
2871                 .rx_vec_en = 1,
2872                 .txq_inline_max = MLX5_ARG_UNSET,
2873                 .txq_inline_min = MLX5_ARG_UNSET,
2874                 .txq_inline_mpw = MLX5_ARG_UNSET,
2875                 .txqs_inline = MLX5_ARG_UNSET,
2876                 .vf_nl_en = 1,
2877                 .mr_ext_memseg_en = 1,
2878                 .mprq = {
2879                         .enabled = 0, /* Disabled by default. */
2880                         .stride_num_n = MLX5_MPRQ_STRIDE_NUM_N,
2881                         .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN,
2882                         .min_rxqs_num = MLX5_MPRQ_MIN_RXQS,
2883                 },
2884                 .dv_esw_en = 1,
2885         };
2886         /* Device specific configuration. */
2887         switch (pci_dev->id.device_id) {
2888         case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
2889         case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
2890         case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
2891         case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
2892         case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF:
2893         case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF:
2894         case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF:
2895                 dev_config.vf = 1;
2896                 break;
2897         default:
2898                 break;
2899         }
2900         for (i = 0; i != ns; ++i) {
2901                 uint32_t restore;
2902
2903                 list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device,
2904                                                  &list[i],
2905                                                  dev_config);
2906                 if (!list[i].eth_dev) {
2907                         if (rte_errno != EBUSY && rte_errno != EEXIST)
2908                                 break;
2909                         /* Device is disabled or already spawned. Ignore it. */
2910                         continue;
2911                 }
2912                 restore = list[i].eth_dev->data->dev_flags;
2913                 rte_eth_copy_pci_info(list[i].eth_dev, pci_dev);
2914                 /* Restore non-PCI flags cleared by the above call. */
2915                 list[i].eth_dev->data->dev_flags |= restore;
2916                 mlx5_dev_interrupt_handler_devx_install(list[i].eth_dev);
2917                 rte_eth_dev_probing_finish(list[i].eth_dev);
2918         }
2919         if (i != ns) {
2920                 DRV_LOG(ERR,
2921                         "probe of PCI device " PCI_PRI_FMT " aborted after"
2922                         " encountering an error: %s",
2923                         pci_dev->addr.domain, pci_dev->addr.bus,
2924                         pci_dev->addr.devid, pci_dev->addr.function,
2925                         strerror(rte_errno));
2926                 ret = -rte_errno;
2927                 /* Roll back. */
2928                 while (i--) {
2929                         if (!list[i].eth_dev)
2930                                 continue;
2931                         mlx5_dev_close(list[i].eth_dev);
2932                         /* mac_addrs must not be freed because in dev_private */
2933                         list[i].eth_dev->data->mac_addrs = NULL;
2934                         claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
2935                 }
2936                 /* Restore original error. */
2937                 rte_errno = -ret;
2938         } else {
2939                 ret = 0;
2940         }
2941 exit:
2942         /*
2943          * Do the routine cleanup:
2944          * - close opened Netlink sockets
2945          * - free allocated spawn data array
2946          * - free the Infiniband device list
2947          */
2948         if (nl_rdma >= 0)
2949                 close(nl_rdma);
2950         if (nl_route >= 0)
2951                 close(nl_route);
2952         if (list)
2953                 rte_free(list);
2954         assert(ibv_list);
2955         mlx5_glue->free_device_list(ibv_list);
2956         return ret;
2957 }
2958
2959 /**
2960  * Look for the ethernet device belonging to mlx5 driver.
2961  *
2962  * @param[in] port_id
2963  *   port_id to start looking for device.
2964  * @param[in] pci_dev
2965  *   Pointer to the hint PCI device. When device is being probed
2966  *   the its siblings (master and preceding representors might
2967  *   not have assigned driver yet (because the mlx5_pci_probe()
2968  *   is not completed yet, for this case match on hint PCI
2969  *   device may be used to detect sibling device.
2970  *
2971  * @return
2972  *   port_id of found device, RTE_MAX_ETHPORT if not found.
2973  */
2974 uint16_t
2975 mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev)
2976 {
2977         while (port_id < RTE_MAX_ETHPORTS) {
2978                 struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2979
2980                 if (dev->state != RTE_ETH_DEV_UNUSED &&
2981                     dev->device &&
2982                     (dev->device == &pci_dev->device ||
2983                      (dev->device->driver &&
2984                      dev->device->driver->name &&
2985                      !strcmp(dev->device->driver->name, MLX5_DRIVER_NAME))))
2986                         break;
2987                 port_id++;
2988         }
2989         if (port_id >= RTE_MAX_ETHPORTS)
2990                 return RTE_MAX_ETHPORTS;
2991         return port_id;
2992 }
2993
2994 /**
2995  * DPDK callback to remove a PCI device.
2996  *
2997  * This function removes all Ethernet devices belong to a given PCI device.
2998  *
2999  * @param[in] pci_dev
3000  *   Pointer to the PCI device.
3001  *
3002  * @return
3003  *   0 on success, the function cannot fail.
3004  */
3005 static int
3006 mlx5_pci_remove(struct rte_pci_device *pci_dev)
3007 {
3008         uint16_t port_id;
3009
3010         RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
3011                 rte_eth_dev_close(port_id);
3012         return 0;
3013 }
3014
3015 static const struct rte_pci_id mlx5_pci_id_map[] = {
3016         {
3017                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3018                                PCI_DEVICE_ID_MELLANOX_CONNECTX4)
3019         },
3020         {
3021                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3022                                PCI_DEVICE_ID_MELLANOX_CONNECTX4VF)
3023         },
3024         {
3025                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3026                                PCI_DEVICE_ID_MELLANOX_CONNECTX4LX)
3027         },
3028         {
3029                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3030                                PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF)
3031         },
3032         {
3033                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3034                                PCI_DEVICE_ID_MELLANOX_CONNECTX5)
3035         },
3036         {
3037                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3038                                PCI_DEVICE_ID_MELLANOX_CONNECTX5VF)
3039         },
3040         {
3041                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3042                                PCI_DEVICE_ID_MELLANOX_CONNECTX5EX)
3043         },
3044         {
3045                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3046                                PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF)
3047         },
3048         {
3049                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3050                                PCI_DEVICE_ID_MELLANOX_CONNECTX5BF)
3051         },
3052         {
3053                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3054                                PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF)
3055         },
3056         {
3057                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3058                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
3059         },
3060         {
3061                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3062                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
3063         },
3064         {
3065                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3066                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
3067         },
3068         {
3069                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
3070                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
3071         },
3072         {
3073                 .vendor_id = 0
3074         }
3075 };
3076
3077 static struct rte_pci_driver mlx5_driver = {
3078         .driver = {
3079                 .name = MLX5_DRIVER_NAME
3080         },
3081         .id_table = mlx5_pci_id_map,
3082         .probe = mlx5_pci_probe,
3083         .remove = mlx5_pci_remove,
3084         .dma_map = mlx5_dma_map,
3085         .dma_unmap = mlx5_dma_unmap,
3086         .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
3087                      RTE_PCI_DRV_PROBE_AGAIN,
3088 };
3089
3090 #ifdef RTE_IBVERBS_LINK_DLOPEN
3091
3092 /**
3093  * Suffix RTE_EAL_PMD_PATH with "-glue".
3094  *
3095  * This function performs a sanity check on RTE_EAL_PMD_PATH before
3096  * suffixing its last component.
3097  *
3098  * @param buf[out]
3099  *   Output buffer, should be large enough otherwise NULL is returned.
3100  * @param size
3101  *   Size of @p out.
3102  *
3103  * @return
3104  *   Pointer to @p buf or @p NULL in case suffix cannot be appended.
3105  */
3106 static char *
3107 mlx5_glue_path(char *buf, size_t size)
3108 {
3109         static const char *const bad[] = { "/", ".", "..", NULL };
3110         const char *path = RTE_EAL_PMD_PATH;
3111         size_t len = strlen(path);
3112         size_t off;
3113         int i;
3114
3115         while (len && path[len - 1] == '/')
3116                 --len;
3117         for (off = len; off && path[off - 1] != '/'; --off)
3118                 ;
3119         for (i = 0; bad[i]; ++i)
3120                 if (!strncmp(path + off, bad[i], (int)(len - off)))
3121                         goto error;
3122         i = snprintf(buf, size, "%.*s-glue", (int)len, path);
3123         if (i == -1 || (size_t)i >= size)
3124                 goto error;
3125         return buf;
3126 error:
3127         DRV_LOG(ERR,
3128                 "unable to append \"-glue\" to last component of"
3129                 " RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\"),"
3130                 " please re-configure DPDK");
3131         return NULL;
3132 }
3133
3134 /**
3135  * Initialization routine for run-time dependency on rdma-core.
3136  */
3137 static int
3138 mlx5_glue_init(void)
3139 {
3140         char glue_path[sizeof(RTE_EAL_PMD_PATH) - 1 + sizeof("-glue")];
3141         const char *path[] = {
3142                 /*
3143                  * A basic security check is necessary before trusting
3144                  * MLX5_GLUE_PATH, which may override RTE_EAL_PMD_PATH.
3145                  */
3146                 (geteuid() == getuid() && getegid() == getgid() ?
3147                  getenv("MLX5_GLUE_PATH") : NULL),
3148                 /*
3149                  * When RTE_EAL_PMD_PATH is set, use its glue-suffixed
3150                  * variant, otherwise let dlopen() look up libraries on its
3151                  * own.
3152                  */
3153                 (*RTE_EAL_PMD_PATH ?
3154                  mlx5_glue_path(glue_path, sizeof(glue_path)) : ""),
3155         };
3156         unsigned int i = 0;
3157         void *handle = NULL;
3158         void **sym;
3159         const char *dlmsg;
3160
3161         while (!handle && i != RTE_DIM(path)) {
3162                 const char *end;
3163                 size_t len;
3164                 int ret;
3165
3166                 if (!path[i]) {
3167                         ++i;
3168                         continue;
3169                 }
3170                 end = strpbrk(path[i], ":;");
3171                 if (!end)
3172                         end = path[i] + strlen(path[i]);
3173                 len = end - path[i];
3174                 ret = 0;
3175                 do {
3176                         char name[ret + 1];
3177
3178                         ret = snprintf(name, sizeof(name), "%.*s%s" MLX5_GLUE,
3179                                        (int)len, path[i],
3180                                        (!len || *(end - 1) == '/') ? "" : "/");
3181                         if (ret == -1)
3182                                 break;
3183                         if (sizeof(name) != (size_t)ret + 1)
3184                                 continue;
3185                         DRV_LOG(DEBUG, "looking for rdma-core glue as \"%s\"",
3186                                 name);
3187                         handle = dlopen(name, RTLD_LAZY);
3188                         break;
3189                 } while (1);
3190                 path[i] = end + 1;
3191                 if (!*end)
3192                         ++i;
3193         }
3194         if (!handle) {
3195                 rte_errno = EINVAL;
3196                 dlmsg = dlerror();
3197                 if (dlmsg)
3198                         DRV_LOG(WARNING, "cannot load glue library: %s", dlmsg);
3199                 goto glue_error;
3200         }
3201         sym = dlsym(handle, "mlx5_glue");
3202         if (!sym || !*sym) {
3203                 rte_errno = EINVAL;
3204                 dlmsg = dlerror();
3205                 if (dlmsg)
3206                         DRV_LOG(ERR, "cannot resolve glue symbol: %s", dlmsg);
3207                 goto glue_error;
3208         }
3209         mlx5_glue = *sym;
3210         return 0;
3211 glue_error:
3212         if (handle)
3213                 dlclose(handle);
3214         DRV_LOG(WARNING,
3215                 "cannot initialize PMD due to missing run-time dependency on"
3216                 " rdma-core libraries (libibverbs, libmlx5)");
3217         return -rte_errno;
3218 }
3219
3220 #endif
3221
3222 /**
3223  * Driver initialization routine.
3224  */
3225 RTE_INIT(rte_mlx5_pmd_init)
3226 {
3227         /* Initialize driver log type. */
3228         mlx5_logtype = rte_log_register("pmd.net.mlx5");
3229         if (mlx5_logtype >= 0)
3230                 rte_log_set_level(mlx5_logtype, RTE_LOG_NOTICE);
3231
3232         /* Build the static tables for Verbs conversion. */
3233         mlx5_set_ptype_table();
3234         mlx5_set_cksum_table();
3235         mlx5_set_swp_types_table();
3236         /*
3237          * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
3238          * huge pages. Calling ibv_fork_init() during init allows
3239          * applications to use fork() safely for purposes other than
3240          * using this PMD, which is not supported in forked processes.
3241          */
3242         setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
3243         /* Match the size of Rx completion entry to the size of a cacheline. */
3244         if (RTE_CACHE_LINE_SIZE == 128)
3245                 setenv("MLX5_CQE_SIZE", "128", 0);
3246         /*
3247          * MLX5_DEVICE_FATAL_CLEANUP tells ibv_destroy functions to
3248          * cleanup all the Verbs resources even when the device was removed.
3249          */
3250         setenv("MLX5_DEVICE_FATAL_CLEANUP", "1", 1);
3251 #ifdef RTE_IBVERBS_LINK_DLOPEN
3252         if (mlx5_glue_init())
3253                 return;
3254         assert(mlx5_glue);
3255 #endif
3256 #ifndef NDEBUG
3257         /* Glue structure must not contain any NULL pointers. */
3258         {
3259                 unsigned int i;
3260
3261                 for (i = 0; i != sizeof(*mlx5_glue) / sizeof(void *); ++i)
3262                         assert(((const void *const *)mlx5_glue)[i]);
3263         }
3264 #endif
3265         if (strcmp(mlx5_glue->version, MLX5_GLUE_VERSION)) {
3266                 DRV_LOG(ERR,
3267                         "rdma-core glue \"%s\" mismatch: \"%s\" is required",
3268                         mlx5_glue->version, MLX5_GLUE_VERSION);
3269                 return;
3270         }
3271         mlx5_glue->fork_init();
3272         rte_pci_register(&mlx5_driver);
3273 }
3274
3275 RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
3276 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map);
3277 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib");