+ int ret = 0;
+
+ if (mlx5_init_shared_data())
+ return -rte_errno;
+ sd = mlx5_shared_data;
+ assert(sd);
+ rte_spinlock_lock(&sd->lock);
+ switch (rte_eal_process_type()) {
+ case RTE_PROC_PRIMARY:
+ if (sd->init_done)
+ break;
+ LIST_INIT(&sd->mem_event_cb_list);
+ rte_rwlock_init(&sd->mem_event_rwlock);
+ rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+ mlx5_mr_mem_event_cb, NULL);
+ ret = mlx5_mp_init_primary();
+ if (ret)
+ goto out;
+ sd->init_done = true;
+ break;
+ case RTE_PROC_SECONDARY:
+ if (ld->init_done)
+ break;
+ ret = mlx5_mp_init_secondary();
+ if (ret)
+ goto out;
+ ++sd->secondary_cnt;
+ ld->init_done = true;
+ break;
+ default:
+ break;
+ }
+out:
+ rte_spinlock_unlock(&sd->lock);
+ return ret;
+}
+
+/**
+ * Configures the minimal amount of data to inline into WQE
+ * while sending packets.
+ *
+ * - the txq_inline_min has the maximal priority, if this
+ * key is specified in devargs
+ * - if DevX is enabled the inline mode is queried from the
+ * device (HCA attributes and NIC vport context if needed).
+ * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4LX
+ * and none (0 bytes) for other NICs
+ *
+ * @param spawn
+ * Verbs device parameters (name, port, switch_info) to spawn.
+ * @param config
+ * Device configuration parameters.
+ */
+static void
+mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
+ struct mlx5_dev_config *config)
+{
+ if (config->txq_inline_min != MLX5_ARG_UNSET) {
+ /* Application defines size of inlined data explicitly. */
+ switch (spawn->pci_dev->id.device_id) {
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
+ if (config->txq_inline_min <
+ (int)MLX5_INLINE_HSIZE_L2) {
+ DRV_LOG(DEBUG,
+ "txq_inline_mix aligned to minimal"
+ " ConnectX-4 required value %d",
+ (int)MLX5_INLINE_HSIZE_L2);
+ config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
+ }
+ break;
+ }
+ goto exit;
+ }
+ if (config->hca_attr.eth_net_offloads) {
+ /* We have DevX enabled, inline mode queried successfully. */
+ switch (config->hca_attr.wqe_inline_mode) {
+ case MLX5_CAP_INLINE_MODE_L2:
+ /* outer L2 header must be inlined. */
+ config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
+ goto exit;
+ case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
+ /* No inline data are required by NIC. */
+ config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
+ config->hw_vlan_insert =
+ config->hca_attr.wqe_vlan_insert;
+ DRV_LOG(DEBUG, "Tx VLAN insertion is supported");
+ goto exit;
+ case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
+ /* inline mode is defined by NIC vport context. */
+ if (!config->hca_attr.eth_virt)
+ break;
+ switch (config->hca_attr.vport_inline_mode) {
+ case MLX5_INLINE_MODE_NONE:
+ config->txq_inline_min =
+ MLX5_INLINE_HSIZE_NONE;
+ goto exit;
+ case MLX5_INLINE_MODE_L2:
+ config->txq_inline_min =
+ MLX5_INLINE_HSIZE_L2;
+ goto exit;
+ case MLX5_INLINE_MODE_IP:
+ config->txq_inline_min =
+ MLX5_INLINE_HSIZE_L3;
+ goto exit;
+ case MLX5_INLINE_MODE_TCP_UDP:
+ config->txq_inline_min =
+ MLX5_INLINE_HSIZE_L4;
+ goto exit;
+ case MLX5_INLINE_MODE_INNER_L2:
+ config->txq_inline_min =
+ MLX5_INLINE_HSIZE_INNER_L2;
+ goto exit;
+ case MLX5_INLINE_MODE_INNER_IP:
+ config->txq_inline_min =
+ MLX5_INLINE_HSIZE_INNER_L3;
+ goto exit;
+ case MLX5_INLINE_MODE_INNER_TCP_UDP:
+ config->txq_inline_min =
+ MLX5_INLINE_HSIZE_INNER_L4;
+ goto exit;
+ }
+ }
+ }
+ /*
+ * We get here if we are unable to deduce
+ * inline data size with DevX. Try PCI ID
+ * to determine old NICs.
+ */
+ switch (spawn->pci_dev->id.device_id) {
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX:
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
+ config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
+ config->hw_vlan_insert = 0;
+ break;
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX5:
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX:
+ case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
+ /*
+ * These NICs support VLAN insertion from WQE and
+ * report the wqe_vlan_insert flag. But there is the bug
+ * and PFC control may be broken, so disable feature.
+ */
+ config->hw_vlan_insert = 0;
+ config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
+ break;
+ default:
+ config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
+ break;
+ }
+exit:
+ DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min);
+}
+
+/**
+ * Configures the metadata mask fields in the shared context.
+ *
+ * @param [in] dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx5_set_metadata_mask(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
+ uint32_t meta, mark, reg_c0;
+
+ reg_c0 = ~priv->vport_meta_mask;
+ switch (priv->config.dv_xmeta_en) {
+ case MLX5_XMETA_MODE_LEGACY:
+ meta = UINT32_MAX;
+ mark = MLX5_FLOW_MARK_MASK;
+ break;
+ case MLX5_XMETA_MODE_META16:
+ meta = reg_c0 >> rte_bsf32(reg_c0);
+ mark = MLX5_FLOW_MARK_MASK;
+ break;
+ case MLX5_XMETA_MODE_META32:
+ meta = UINT32_MAX;
+ mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK;
+ break;
+ default:
+ meta = 0;
+ mark = 0;
+ assert(false);
+ break;
+ }
+ if (sh->dv_mark_mask && sh->dv_mark_mask != mark)
+ DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X",
+ sh->dv_mark_mask, mark);
+ else
+ sh->dv_mark_mask = mark;
+ if (sh->dv_meta_mask && sh->dv_meta_mask != meta)
+ DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X",
+ sh->dv_meta_mask, meta);
+ else
+ sh->dv_meta_mask = meta;
+ if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0)
+ DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X",
+ sh->dv_meta_mask, reg_c0);
+ else
+ sh->dv_regc0_mask = reg_c0;
+ DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en);
+ DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask);
+ DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask);
+ DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask);
+}
+
+/**
+ * Allocate page of door-bells and register it using DevX API.
+ *
+ * @param [in] dev
+ * Pointer to Ethernet device.
+ *
+ * @return
+ * Pointer to new page on success, NULL otherwise.
+ */
+static struct mlx5_devx_dbr_page *
+mlx5_alloc_dbr_page(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_devx_dbr_page *page;
+
+ /* Allocate space for door-bell page and management data. */
+ page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
+ RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+ if (!page) {
+ DRV_LOG(ERR, "port %u cannot allocate dbr page",
+ dev->data->port_id);
+ return NULL;
+ }
+ /* Register allocated memory. */
+ page->umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, page->dbrs,
+ MLX5_DBR_PAGE_SIZE, 0);
+ if (!page->umem) {
+ DRV_LOG(ERR, "port %u cannot umem reg dbr page",
+ dev->data->port_id);
+ rte_free(page);
+ return NULL;
+ }
+ return page;
+}
+
+/**
+ * Find the next available door-bell, allocate new page if needed.
+ *
+ * @param [in] dev
+ * Pointer to Ethernet device.
+ * @param [out] dbr_page
+ * Door-bell page containing the page data.
+ *
+ * @return
+ * Door-bell address offset on success, a negative error value otherwise.
+ */
+int64_t
+mlx5_get_dbr(struct rte_eth_dev *dev, struct mlx5_devx_dbr_page **dbr_page)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_devx_dbr_page *page = NULL;
+ uint32_t i, j;
+
+ LIST_FOREACH(page, &priv->dbrpgs, next)
+ if (page->dbr_count < MLX5_DBR_PER_PAGE)
+ break;
+ if (!page) { /* No page with free door-bell exists. */
+ page = mlx5_alloc_dbr_page(dev);
+ if (!page) /* Failed to allocate new page. */
+ return (-1);
+ LIST_INSERT_HEAD(&priv->dbrpgs, page, next);
+ }
+ /* Loop to find bitmap part with clear bit. */
+ for (i = 0;
+ i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
+ i++)
+ ; /* Empty. */
+ /* Find the first clear bit. */
+ j = rte_bsf64(~page->dbr_bitmap[i]);
+ assert(i < (MLX5_DBR_PER_PAGE / 64));
+ page->dbr_bitmap[i] |= (1 << j);
+ page->dbr_count++;
+ *dbr_page = page;
+ return (((i * 64) + j) * sizeof(uint64_t));
+}
+
+/**
+ * Release a door-bell record.
+ *
+ * @param [in] dev
+ * Pointer to Ethernet device.
+ * @param [in] umem_id
+ * UMEM ID of page containing the door-bell record to release.
+ * @param [in] offset
+ * Offset of door-bell record in page.
+ *
+ * @return
+ * 0 on success, a negative error value otherwise.
+ */
+int32_t
+mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id, uint64_t offset)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_devx_dbr_page *page = NULL;
+ int ret = 0;
+
+ LIST_FOREACH(page, &priv->dbrpgs, next)
+ /* Find the page this address belongs to. */
+ if (page->umem->umem_id == umem_id)
+ break;
+ if (!page)
+ return -EINVAL;
+ page->dbr_count--;
+ if (!page->dbr_count) {
+ /* Page not used, free it and remove from list. */
+ LIST_REMOVE(page, next);
+ if (page->umem)
+ ret = -mlx5_glue->devx_umem_dereg(page->umem);
+ rte_free(page);
+ } else {
+ /* Mark in bitmap that this door-bell is not in use. */
+ offset /= MLX5_DBR_SIZE;
+ int i = offset / 64;
+ int j = offset % 64;
+
+ page->dbr_bitmap[i] &= ~(1 << j);
+ }
+ return ret;
+}
+
+/**
+ * Check sibling device configurations.
+ *
+ * Sibling devices sharing the Infiniband device context
+ * should have compatible configurations. This regards
+ * representors and bonding slaves.
+ *
+ * @param priv
+ * Private device descriptor.
+ * @param config
+ * Configuration of the device is going to be created.
+ *
+ * @return
+ * 0 on success, EINVAL otherwise
+ */
+static int
+mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
+ struct mlx5_dev_config *config)
+{
+ struct mlx5_ibv_shared *sh = priv->sh;
+ struct mlx5_dev_config *sh_conf = NULL;
+ uint16_t port_id;
+
+ assert(sh);
+ /* Nothing to compare for the single/first device. */
+ if (sh->refcnt == 1)
+ return 0;
+ /* Find the device with shared context. */
+ MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+ struct mlx5_priv *opriv =
+ rte_eth_devices[port_id].data->dev_private;