X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4.c;h=640e06f747adaef2c95b2bb506b2d6fe3b8910dd;hb=35b2d13fd6fdcbd191f2a30d74648faeb1186c65;hp=abed2f5dcf267da4026f673260222c9eb226d905;hpb=9797bfcce1c9fcbf75e78dc17ecdc9b0da3ae945;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index abed2f5dcf..640e06f747 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -17,6 +17,7 @@ #include #include #include +#include #include /* Verbs headers do not support -pedantic. */ @@ -48,10 +49,16 @@ #include "mlx4_rxtx.h" #include "mlx4_utils.h" -struct mlx4_dev_list mlx4_mem_event_cb_list = - LIST_HEAD_INITIALIZER(mlx4_mem_event_cb_list); +static const char *MZ_MLX4_PMD_SHARED_DATA = "mlx4_pmd_shared_data"; -rte_rwlock_t mlx4_mem_event_rwlock = RTE_RWLOCK_INITIALIZER; +/* Shared memory between primary and secondary processes. */ +struct mlx4_shared_data *mlx4_shared_data; + +/* Spinlock for mlx4_shared_data allocation. */ +static rte_spinlock_t mlx4_shared_data_lock = RTE_SPINLOCK_INITIALIZER; + +/* Process local data for secondary processes. */ +static struct mlx4_local_data mlx4_local_data; /** Configuration structure for device arguments. */ struct mlx4_conf { @@ -59,16 +66,169 @@ struct mlx4_conf { uint32_t present; /**< Bit-field for existing ports. */ uint32_t enabled; /**< Bit-field for user-enabled ports. */ } ports; + int mr_ext_memseg_en; + /** Whether memseg should be extended for MR creation. */ }; /* Available parameters list. */ const char *pmd_mlx4_init_params[] = { MLX4_PMD_PORT_KVARG, + MLX4_MR_EXT_MEMSEG_EN_KVARG, NULL, }; static void mlx4_dev_stop(struct rte_eth_dev *dev); +/** + * Initialize shared data between primary and secondary process. + * + * A memzone is reserved by primary process and secondary processes attach to + * the memzone. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_init_shared_data(void) +{ + const struct rte_memzone *mz; + int ret = 0; + + rte_spinlock_lock(&mlx4_shared_data_lock); + if (mlx4_shared_data == NULL) { + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + /* Allocate shared memory. */ + mz = rte_memzone_reserve(MZ_MLX4_PMD_SHARED_DATA, + sizeof(*mlx4_shared_data), + SOCKET_ID_ANY, 0); + if (mz == NULL) { + ERROR("Cannot allocate mlx4 shared data\n"); + ret = -rte_errno; + goto error; + } + mlx4_shared_data = mz->addr; + memset(mlx4_shared_data, 0, sizeof(*mlx4_shared_data)); + rte_spinlock_init(&mlx4_shared_data->lock); + } else { + /* Lookup allocated shared memory. */ + mz = rte_memzone_lookup(MZ_MLX4_PMD_SHARED_DATA); + if (mz == NULL) { + ERROR("Cannot attach mlx4 shared data\n"); + ret = -rte_errno; + goto error; + } + mlx4_shared_data = mz->addr; + memset(&mlx4_local_data, 0, sizeof(mlx4_local_data)); + } + } +error: + rte_spinlock_unlock(&mlx4_shared_data_lock); + return ret; +} + +#ifdef HAVE_IBV_MLX4_BUF_ALLOCATORS +/** + * Verbs callback to allocate a memory. This function should allocate the space + * according to the size provided residing inside a huge page. + * Please note that all allocation must respect the alignment from libmlx4 + * (i.e. currently sysconf(_SC_PAGESIZE)). + * + * @param[in] size + * The size in bytes of the memory to allocate. + * @param[in] data + * A pointer to the callback data. + * + * @return + * Allocated buffer, NULL otherwise and rte_errno is set. + */ +static void * +mlx4_alloc_verbs_buf(size_t size, void *data) +{ + struct mlx4_priv *priv = data; + void *ret; + size_t alignment = sysconf(_SC_PAGESIZE); + unsigned int socket = SOCKET_ID_ANY; + + if (priv->verbs_alloc_ctx.type == MLX4_VERBS_ALLOC_TYPE_TX_QUEUE) { + const struct txq *txq = priv->verbs_alloc_ctx.obj; + + socket = txq->socket; + } else if (priv->verbs_alloc_ctx.type == + MLX4_VERBS_ALLOC_TYPE_RX_QUEUE) { + const struct rxq *rxq = priv->verbs_alloc_ctx.obj; + + socket = rxq->socket; + } + assert(data != NULL); + ret = rte_malloc_socket(__func__, size, alignment, socket); + if (!ret && size) + rte_errno = ENOMEM; + return ret; +} + +/** + * Verbs callback to free a memory. + * + * @param[in] ptr + * A pointer to the memory to free. + * @param[in] data + * A pointer to the callback data. + */ +static void +mlx4_free_verbs_buf(void *ptr, void *data __rte_unused) +{ + assert(data != NULL); + rte_free(ptr); +} +#endif + +/** + * Initialize process private data structure. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_proc_priv_init(struct rte_eth_dev *dev) +{ + struct mlx4_proc_priv *ppriv; + size_t ppriv_size; + + /* + * UAR register table follows the process private structure. BlueFlame + * registers for Tx queues are stored in the table. + */ + ppriv_size = sizeof(struct mlx4_proc_priv) + + dev->data->nb_tx_queues * sizeof(void *); + ppriv = rte_malloc_socket("mlx4_proc_priv", ppriv_size, + RTE_CACHE_LINE_SIZE, dev->device->numa_node); + if (!ppriv) { + rte_errno = ENOMEM; + return -rte_errno; + } + ppriv->uar_table_sz = ppriv_size; + dev->process_private = ppriv; + return 0; +} + +/** + * Un-initialize process private data structure. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mlx4_proc_priv_uninit(struct rte_eth_dev *dev) +{ + if (!dev->process_private) + return; + rte_free(dev->process_private); + dev->process_private = NULL; +} + /** * DPDK callback for Ethernet device configuration. * @@ -81,7 +241,7 @@ static void mlx4_dev_stop(struct rte_eth_dev *dev); static int mlx4_dev_configure(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx4_priv *priv = dev->data->dev_private; struct rte_flow_error error; int ret; @@ -95,23 +255,17 @@ mlx4_dev_configure(struct rte_eth_dev *dev) goto exit; } ret = mlx4_intr_install(priv); - if (ret) + if (ret) { ERROR("%p: interrupt handler installation failed", (void *)dev); - /* - * Once the device is added to the list of memory event callback, its - * global MR cache table cannot be expanded on the fly because of - * deadlock. If it overflows, lookup should be done by searching MR list - * linearly, which is slow. - */ - if (mlx4_mr_btree_init(&priv->mr.cache, MLX4_MR_BTREE_CACHE_N * 2, - dev->device->numa_node)) { - /* rte_errno is already set. */ - return -rte_errno; + goto exit; + } + ret = mlx4_proc_priv_init(dev); + if (ret) { + ERROR("%p: process private data allocation failed", + (void *)dev); + goto exit; } - rte_rwlock_write_lock(&mlx4_mem_event_rwlock); - LIST_INSERT_HEAD(&mlx4_mem_event_cb_list, priv, mem_event_cb); - rte_rwlock_write_unlock(&mlx4_mem_event_rwlock); exit: return ret; } @@ -131,7 +285,7 @@ exit: static int mlx4_dev_start(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx4_priv *priv = dev->data->dev_private; struct rte_flow_error error; int ret; @@ -166,6 +320,8 @@ mlx4_dev_start(struct rte_eth_dev *dev) rte_wmb(); dev->tx_pkt_burst = mlx4_tx_burst; dev->rx_pkt_burst = mlx4_rx_burst; + /* Enable datapath on secondary process. */ + mlx4_mp_req_start_rxtx(dev); return 0; err: mlx4_dev_stop(dev); @@ -183,7 +339,7 @@ err: static void mlx4_dev_stop(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx4_priv *priv = dev->data->dev_private; if (!priv->started) return; @@ -192,6 +348,8 @@ mlx4_dev_stop(struct rte_eth_dev *dev) dev->tx_pkt_burst = mlx4_tx_burst_removed; dev->rx_pkt_burst = mlx4_rx_burst_removed; rte_wmb(); + /* Disable datapath on secondary process. */ + mlx4_mp_req_stop_rxtx(dev); mlx4_flow_sync(priv, NULL); mlx4_rxq_intr_disable(priv); mlx4_rss_deinit(priv); @@ -208,7 +366,7 @@ mlx4_dev_stop(struct rte_eth_dev *dev) static void mlx4_dev_close(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx4_priv *priv = dev->data->dev_private; unsigned int i; DEBUG("%p: closing device \"%s\"", @@ -217,12 +375,15 @@ mlx4_dev_close(struct rte_eth_dev *dev) dev->rx_pkt_burst = mlx4_rx_burst_removed; dev->tx_pkt_burst = mlx4_tx_burst_removed; rte_wmb(); + /* Disable datapath on secondary process. */ + mlx4_mp_req_stop_rxtx(dev); mlx4_flow_clean(priv); mlx4_rss_deinit(priv); for (i = 0; i != dev->data->nb_rx_queues; ++i) mlx4_rx_queue_release(dev->data->rx_queues[i]); for (i = 0; i != dev->data->nb_tx_queues; ++i) mlx4_tx_queue_release(dev->data->tx_queues[i]); + mlx4_proc_priv_uninit(dev); mlx4_mr_release(dev); if (priv->pd != NULL) { assert(priv->ctx != NULL); @@ -249,8 +410,10 @@ static const struct eth_dev_ops mlx4_dev_ops = { .mac_addr_remove = mlx4_mac_addr_remove, .mac_addr_add = mlx4_mac_addr_add, .mac_addr_set = mlx4_mac_addr_set, + .set_mc_addr_list = mlx4_set_mc_addr_list, .stats_get = mlx4_stats_get, .stats_reset = mlx4_stats_reset, + .fw_version_get = mlx4_fw_version_get, .dev_infos_get = mlx4_dev_infos_get, .dev_supported_ptypes_get = mlx4_dev_supported_ptypes_get, .vlan_filter_set = mlx4_vlan_filter_set, @@ -267,6 +430,14 @@ static const struct eth_dev_ops mlx4_dev_ops = { .is_removed = mlx4_is_removed, }; +/* Available operations from secondary process. */ +static const struct eth_dev_ops mlx4_dev_sec_ops = { + .stats_get = mlx4_stats_get, + .stats_reset = mlx4_stats_reset, + .fw_version_get = mlx4_fw_version_get, + .dev_infos_get = mlx4_dev_infos_get, +}; + /** * Get PCI information from struct ibv_device. * @@ -358,6 +529,8 @@ mlx4_arg_parse(const char *key, const char *val, struct mlx4_conf *conf) return -rte_errno; } conf->ports.enabled |= 1 << tmp; + } else if (strcmp(MLX4_MR_EXT_MEMSEG_EN_KVARG, key) == 0) { + conf->mr_ext_memseg_en = !!tmp; } else { rte_errno = EINVAL; WARN("%s: unknown parameter", key); @@ -393,10 +566,10 @@ mlx4_args(struct rte_devargs *devargs, struct mlx4_conf *conf) } /* Process parameters. */ for (i = 0; pmd_mlx4_init_params[i]; ++i) { - arg_count = rte_kvargs_count(kvlist, MLX4_PMD_PORT_KVARG); + arg_count = rte_kvargs_count(kvlist, pmd_mlx4_init_params[i]); while (arg_count-- > 0) { ret = rte_kvargs_process(kvlist, - MLX4_PMD_PORT_KVARG, + pmd_mlx4_init_params[i], (int (*)(const char *, const char *, void *)) @@ -506,6 +679,52 @@ mlx4_hw_rss_sup(struct ibv_context *ctx, struct ibv_pd *pd, static struct rte_pci_driver mlx4_driver; +/** + * PMD global initialization. + * + * Independent from individual device, this function initializes global + * per-PMD data structures distinguishing primary and secondary processes. + * Hence, each initialization is called once per a process. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_init_once(void) +{ + struct mlx4_shared_data *sd; + struct mlx4_local_data *ld = &mlx4_local_data; + + if (mlx4_init_shared_data()) + return -rte_errno; + sd = mlx4_shared_data; + assert(sd); + rte_spinlock_lock(&sd->lock); + switch (rte_eal_process_type()) { + case RTE_PROC_PRIMARY: + if (sd->init_done) + break; + LIST_INIT(&sd->mem_event_cb_list); + rte_rwlock_init(&sd->mem_event_rwlock); + rte_mem_event_callback_register("MLX4_MEM_EVENT_CB", + mlx4_mr_mem_event_cb, NULL); + mlx4_mp_init_primary(); + sd->init_done = true; + break; + case RTE_PROC_SECONDARY: + if (ld->init_done) + break; + mlx4_mp_init_secondary(); + ++sd->secondary_cnt; + ld->init_done = true; + break; + default: + break; + } + rte_spinlock_unlock(&sd->lock); + return 0; +} + /** * DPDK callback to register a PCI device. * @@ -531,11 +750,18 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct ibv_device_attr_ex device_attr_ex; struct mlx4_conf conf = { .ports.present = 0, + .mr_ext_memseg_en = 1, }; unsigned int vf; int i; (void)pci_drv; + err = mlx4_init_once(); + if (err) { + ERROR("unable to init PMD global data: %s", + strerror(rte_errno)); + return -rte_errno; + } assert(pci_drv == &mlx4_driver); list = mlx4_glue->get_device_list(&i); if (list == NULL) { @@ -589,14 +815,14 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ibv_dev = list[i]; DEBUG("device opened"); if (mlx4_glue->query_device(attr_ctx, &device_attr)) { - rte_errno = ENODEV; + err = ENODEV; goto error; } INFO("%u port(s) detected", device_attr.phys_port_cnt); conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1; if (mlx4_args(pci_dev->device.devargs, &conf)) { ERROR("failed to process device arguments"); - rte_errno = EINVAL; + err = EINVAL; goto error; } /* Use all ports when none are defined */ @@ -604,7 +830,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) conf.ports.enabled = conf.ports.present; /* Retrieve extended device attributes. */ if (mlx4_glue->query_device_ex(attr_ctx, NULL, &device_attr_ex)) { - rte_errno = ENODEV; + err = ENODEV; goto error; } assert(device_attr.max_sge >= MLX4_MAX_SGE); @@ -613,9 +839,10 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct ibv_context *ctx = NULL; struct ibv_port_attr port_attr; struct ibv_pd *pd = NULL; - struct priv *priv = NULL; + struct mlx4_priv *priv = NULL; struct rte_eth_dev *eth_dev = NULL; - struct ether_addr mac; + struct rte_ether_addr mac; + char name[RTE_ETH_NAME_MAX_LEN]; /* If port is not enabled, skip. */ if (!(conf.ports.enabled & (1 << i))) @@ -623,18 +850,66 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) DEBUG("using port %u", port); ctx = mlx4_glue->open_device(ibv_dev); if (ctx == NULL) { - rte_errno = ENODEV; + err = ENODEV; goto port_error; } + snprintf(name, sizeof(name), "%s port %u", + mlx4_glue->get_device_name(ibv_dev), port); + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + eth_dev = rte_eth_dev_attach_secondary(name); + if (eth_dev == NULL) { + ERROR("can not attach rte ethdev"); + rte_errno = ENOMEM; + err = rte_errno; + goto error; + } + priv = eth_dev->data->dev_private; + if (!priv->verbs_alloc_ctx.enabled) { + ERROR("secondary process is not supported" + " due to lack of external allocator" + " from Verbs"); + rte_errno = ENOTSUP; + err = rte_errno; + goto error; + } + eth_dev->device = &pci_dev->device; + eth_dev->dev_ops = &mlx4_dev_sec_ops; + err = mlx4_proc_priv_init(eth_dev); + if (err) + goto error; + /* Receive command fd from primary process. */ + err = mlx4_mp_req_verbs_cmd_fd(eth_dev); + if (err < 0) { + err = rte_errno; + goto error; + } + /* Remap UAR for Tx queues. */ + err = mlx4_tx_uar_init_secondary(eth_dev, err); + if (err) { + err = rte_errno; + goto error; + } + /* + * Ethdev pointer is still required as input since + * the primary device is not accessible from the + * secondary process. + */ + eth_dev->tx_pkt_burst = mlx4_tx_burst; + eth_dev->rx_pkt_burst = mlx4_rx_burst; + claim_zero(mlx4_glue->close_device(ctx)); + rte_eth_copy_pci_info(eth_dev, pci_dev); + rte_eth_dev_probing_finish(eth_dev); + continue; + } /* Check port status. */ err = mlx4_glue->query_port(ctx, port, &port_attr); if (err) { - rte_errno = err; - ERROR("port query failed: %s", strerror(rte_errno)); + err = ENODEV; + ERROR("port query failed: %s", strerror(err)); goto port_error; } if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { - rte_errno = ENOTSUP; + err = ENOTSUP; ERROR("port %d is not configured in Ethernet mode", port); goto port_error; @@ -644,15 +919,16 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) port, mlx4_glue->port_state_str(port_attr.state), port_attr.state); /* Make asynchronous FD non-blocking to handle interrupts. */ - if (mlx4_fd_set_non_blocking(ctx->async_fd) < 0) { + err = mlx4_fd_set_non_blocking(ctx->async_fd); + if (err) { ERROR("cannot make asynchronous FD non-blocking: %s", - strerror(rte_errno)); + strerror(err)); goto port_error; } /* Allocate protection domain. */ pd = mlx4_glue->alloc_pd(ctx); if (pd == NULL) { - rte_errno = ENOMEM; + err = ENOMEM; ERROR("PD allocation failure"); goto port_error; } @@ -661,7 +937,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) sizeof(*priv), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - rte_errno = ENOMEM; + err = ENOMEM; ERROR("priv allocation failure"); goto port_error; } @@ -669,7 +945,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) priv->device_attr = device_attr; priv->port = port; priv->pd = pd; - priv->mtu = ETHER_MTU; + priv->mtu = RTE_ETHER_MTU; priv->vf = vf; priv->hw_csum = !!(device_attr.device_cap_flags & IBV_DEVICE_RAW_IP_CSUM); @@ -686,14 +962,28 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) &device_attr_ex); DEBUG("supported RSS hash fields mask: %016" PRIx64, priv->hw_rss_sup); + priv->hw_rss_max_qps = + device_attr_ex.rss_caps.max_rwq_indirection_table_size; + DEBUG("MAX RSS queues %d", priv->hw_rss_max_qps); priv->hw_fcs_strip = !!(device_attr_ex.raw_packet_caps & IBV_RAW_PACKET_CAP_SCATTER_FCS); DEBUG("FCS stripping toggling is %ssupported", priv->hw_fcs_strip ? "" : "not "); + priv->tso = + ((device_attr_ex.tso_caps.max_tso > 0) && + (device_attr_ex.tso_caps.supported_qpts & + (1 << IBV_QPT_RAW_PACKET))); + if (priv->tso) + priv->tso_max_payload_sz = + device_attr_ex.tso_caps.max_tso; + DEBUG("TSO is %ssupported", + priv->tso ? "" : "not "); + priv->mr_ext_memseg_en = conf.mr_ext_memseg_en; /* Configure the first MAC address by default. */ - if (mlx4_get_mac(priv, &mac.addr_bytes)) { + err = mlx4_get_mac(priv, &mac.addr_bytes); + if (err) { ERROR("cannot get MAC address, is mlx4_en loaded?" - " (rte_errno: %s)", strerror(rte_errno)); + " (error: %s)", strerror(err)); goto port_error; } INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", @@ -717,24 +1007,16 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) /* Get actual MTU if possible. */ mlx4_mtu_get(priv, &priv->mtu); DEBUG("port %u MTU is %u", priv->port, priv->mtu); - /* from rte_ethdev.c */ - { - char name[RTE_ETH_NAME_MAX_LEN]; - - snprintf(name, sizeof(name), "%s port %u", - mlx4_glue->get_device_name(ibv_dev), port); - eth_dev = rte_eth_dev_allocate(name); - } + eth_dev = rte_eth_dev_allocate(name); if (eth_dev == NULL) { + err = ENOMEM; ERROR("can not allocate rte ethdev"); - rte_errno = ENOMEM; goto port_error; } eth_dev->data->dev_private = priv; eth_dev->data->mac_addrs = priv->mac; eth_dev->device = &pci_dev->device; rte_eth_copy_pci_info(eth_dev, pci_dev); - eth_dev->device->driver = &mlx4_driver.driver; /* Initialize local interrupt handle for current port. */ priv->intr_handle = (struct rte_intr_handle){ .fd = -1, @@ -753,27 +1035,64 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) * handled by rte_intr_rx_ctl(). */ eth_dev->intr_handle = &priv->intr_handle; - priv->dev = eth_dev; + priv->dev_data = eth_dev->data; eth_dev->dev_ops = &mlx4_dev_ops; +#ifdef HAVE_IBV_MLX4_BUF_ALLOCATORS + /* Hint libmlx4 to use PMD allocator for data plane resources */ + struct mlx4dv_ctx_allocators alctr = { + .alloc = &mlx4_alloc_verbs_buf, + .free = &mlx4_free_verbs_buf, + .data = priv, + }; + err = mlx4_glue->dv_set_context_attr + (ctx, MLX4DV_SET_CTX_ATTR_BUF_ALLOCATORS, + (void *)((uintptr_t)&alctr)); + if (err) + WARN("Verbs external allocator is not supported"); + else + priv->verbs_alloc_ctx.enabled = 1; +#endif /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); - mlx4_dev_set_link_up(priv->dev); + mlx4_dev_set_link_up(eth_dev); /* Update link status once if waiting for LSC. */ if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) mlx4_link_update(eth_dev, 0); + /* + * Once the device is added to the list of memory event + * callback, its global MR cache table cannot be expanded + * on the fly because of deadlock. If it overflows, lookup + * should be done by searching MR list linearly, which is slow. + */ + err = mlx4_mr_btree_init(&priv->mr.cache, + MLX4_MR_BTREE_CACHE_N * 2, + eth_dev->device->numa_node); + if (err) { + /* rte_errno is already set. */ + goto port_error; + } + /* Add device to memory callback list. */ + rte_rwlock_write_lock(&mlx4_shared_data->mem_event_rwlock); + LIST_INSERT_HEAD(&mlx4_shared_data->mem_event_cb_list, + priv, mem_event_cb); + rte_rwlock_write_unlock(&mlx4_shared_data->mem_event_rwlock); + rte_eth_dev_probing_finish(eth_dev); continue; port_error: rte_free(priv); + if (eth_dev != NULL) + eth_dev->data->dev_private = NULL; if (pd) claim_zero(mlx4_glue->dealloc_pd(pd)); if (ctx) claim_zero(mlx4_glue->close_device(ctx)); - if (eth_dev) + if (eth_dev != NULL) { + /* mac_addrs must not be freed because part of dev_private */ + eth_dev->data->mac_addrs = NULL; rte_eth_dev_release_port(eth_dev); + } break; } - if (i == device_attr.phys_port_cnt) - return 0; /* * XXX if something went wrong in the loop above, there is a resource * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as @@ -785,8 +1104,9 @@ error: claim_zero(mlx4_glue->close_device(attr_ctx)); if (list) mlx4_glue->free_device_list(list); - assert(rte_errno >= 0); - return -rte_errno; + if (err) + rte_errno = err; + return -err; } static const struct rte_pci_id mlx4_pci_id_map[] = { @@ -817,7 +1137,7 @@ static struct rte_pci_driver mlx4_driver = { RTE_PCI_DRV_INTR_RMV, }; -#ifdef RTE_LIBRTE_MLX4_DLOPEN_DEPS +#ifdef RTE_IBVERBS_LINK_DLOPEN /** * Suffix RTE_EAL_PMD_PATH with "-glue". @@ -950,9 +1270,7 @@ glue_error: /** * Driver initialization routine. */ -RTE_INIT(rte_mlx4_pmd_init); -static void -rte_mlx4_pmd_init(void) +RTE_INIT(rte_mlx4_pmd_init) { /* * MLX4_DEVICE_FATAL_CLEANUP tells ibv_destroy functions we @@ -967,7 +1285,7 @@ rte_mlx4_pmd_init(void) * using this PMD, which is not supported in forked processes. */ setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); -#ifdef RTE_LIBRTE_MLX4_DLOPEN_DEPS +#ifdef RTE_IBVERBS_LINK_DLOPEN if (mlx4_glue_init()) return; assert(mlx4_glue); @@ -988,8 +1306,6 @@ rte_mlx4_pmd_init(void) } mlx4_glue->fork_init(); rte_pci_register(&mlx4_driver); - rte_mem_event_callback_register("MLX4_MEM_EVENT_CB", - mlx4_mr_mem_event_cb, NULL); } RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__);