X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4.c;h=67851b6913184185a55c628bc1498844aa665806;hb=84121f197187;hp=4f57e2d74d39c5513fc6884a017c76fdc0defd86;hpb=cacd65c8bb073fd46eee9b8deed582b01be610e6;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index 4f57e2d74d..67851b6913 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -73,7 +73,6 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-pedantic" #endif -#include #include #include #include @@ -87,6 +86,7 @@ #include #include #include +#include #ifdef PEDANTIC #pragma GCC diagnostic error "-pedantic" #endif @@ -132,15 +132,6 @@ typedef union { #define WR_ID(o) (((wr_id_t *)&(o))->data) -/* Compile-time check. */ -static inline void wr_id_t_check(void) -{ - wr_id_t check[1 + (2 * -!(sizeof(wr_id_t) == sizeof(uint64_t)))]; - - (void)check; - (void)wr_id_t_check; -} - /* Transpose flags. Useful to convert IBV to DPDK flags. */ #define TRANSPOSE(val, from, to) \ (((from) >= (to)) ? \ @@ -299,6 +290,46 @@ struct priv { rte_spinlock_t lock; /* Lock for control functions. */ }; +/* Local storage for secondary process data. */ +struct mlx4_secondary_data { + struct rte_eth_dev_data data; /* Local device data. */ + struct priv *primary_priv; /* Private structure from primary. */ + struct rte_eth_dev_data *shared_dev_data; /* Shared device data. */ + rte_spinlock_t lock; /* Port configuration lock. */ +} mlx4_secondary_data[RTE_MAX_ETHPORTS]; + +/** + * Check if running as a secondary process. + * + * @return + * Nonzero if running as a secondary process. + */ +static inline int +mlx4_is_secondary(void) +{ + return rte_eal_process_type() != RTE_PROC_PRIMARY; +} + +/** + * Return private structure associated with an Ethernet device. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * Pointer to private structure. + */ +static struct priv * +mlx4_get_priv(struct rte_eth_dev *dev) +{ + struct mlx4_secondary_data *sd; + + if (!mlx4_is_secondary()) + return dev->data->dev_private; + sd = &mlx4_secondary_data[dev->data->port_id]; + return sd->data.dev_private; +} + /** * Lock private structure to protect it from concurrent access in the * control path. @@ -658,9 +689,16 @@ priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) /* Device configuration. */ +static int +txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf); + +static void +txq_cleanup(struct txq *txq); + static int rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, - unsigned int socket, const struct rte_eth_rxconf *conf, + unsigned int socket, int inactive, const struct rte_eth_rxconf *conf, struct rte_mempool *mp); static void @@ -696,6 +734,15 @@ dev_configure(struct rte_eth_dev *dev) } if (rxqs_n == priv->rxqs_n) return 0; + if (!rte_is_power_of_2(rxqs_n)) { + unsigned n_active; + + n_active = rte_align32pow2(rxqs_n + 1) >> 1; + WARN("%p: number of RX queues must be a power" + " of 2: %u queues among %u will be active", + (void *)dev, n_active, rxqs_n); + } + INFO("%p: RX queues number update: %u -> %u", (void *)dev, priv->rxqs_n, rxqs_n); /* If RSS is enabled, disable it first. */ @@ -731,7 +778,7 @@ dev_configure(struct rte_eth_dev *dev) priv->rss = 1; tmp = priv->rxqs_n; priv->rxqs_n = rxqs_n; - ret = rxq_setup(dev, &priv->rxq_parent, 0, 0, NULL, NULL); + ret = rxq_setup(dev, &priv->rxq_parent, 0, 0, 0, NULL, NULL); if (!ret) return 0; /* Failure, rollback. */ @@ -756,6 +803,8 @@ mlx4_dev_configure(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); ret = dev_configure(dev); assert(ret >= 0); @@ -763,6 +812,160 @@ mlx4_dev_configure(struct rte_eth_dev *dev) return -ret; } +static uint16_t mlx4_tx_burst(void *, struct rte_mbuf **, uint16_t); +static uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t); + +/** + * Configure secondary process queues from a private data pointer (primary + * or secondary) and update burst callbacks. Can take place only once. + * + * All queues must have been previously created by the primary process to + * avoid undefined behavior. + * + * @param priv + * Private data pointer from either primary or secondary process. + * + * @return + * Private data pointer from secondary process, NULL in case of error. + */ +static struct priv * +mlx4_secondary_data_setup(struct priv *priv) +{ + unsigned int port_id = 0; + struct mlx4_secondary_data *sd; + void **tx_queues; + void **rx_queues; + unsigned int nb_tx_queues; + unsigned int nb_rx_queues; + unsigned int i; + + /* priv must be valid at this point. */ + assert(priv != NULL); + /* priv->dev must also be valid but may point to local memory from + * another process, possibly with the same address and must not + * be dereferenced yet. */ + assert(priv->dev != NULL); + /* Determine port ID by finding out where priv comes from. */ + while (1) { + sd = &mlx4_secondary_data[port_id]; + rte_spinlock_lock(&sd->lock); + /* Primary process? */ + if (sd->primary_priv == priv) + break; + /* Secondary process? */ + if (sd->data.dev_private == priv) + break; + rte_spinlock_unlock(&sd->lock); + if (++port_id == RTE_DIM(mlx4_secondary_data)) + port_id = 0; + } + /* Switch to secondary private structure. If private data has already + * been updated by another thread, there is nothing else to do. */ + priv = sd->data.dev_private; + if (priv->dev->data == &sd->data) + goto end; + /* Sanity checks. Secondary private structure is supposed to point + * to local eth_dev, itself still pointing to the shared device data + * structure allocated by the primary process. */ + assert(sd->shared_dev_data != &sd->data); + assert(sd->data.nb_tx_queues == 0); + assert(sd->data.tx_queues == NULL); + assert(sd->data.nb_rx_queues == 0); + assert(sd->data.rx_queues == NULL); + assert(priv != sd->primary_priv); + assert(priv->dev->data == sd->shared_dev_data); + assert(priv->txqs_n == 0); + assert(priv->txqs == NULL); + assert(priv->rxqs_n == 0); + assert(priv->rxqs == NULL); + nb_tx_queues = sd->shared_dev_data->nb_tx_queues; + nb_rx_queues = sd->shared_dev_data->nb_rx_queues; + /* Allocate local storage for queues. */ + tx_queues = rte_zmalloc("secondary ethdev->tx_queues", + sizeof(sd->data.tx_queues[0]) * nb_tx_queues, + RTE_CACHE_LINE_SIZE); + rx_queues = rte_zmalloc("secondary ethdev->rx_queues", + sizeof(sd->data.rx_queues[0]) * nb_rx_queues, + RTE_CACHE_LINE_SIZE); + if (tx_queues == NULL || rx_queues == NULL) + goto error; + /* Lock to prevent control operations during setup. */ + priv_lock(priv); + /* TX queues. */ + for (i = 0; i != nb_tx_queues; ++i) { + struct txq *primary_txq = (*sd->primary_priv->txqs)[i]; + struct txq *txq; + + if (primary_txq == NULL) + continue; + txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0, + primary_txq->socket); + if (txq != NULL) { + if (txq_setup(priv->dev, + txq, + primary_txq->elts_n * MLX4_PMD_SGE_WR_N, + primary_txq->socket, + NULL) == 0) { + txq->stats.idx = primary_txq->stats.idx; + tx_queues[i] = txq; + continue; + } + rte_free(txq); + } + while (i) { + txq = tx_queues[--i]; + txq_cleanup(txq); + rte_free(txq); + } + goto error; + } + /* RX queues. */ + for (i = 0; i != nb_rx_queues; ++i) { + struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i]; + + if (primary_rxq == NULL) + continue; + /* Not supported yet. */ + rx_queues[i] = NULL; + } + /* Update everything. */ + priv->txqs = (void *)tx_queues; + priv->txqs_n = nb_tx_queues; + priv->rxqs = (void *)rx_queues; + priv->rxqs_n = nb_rx_queues; + sd->data.rx_queues = rx_queues; + sd->data.tx_queues = tx_queues; + sd->data.nb_rx_queues = nb_rx_queues; + sd->data.nb_tx_queues = nb_tx_queues; + sd->data.dev_link = sd->shared_dev_data->dev_link; + sd->data.mtu = sd->shared_dev_data->mtu; + memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state, + sizeof(sd->data.rx_queue_state)); + memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state, + sizeof(sd->data.tx_queue_state)); + sd->data.dev_flags = sd->shared_dev_data->dev_flags; + /* Use local data from now on. */ + rte_mb(); + priv->dev->data = &sd->data; + rte_mb(); + priv->dev->tx_pkt_burst = mlx4_tx_burst; + priv->dev->rx_pkt_burst = removed_rx_burst; + priv_unlock(priv); +end: + /* More sanity checks. */ + assert(priv->dev->tx_pkt_burst == mlx4_tx_burst); + assert(priv->dev->rx_pkt_burst == removed_rx_burst); + assert(priv->dev->data == &sd->data); + rte_spinlock_unlock(&sd->lock); + return priv; +error: + priv_unlock(priv); + rte_free(tx_queues); + rte_free(rx_queues); + rte_spinlock_unlock(&sd->lock); + return NULL; +} + /* TX queues handling. */ /** @@ -844,14 +1047,20 @@ error: static void txq_free_elts(struct txq *txq) { - unsigned int i; unsigned int elts_n = txq->elts_n; + unsigned int elts_head = txq->elts_head; + unsigned int elts_tail = txq->elts_tail; struct txq_elt (*elts)[elts_n] = txq->elts; linear_t (*elts_linear)[elts_n] = txq->elts_linear; struct ibv_mr *mr_linear = txq->mr_linear; DEBUG("%p: freeing WRs", (void *)txq); txq->elts_n = 0; + txq->elts_head = 0; + txq->elts_tail = 0; + txq->elts_comp = 0; + txq->elts_comp_cd = 0; + txq->elts_comp_cd_init = 0; txq->elts = NULL; txq->elts_linear = NULL; txq->mr_linear = NULL; @@ -861,12 +1070,17 @@ txq_free_elts(struct txq *txq) rte_free(elts_linear); if (elts == NULL) return; - for (i = 0; (i != elemof(*elts)); ++i) { - struct txq_elt *elt = &(*elts)[i]; + while (elts_tail != elts_head) { + struct txq_elt *elt = &(*elts)[elts_tail]; - if (elt->buf == NULL) - continue; + assert(elt->buf != NULL); rte_pktmbuf_free(elt->buf); +#ifndef NDEBUG + /* Poisoning. */ + memset(elt, 0x77, sizeof(*elt)); +#endif + if (++elts_tail == elts_n) + elts_tail = 0; } rte_free(elts); } @@ -984,6 +1198,52 @@ txq_complete(struct txq *txq) return 0; } +/* For best performance, this function should not be inlined. */ +static struct ibv_mr *mlx4_mp2mr(struct ibv_pd *, const struct rte_mempool *) + __attribute__((noinline)); + +/** + * Register mempool as a memory region. + * + * @param pd + * Pointer to protection domain. + * @param mp + * Pointer to memory pool. + * + * @return + * Memory region pointer, NULL in case of error. + */ +static struct ibv_mr * +mlx4_mp2mr(struct ibv_pd *pd, const struct rte_mempool *mp) +{ + const struct rte_memseg *ms = rte_eal_get_physmem_layout(); + uintptr_t start = (uintptr_t)STAILQ_FIRST(&mp->mem_list)->addr; + uintptr_t end = start + STAILQ_FIRST(&mp->mem_list)->len; + unsigned int i; + + DEBUG("mempool %p area start=%p end=%p size=%zu", + (const void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); + /* Round start and end to page boundary if found in memory segments. */ + for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { + uintptr_t addr = (uintptr_t)ms[i].addr; + size_t len = ms[i].len; + unsigned int align = ms[i].hugepage_sz; + + if ((start > addr) && (start < addr + len)) + start = RTE_ALIGN_FLOOR(start, align); + if ((end > addr) && (end < addr + len)) + end = RTE_ALIGN_CEIL(end, align); + } + DEBUG("mempool %p using start=%p end=%p size=%zu for MR", + (const void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); + return ibv_reg_mr(pd, + (void *)start, + end - start, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); +} + /** * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which * the cloned mbuf is allocated is returned instead. @@ -1035,10 +1295,7 @@ txq_mp2mr(struct txq *txq, const struct rte_mempool *mp) /* Add a new entry, register MR first. */ DEBUG("%p: discovered new memory pool \"%s\" (%p)", (void *)txq, mp->name, (const void *)mp); - mr = ibv_reg_mr(txq->priv->pd, - (void *)mp->elt_va_start, - (mp->elt_va_end - mp->elt_va_start), - (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE)); + mr = mlx4_mp2mr(txq->priv->pd, mp); if (unlikely(mr == NULL)) { DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.", (void *)txq); @@ -1063,7 +1320,6 @@ txq_mp2mr(struct txq *txq, const struct rte_mempool *mp) } struct txq_mp2mr_mbuf_check_data { - const struct rte_mempool *mp; int ret; }; @@ -1071,34 +1327,26 @@ struct txq_mp2mr_mbuf_check_data { * Callback function for rte_mempool_obj_iter() to check whether a given * mempool object looks like a mbuf. * - * @param[in, out] arg - * Context data (struct txq_mp2mr_mbuf_check_data). Contains mempool pointer - * and return value. - * @param[in] start - * Object start address. - * @param[in] end - * Object end address. + * @param[in] mp + * The mempool pointer + * @param[in] arg + * Context data (struct txq_mp2mr_mbuf_check_data). Contains the + * return value. + * @param[in] obj + * Object address. * @param index - * Unused. - * - * @return - * Nonzero value when object is not a mbuf. + * Object index, unused. */ static void -txq_mp2mr_mbuf_check(void *arg, void *start, void *end, - uint32_t index __rte_unused) +txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj, + uint32_t index __rte_unused) { struct txq_mp2mr_mbuf_check_data *data = arg; - struct rte_mbuf *buf = - (void *)((uintptr_t)start + data->mp->header_size); + struct rte_mbuf *buf = obj; - (void)index; /* Check whether mbuf structure fits element size and whether mempool * pointer is valid. */ - if (((uintptr_t)end >= (uintptr_t)(buf + 1)) && - (buf->pool == data->mp)) - data->ret = 0; - else + if (sizeof(*buf) > mp->elt_size || buf->pool != mp) data->ret = -1; } @@ -1112,28 +1360,16 @@ txq_mp2mr_mbuf_check(void *arg, void *start, void *end, * Pointer to TX queue structure. */ static void -txq_mp2mr_iter(const struct rte_mempool *mp, void *arg) +txq_mp2mr_iter(struct rte_mempool *mp, void *arg) { struct txq *txq = arg; struct txq_mp2mr_mbuf_check_data data = { - .mp = mp, - .ret = -1, + .ret = 0, }; - /* Discard empty mempools. */ - if (mp->size == 0) - return; /* Register mempool only if the first element looks like a mbuf. */ - rte_mempool_obj_iter((void *)mp->elt_va_start, - 1, - mp->header_size + mp->elt_size + mp->trailer_size, - 1, - mp->elt_pa, - mp->pg_num, - mp->pg_shift, - txq_mp2mr_mbuf_check, - &data); - if (data.ret) + if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 || + data.ret == -1) return; txq_mp2mr(txq, mp); } @@ -1342,6 +1578,10 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) if (likely(elt->buf != NULL)) { struct rte_mbuf *tmp = elt->buf; +#ifndef NDEBUG + /* Poisoning. */ + memset(elt, 0x66, sizeof(*elt)); +#endif /* Faster than rte_pktmbuf_free(). */ do { struct rte_mbuf *next = NEXT(tmp); @@ -1467,6 +1707,46 @@ stop: return i; } +/** + * DPDK callback for TX in secondary processes. + * + * This function configures all queues from primary process information + * if necessary before reverting to the normal TX burst callback. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static uint16_t +mlx4_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + struct txq *txq = dpdk_txq; + struct priv *priv = mlx4_secondary_data_setup(txq->priv); + struct priv *primary_priv; + unsigned int index; + + if (priv == NULL) + return 0; + primary_priv = + mlx4_secondary_data[priv->dev->data->port_id].primary_priv; + /* Look for queue index in both private structures. */ + for (index = 0; index != priv->txqs_n; ++index) + if (((*primary_priv->txqs)[index] == txq) || + ((*priv->txqs)[index] == txq)) + break; + if (index == priv->txqs_n) + return 0; + txq = (*priv->txqs)[index]; + return priv->dev->tx_pkt_burst(txq, pkts, pkts_n); +} + /** * Configure a TX queue. * @@ -1488,7 +1768,7 @@ static int txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, unsigned int socket, const struct rte_eth_txconf *conf) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct txq tmpl = { .priv = priv, .socket = socket @@ -1504,6 +1784,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, int ret = 0; (void)conf; /* Thresholds configuration (ignored). */ + if (priv == NULL) + return EINVAL; if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) { ERROR("%p: invalid number of TX descriptors (must be a" " multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N); @@ -1682,6 +1964,8 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct txq *txq = (*priv->txqs)[idx]; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); @@ -1737,6 +2021,8 @@ mlx4_tx_queue_release(void *dpdk_txq) struct priv *priv; unsigned int i; + if (mlx4_is_secondary()) + return; if (txq == NULL) return; priv = txq->priv; @@ -2587,6 +2873,8 @@ rxq_cleanup(struct rxq *rxq) * @param flags * RX completion flags returned by poll_length_flags(). * + * @note: fix mlx4_dev_supported_ptypes_get() if any change here. + * * @return * Packet type for struct rte_mbuf. */ @@ -2766,7 +3054,7 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) * cacheline while allocating rep. */ rte_prefetch0(seg); - rep = __rte_mbuf_raw_alloc(rxq->mp); + rep = rte_mbuf_raw_alloc(rxq->mp); if (unlikely(rep == NULL)) { /* * Unable to allocate a replacement mbuf, @@ -2965,7 +3253,7 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) if (ret == 0) break; len = ret; - rep = __rte_mbuf_raw_alloc(rxq->mp); + rep = rte_mbuf_raw_alloc(rxq->mp); if (unlikely(rep == NULL)) { /* * Unable to allocate a replacement mbuf, @@ -3035,6 +3323,46 @@ repost: return pkts_ret; } +/** + * DPDK callback for RX in secondary processes. + * + * This function configures all queues from primary process information + * if necessary before reverting to the normal RX burst callback. + * + * @param dpdk_rxq + * Generic pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +static uint16_t +mlx4_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + struct rxq *rxq = dpdk_rxq; + struct priv *priv = mlx4_secondary_data_setup(rxq->priv); + struct priv *primary_priv; + unsigned int index; + + if (priv == NULL) + return 0; + primary_priv = + mlx4_secondary_data[priv->dev->data->port_id].primary_priv; + /* Look for queue index in both private structures. */ + for (index = 0; index != priv->rxqs_n; ++index) + if (((*primary_priv->rxqs)[index] == rxq) || + ((*priv->rxqs)[index] == rxq)) + break; + if (index == priv->rxqs_n) + return 0; + rxq = (*priv->rxqs)[index]; + return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n); +} + /** * Allocate a Queue Pair. * Optionally setup inline receive if supported. @@ -3137,7 +3465,8 @@ rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, attr.qpg.qpg_type = IBV_EXP_QPG_PARENT; /* TSS isn't necessary. */ attr.qpg.parent_attrib.tss_child_count = 0; - attr.qpg.parent_attrib.rss_child_count = priv->rxqs_n; + attr.qpg.parent_attrib.rss_child_count = + rte_align32pow2(priv->rxqs_n + 1) >> 1; DEBUG("initializing parent RSS queue"); } else { attr.qpg.qpg_type = IBV_EXP_QPG_CHILD_RX; @@ -3360,6 +3689,9 @@ skip_rtr: * Number of descriptors to configure in queue. * @param socket * NUMA socket on which memory must be allocated. + * @param inactive + * If true, the queue is disabled because its index is higher or + * equal to the real number of queues, which must be a power of 2. * @param[in] conf * Thresholds parameters. * @param mp @@ -3370,7 +3702,7 @@ skip_rtr: */ static int rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, - unsigned int socket, const struct rte_eth_rxconf *conf, + unsigned int socket, int inactive, const struct rte_eth_rxconf *conf, struct rte_mempool *mp) { struct priv *priv = dev->data->dev_private; @@ -3434,11 +3766,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, DEBUG("%p: %s scattered packets support (%u WRs)", (void *)dev, (tmpl.sp ? "enabling" : "disabling"), desc); /* Use the entire RX mempool as the memory region. */ - tmpl.mr = ibv_reg_mr(priv->pd, - (void *)mp->elt_va_start, - (mp->elt_va_end - mp->elt_va_start), - (IBV_ACCESS_LOCAL_WRITE | - IBV_ACCESS_REMOTE_WRITE)); + tmpl.mr = mlx4_mp2mr(priv->pd, mp); if (tmpl.mr == NULL) { ret = EINVAL; ERROR("%p: MR creation failure: %s", @@ -3475,7 +3803,7 @@ skip_mr: DEBUG("priv->device_attr.max_sge is %d", priv->device_attr.max_sge); #ifdef RSS_SUPPORT - if (priv->rss) + if (priv->rss && !inactive) tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent, tmpl.rd); else @@ -3611,8 +3939,11 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, { struct priv *priv = dev->data->dev_private; struct rxq *rxq = (*priv->rxqs)[idx]; + int inactive = 0; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); @@ -3640,7 +3971,9 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, return -ENOMEM; } } - ret = rxq_setup(dev, rxq, desc, socket, conf, mp); + if (idx >= rte_align32pow2(priv->rxqs_n + 1) >> 1) + inactive = 1; + ret = rxq_setup(dev, rxq, desc, socket, inactive, conf, mp); if (ret) rte_free(rxq); else { @@ -3671,6 +4004,8 @@ mlx4_rx_queue_release(void *dpdk_rxq) struct priv *priv; unsigned int i; + if (mlx4_is_secondary()) + return; if (rxq == NULL) return; priv = rxq->priv; @@ -3710,6 +4045,8 @@ mlx4_dev_start(struct rte_eth_dev *dev) unsigned int r; struct rxq *rxq; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); if (priv->started) { priv_unlock(priv); @@ -3774,6 +4111,8 @@ mlx4_dev_stop(struct rte_eth_dev *dev) unsigned int r; struct rxq *rxq; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->started) { priv_unlock(priv); @@ -3864,10 +4203,12 @@ priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); static void mlx4_dev_close(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); void *tmp; unsigned int i; + if (priv == NULL) + return; priv_lock(priv); DEBUG("%p: closing device \"%s\"", (void *)dev, @@ -3929,10 +4270,12 @@ mlx4_dev_close(struct rte_eth_dev *dev) static void mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); unsigned int max; char ifname[IF_NAMESIZE]; + if (priv == NULL) + return; priv_lock(priv); /* FIXME: we should ask the device for these values. */ info->min_rx_bufsize = 32; @@ -3964,9 +4307,33 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) 0); if (priv_get_ifname(priv, &ifname) == 0) info->if_index = if_nametoindex(ifname); + info->speed_capa = + ETH_LINK_SPEED_1G | + ETH_LINK_SPEED_10G | + ETH_LINK_SPEED_20G | + ETH_LINK_SPEED_40G | + ETH_LINK_SPEED_56G; priv_unlock(priv); } +static const uint32_t * +mlx4_dev_supported_ptypes_get(struct rte_eth_dev *dev) +{ + static const uint32_t ptypes[] = { + /* refers to rxq_cq_to_pkt_type() */ + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_INNER_L3_IPV4, + RTE_PTYPE_INNER_L3_IPV6, + RTE_PTYPE_UNKNOWN + }; + + if (dev->rx_pkt_burst == mlx4_rx_burst || + dev->rx_pkt_burst == mlx4_rx_burst_sp) + return ptypes; + return NULL; +} + /** * DPDK callback to get device statistics. * @@ -3978,11 +4345,13 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) static void mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct rte_eth_stats tmp = {0}; unsigned int i; unsigned int idx; + if (priv == NULL) + return; priv_lock(priv); /* Add software counters. */ for (i = 0; (i != priv->rxqs_n); ++i) { @@ -4041,10 +4410,12 @@ mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) static void mlx4_stats_reset(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); unsigned int i; unsigned int idx; + if (priv == NULL) + return; priv_lock(priv); for (i = 0; (i != priv->rxqs_n); ++i) { if ((*priv->rxqs)[i] == NULL) @@ -4079,6 +4450,8 @@ mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) { struct priv *priv = dev->data->dev_private; + if (mlx4_is_secondary()) + return; priv_lock(priv); DEBUG("%p: removing MAC address from index %" PRIu32, (void *)dev, index); @@ -4108,6 +4481,8 @@ mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, { struct priv *priv = dev->data->dev_private; + if (mlx4_is_secondary()) + return; (void)vmdq; priv_lock(priv); DEBUG("%p: adding MAC address at index %" PRIu32, @@ -4122,6 +4497,22 @@ end: priv_unlock(priv); } +/** + * DPDK callback to set the primary MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + */ +static void +mlx4_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + DEBUG("%p: setting primary MAC address", (void *)dev); + mlx4_mac_addr_remove(dev, 0); + mlx4_mac_addr_add(dev, mac_addr, 0, 0); +} + /** * DPDK callback to enable promiscuous mode. * @@ -4135,6 +4526,8 @@ mlx4_promiscuous_enable(struct rte_eth_dev *dev) unsigned int i; int ret; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (priv->promisc) { priv_unlock(priv); @@ -4181,6 +4574,8 @@ mlx4_promiscuous_disable(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; unsigned int i; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->promisc) { priv_unlock(priv); @@ -4211,6 +4606,8 @@ mlx4_allmulticast_enable(struct rte_eth_dev *dev) unsigned int i; int ret; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (priv->allmulti) { priv_unlock(priv); @@ -4257,6 +4654,8 @@ mlx4_allmulticast_disable(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; unsigned int i; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->allmulti) { priv_unlock(priv); @@ -4285,7 +4684,7 @@ end: static int mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct ethtool_cmd edata = { .cmd = ETHTOOL_GSET }; @@ -4293,6 +4692,8 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) struct rte_eth_link dev_link; int link_speed = 0; + if (priv == NULL) + return -EINVAL; (void)wait_to_complete; if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); @@ -4314,6 +4715,8 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) dev_link.link_speed = link_speed; dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); + dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & + ETH_LINK_SPEED_FIXED); if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { /* Link status changed. */ dev->data->dev_link = dev_link; @@ -4334,9 +4737,11 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) static int mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); int ret; + if (priv == NULL) + return -EINVAL; priv_lock(priv); ret = mlx4_link_update_unlocked(dev, wait_to_complete); priv_unlock(priv); @@ -4369,6 +4774,8 @@ mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) = mlx4_rx_burst; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); /* Set kernel interface MTU first. */ if (priv_set_mtu(priv, mtu)) { @@ -4452,6 +4859,8 @@ mlx4_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; ifr.ifr_data = ðpause; priv_lock(priv); if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { @@ -4500,6 +4909,8 @@ mlx4_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; ifr.ifr_data = ðpause; ethpause.autoneg = fc_conf->autoneg; if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || @@ -4639,6 +5050,8 @@ mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) struct priv *priv = dev->data->dev_private; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); ret = vlan_filter_set(dev, vlan_id, on); priv_unlock(priv); @@ -4660,6 +5073,7 @@ static const struct eth_dev_ops mlx4_dev_ops = { .stats_reset = mlx4_stats_reset, .queue_stats_mapping_set = NULL, .dev_infos_get = mlx4_dev_infos_get, + .dev_supported_ptypes_get = mlx4_dev_supported_ptypes_get, .vlan_filter_set = mlx4_vlan_filter_set, .vlan_tpid_set = NULL, .vlan_strip_queue_set = NULL, @@ -4675,9 +5089,8 @@ static const struct eth_dev_ops mlx4_dev_ops = { .priority_flow_ctrl_set = NULL, .mac_addr_remove = mlx4_mac_addr_remove, .mac_addr_add = mlx4_mac_addr_add, + .mac_addr_set = mlx4_mac_addr_set, .mtu_set = mlx4_dev_set_mtu, - .udp_tunnel_add = NULL, - .udp_tunnel_del = NULL, }; /** @@ -5058,7 +5471,7 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct ibv_port_attr port_attr; struct ibv_pd *pd = NULL; struct priv *priv = NULL; - struct rte_eth_dev *eth_dev; + struct rte_eth_dev *eth_dev = NULL; #ifdef HAVE_EXP_QUERY_DEVICE struct ibv_exp_device_attr exp_device_attr; #endif /* HAVE_EXP_QUERY_DEVICE */ @@ -5083,6 +5496,13 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ERROR("port query failed: %s", strerror(err)); goto port_error; } + + if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { + ERROR("port %d is not configured in Ethernet mode", + port); + goto port_error; + } + if (port_attr.state != IBV_PORT_ACTIVE) DEBUG("port %d is not active: \"%s\" (%d)", port, ibv_port_state_str(port_attr.state), @@ -5234,18 +5654,46 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) goto port_error; } - eth_dev->data->dev_private = priv; + /* Secondary processes have to use local storage for their + * private data as well as a copy of eth_dev->data, but this + * pointer must not be modified before burst functions are + * actually called. */ + if (mlx4_is_secondary()) { + struct mlx4_secondary_data *sd = + &mlx4_secondary_data[eth_dev->data->port_id]; + + sd->primary_priv = eth_dev->data->dev_private; + if (sd->primary_priv == NULL) { + ERROR("no private data for port %u", + eth_dev->data->port_id); + err = EINVAL; + goto port_error; + } + sd->shared_dev_data = eth_dev->data; + rte_spinlock_init(&sd->lock); + memcpy(sd->data.name, sd->shared_dev_data->name, + sizeof(sd->data.name)); + sd->data.dev_private = priv; + sd->data.rx_mbuf_alloc_failed = 0; + sd->data.mtu = ETHER_MTU; + sd->data.port_id = sd->shared_dev_data->port_id; + sd->data.mac_addrs = priv->mac; + eth_dev->tx_pkt_burst = mlx4_tx_burst_secondary_setup; + eth_dev->rx_pkt_burst = mlx4_rx_burst_secondary_setup; + } else { + eth_dev->data->dev_private = priv; + eth_dev->data->rx_mbuf_alloc_failed = 0; + eth_dev->data->mtu = ETHER_MTU; + eth_dev->data->mac_addrs = priv->mac; + } eth_dev->pci_dev = pci_dev; rte_eth_copy_pci_info(eth_dev, pci_dev); eth_dev->driver = &mlx4_driver; - eth_dev->data->rx_mbuf_alloc_failed = 0; - eth_dev->data->mtu = ETHER_MTU; priv->dev = eth_dev; eth_dev->dev_ops = &mlx4_dev_ops; - eth_dev->data->mac_addrs = priv->mac; TAILQ_INIT(ð_dev->link_intr_cbs); /* Bring Ethernet device up. */ @@ -5259,6 +5707,8 @@ port_error: claim_zero(ibv_dealloc_pd(pd)); if (ctx) claim_zero(ibv_close_device(ctx)); + if (eth_dev) + rte_eth_dev_release_port(eth_dev); break; } @@ -5326,6 +5776,8 @@ rte_mlx4_pmd_init(const char *name, const char *args) { (void)name; (void)args; + + RTE_BUILD_BUG_ON(sizeof(wr_id_t) != sizeof(uint64_t)); /* * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use * huge pages. Calling ibv_fork_init() during init allows