X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4.c;h=cc4e9aa31cddc47882854483f617b36f6dc6fcc3;hb=cc5436bccb34145ba63ea93a96664a90e69fba86;hp=75543bb835d14de5da798f787f42ba410e7acefc;hpb=2921677f15bf68dc7bd5d6fec2eb1f5d94bc32cf;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index 75543bb835..cc4e9aa31c 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -73,7 +73,6 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-pedantic" #endif -#include #include #include #include @@ -87,6 +86,7 @@ #include #include #include +#include #ifdef PEDANTIC #pragma GCC diagnostic error "-pedantic" #endif @@ -132,15 +132,6 @@ typedef union { #define WR_ID(o) (((wr_id_t *)&(o))->data) -/* Compile-time check. */ -static inline void wr_id_t_check(void) -{ - wr_id_t check[1 + (2 * -!(sizeof(wr_id_t) == sizeof(uint64_t)))]; - - (void)check; - (void)wr_id_t_check; -} - /* Transpose flags. Useful to convert IBV to DPDK flags. */ #define TRANSPOSE(val, from, to) \ (((from) >= (to)) ? \ @@ -229,7 +220,7 @@ typedef uint8_t linear_t[16384]; struct txq { struct priv *priv; /* Back pointer to private data. */ struct { - struct rte_mempool *mp; /* Cached Memory Pool. */ + const struct rte_mempool *mp; /* Cached Memory Pool. */ struct ibv_mr *mr; /* Memory Region (for mp). */ uint32_t lkey; /* mr->lkey */ } mp2mr[MLX4_PMD_TX_MP_CACHE]; /* MP to MR translation table. */ @@ -299,6 +290,46 @@ struct priv { rte_spinlock_t lock; /* Lock for control functions. */ }; +/* Local storage for secondary process data. */ +struct mlx4_secondary_data { + struct rte_eth_dev_data data; /* Local device data. */ + struct priv *primary_priv; /* Private structure from primary. */ + struct rte_eth_dev_data *shared_dev_data; /* Shared device data. */ + rte_spinlock_t lock; /* Port configuration lock. */ +} mlx4_secondary_data[RTE_MAX_ETHPORTS]; + +/** + * Check if running as a secondary process. + * + * @return + * Nonzero if running as a secondary process. + */ +static inline int +mlx4_is_secondary(void) +{ + return rte_eal_process_type() != RTE_PROC_PRIMARY; +} + +/** + * Return private structure associated with an Ethernet device. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * Pointer to private structure. + */ +static struct priv * +mlx4_get_priv(struct rte_eth_dev *dev) +{ + struct mlx4_secondary_data *sd; + + if (!mlx4_is_secondary()) + return dev->data->dev_private; + sd = &mlx4_secondary_data[dev->data->port_id]; + return sd->data.dev_private; +} + /** * Lock private structure to protect it from concurrent access in the * control path. @@ -658,6 +689,13 @@ priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) /* Device configuration. */ +static int +txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf); + +static void +txq_cleanup(struct txq *txq); + static int rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, @@ -696,6 +734,12 @@ dev_configure(struct rte_eth_dev *dev) } if (rxqs_n == priv->rxqs_n) return 0; + if ((rxqs_n & (rxqs_n - 1)) != 0) { + ERROR("%p: invalid number of RX queues (%u)," + " must be a power of 2", + (void *)dev, rxqs_n); + return EINVAL; + } INFO("%p: RX queues number update: %u -> %u", (void *)dev, priv->rxqs_n, rxqs_n); /* If RSS is enabled, disable it first. */ @@ -756,6 +800,8 @@ mlx4_dev_configure(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); ret = dev_configure(dev); assert(ret >= 0); @@ -763,6 +809,160 @@ mlx4_dev_configure(struct rte_eth_dev *dev) return -ret; } +static uint16_t mlx4_tx_burst(void *, struct rte_mbuf **, uint16_t); +static uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t); + +/** + * Configure secondary process queues from a private data pointer (primary + * or secondary) and update burst callbacks. Can take place only once. + * + * All queues must have been previously created by the primary process to + * avoid undefined behavior. + * + * @param priv + * Private data pointer from either primary or secondary process. + * + * @return + * Private data pointer from secondary process, NULL in case of error. + */ +static struct priv * +mlx4_secondary_data_setup(struct priv *priv) +{ + unsigned int port_id = 0; + struct mlx4_secondary_data *sd; + void **tx_queues; + void **rx_queues; + unsigned int nb_tx_queues; + unsigned int nb_rx_queues; + unsigned int i; + + /* priv must be valid at this point. */ + assert(priv != NULL); + /* priv->dev must also be valid but may point to local memory from + * another process, possibly with the same address and must not + * be dereferenced yet. */ + assert(priv->dev != NULL); + /* Determine port ID by finding out where priv comes from. */ + while (1) { + sd = &mlx4_secondary_data[port_id]; + rte_spinlock_lock(&sd->lock); + /* Primary process? */ + if (sd->primary_priv == priv) + break; + /* Secondary process? */ + if (sd->data.dev_private == priv) + break; + rte_spinlock_unlock(&sd->lock); + if (++port_id == RTE_DIM(mlx4_secondary_data)) + port_id = 0; + } + /* Switch to secondary private structure. If private data has already + * been updated by another thread, there is nothing else to do. */ + priv = sd->data.dev_private; + if (priv->dev->data == &sd->data) + goto end; + /* Sanity checks. Secondary private structure is supposed to point + * to local eth_dev, itself still pointing to the shared device data + * structure allocated by the primary process. */ + assert(sd->shared_dev_data != &sd->data); + assert(sd->data.nb_tx_queues == 0); + assert(sd->data.tx_queues == NULL); + assert(sd->data.nb_rx_queues == 0); + assert(sd->data.rx_queues == NULL); + assert(priv != sd->primary_priv); + assert(priv->dev->data == sd->shared_dev_data); + assert(priv->txqs_n == 0); + assert(priv->txqs == NULL); + assert(priv->rxqs_n == 0); + assert(priv->rxqs == NULL); + nb_tx_queues = sd->shared_dev_data->nb_tx_queues; + nb_rx_queues = sd->shared_dev_data->nb_rx_queues; + /* Allocate local storage for queues. */ + tx_queues = rte_zmalloc("secondary ethdev->tx_queues", + sizeof(sd->data.tx_queues[0]) * nb_tx_queues, + RTE_CACHE_LINE_SIZE); + rx_queues = rte_zmalloc("secondary ethdev->rx_queues", + sizeof(sd->data.rx_queues[0]) * nb_rx_queues, + RTE_CACHE_LINE_SIZE); + if (tx_queues == NULL || rx_queues == NULL) + goto error; + /* Lock to prevent control operations during setup. */ + priv_lock(priv); + /* TX queues. */ + for (i = 0; i != nb_tx_queues; ++i) { + struct txq *primary_txq = (*sd->primary_priv->txqs)[i]; + struct txq *txq; + + if (primary_txq == NULL) + continue; + txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0, + primary_txq->socket); + if (txq != NULL) { + if (txq_setup(priv->dev, + txq, + primary_txq->elts_n * MLX4_PMD_SGE_WR_N, + primary_txq->socket, + NULL) == 0) { + txq->stats.idx = primary_txq->stats.idx; + tx_queues[i] = txq; + continue; + } + rte_free(txq); + } + while (i) { + txq = tx_queues[--i]; + txq_cleanup(txq); + rte_free(txq); + } + goto error; + } + /* RX queues. */ + for (i = 0; i != nb_rx_queues; ++i) { + struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i]; + + if (primary_rxq == NULL) + continue; + /* Not supported yet. */ + rx_queues[i] = NULL; + } + /* Update everything. */ + priv->txqs = (void *)tx_queues; + priv->txqs_n = nb_tx_queues; + priv->rxqs = (void *)rx_queues; + priv->rxqs_n = nb_rx_queues; + sd->data.rx_queues = rx_queues; + sd->data.tx_queues = tx_queues; + sd->data.nb_rx_queues = nb_rx_queues; + sd->data.nb_tx_queues = nb_tx_queues; + sd->data.dev_link = sd->shared_dev_data->dev_link; + sd->data.mtu = sd->shared_dev_data->mtu; + memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state, + sizeof(sd->data.rx_queue_state)); + memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state, + sizeof(sd->data.tx_queue_state)); + sd->data.dev_flags = sd->shared_dev_data->dev_flags; + /* Use local data from now on. */ + rte_mb(); + priv->dev->data = &sd->data; + rte_mb(); + priv->dev->tx_pkt_burst = mlx4_tx_burst; + priv->dev->rx_pkt_burst = removed_rx_burst; + priv_unlock(priv); +end: + /* More sanity checks. */ + assert(priv->dev->tx_pkt_burst == mlx4_tx_burst); + assert(priv->dev->rx_pkt_burst == removed_rx_burst); + assert(priv->dev->data == &sd->data); + rte_spinlock_unlock(&sd->lock); + return priv; +error: + priv_unlock(priv); + rte_free(tx_queues); + rte_free(rx_queues); + rte_spinlock_unlock(&sd->lock); + return NULL; +} + /* TX queues handling. */ /** @@ -984,6 +1184,52 @@ txq_complete(struct txq *txq) return 0; } +/* For best performance, this function should not be inlined. */ +static struct ibv_mr *mlx4_mp2mr(struct ibv_pd *, const struct rte_mempool *) + __attribute__((noinline)); + +/** + * Register mempool as a memory region. + * + * @param pd + * Pointer to protection domain. + * @param mp + * Pointer to memory pool. + * + * @return + * Memory region pointer, NULL in case of error. + */ +static struct ibv_mr * +mlx4_mp2mr(struct ibv_pd *pd, const struct rte_mempool *mp) +{ + const struct rte_memseg *ms = rte_eal_get_physmem_layout(); + uintptr_t start = mp->elt_va_start; + uintptr_t end = mp->elt_va_end; + unsigned int i; + + DEBUG("mempool %p area start=%p end=%p size=%zu", + (const void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); + /* Round start and end to page boundary if found in memory segments. */ + for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { + uintptr_t addr = (uintptr_t)ms[i].addr; + size_t len = ms[i].len; + unsigned int align = ms[i].hugepage_sz; + + if ((start > addr) && (start < addr + len)) + start = RTE_ALIGN_FLOOR(start, align); + if ((end > addr) && (end < addr + len)) + end = RTE_ALIGN_CEIL(end, align); + } + DEBUG("mempool %p using start=%p end=%p size=%zu for MR", + (const void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); + return ibv_reg_mr(pd, + (void *)start, + end - start, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); +} + /** * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which * the cloned mbuf is allocated is returned instead. @@ -1016,7 +1262,7 @@ txq_mb2mp(struct rte_mbuf *buf) * mr->lkey on success, (uint32_t)-1 on failure. */ static uint32_t -txq_mp2mr(struct txq *txq, struct rte_mempool *mp) +txq_mp2mr(struct txq *txq, const struct rte_mempool *mp) { unsigned int i; struct ibv_mr *mr; @@ -1033,11 +1279,9 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) } } /* Add a new entry, register MR first. */ - DEBUG("%p: discovered new memory pool %p", (void *)txq, (void *)mp); - mr = ibv_reg_mr(txq->priv->pd, - (void *)mp->elt_va_start, - (mp->elt_va_end - mp->elt_va_start), - (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE)); + DEBUG("%p: discovered new memory pool \"%s\" (%p)", + (void *)txq, mp->name, (const void *)mp); + mr = mlx4_mp2mr(txq->priv->pd, mp); if (unlikely(mr == NULL)) { DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.", (void *)txq); @@ -1048,7 +1292,7 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) DEBUG("%p: MR <-> MP table full, dropping oldest entry.", (void *)txq); --i; - claim_zero(ibv_dereg_mr(txq->mp2mr[i].mr)); + claim_zero(ibv_dereg_mr(txq->mp2mr[0].mr)); memmove(&txq->mp2mr[0], &txq->mp2mr[1], (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); } @@ -1056,11 +1300,87 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) txq->mp2mr[i].mp = mp; txq->mp2mr[i].mr = mr; txq->mp2mr[i].lkey = mr->lkey; - DEBUG("%p: new MR lkey for MP %p: 0x%08" PRIu32, - (void *)txq, (void *)mp, txq->mp2mr[i].lkey); + DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, + (void *)txq, mp->name, (const void *)mp, txq->mp2mr[i].lkey); return txq->mp2mr[i].lkey; } +struct txq_mp2mr_mbuf_check_data { + const struct rte_mempool *mp; + int ret; +}; + +/** + * Callback function for rte_mempool_obj_iter() to check whether a given + * mempool object looks like a mbuf. + * + * @param[in, out] arg + * Context data (struct txq_mp2mr_mbuf_check_data). Contains mempool pointer + * and return value. + * @param[in] start + * Object start address. + * @param[in] end + * Object end address. + * @param index + * Unused. + * + * @return + * Nonzero value when object is not a mbuf. + */ +static void +txq_mp2mr_mbuf_check(void *arg, void *start, void *end, + uint32_t index __rte_unused) +{ + struct txq_mp2mr_mbuf_check_data *data = arg; + struct rte_mbuf *buf = + (void *)((uintptr_t)start + data->mp->header_size); + + (void)index; + /* Check whether mbuf structure fits element size and whether mempool + * pointer is valid. */ + if (((uintptr_t)end >= (uintptr_t)(buf + 1)) && + (buf->pool == data->mp)) + data->ret = 0; + else + data->ret = -1; +} + +/** + * Iterator function for rte_mempool_walk() to register existing mempools and + * fill the MP to MR cache of a TX queue. + * + * @param[in] mp + * Memory Pool to register. + * @param *arg + * Pointer to TX queue structure. + */ +static void +txq_mp2mr_iter(const struct rte_mempool *mp, void *arg) +{ + struct txq *txq = arg; + struct txq_mp2mr_mbuf_check_data data = { + .mp = mp, + .ret = -1, + }; + + /* Discard empty mempools. */ + if (mp->size == 0) + return; + /* Register mempool only if the first element looks like a mbuf. */ + rte_mempool_obj_iter((void *)mp->elt_va_start, + 1, + mp->header_size + mp->elt_size + mp->trailer_size, + 1, + mp->elt_pa, + mp->pg_num, + mp->pg_shift, + txq_mp2mr_mbuf_check, + &data); + if (data.ret) + return; + txq_mp2mr(txq, mp); +} + #if MLX4_PMD_SGE_WR_N > 1 /** @@ -1229,7 +1549,6 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct txq *txq = (struct txq *)dpdk_txq; unsigned int elts_head = txq->elts_head; - const unsigned int elts_tail = txq->elts_tail; const unsigned int elts_n = txq->elts_n; unsigned int elts_comp_cd = txq->elts_comp_cd; unsigned int elts_comp = 0; @@ -1239,7 +1558,7 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) assert(elts_comp_cd != 0); txq_complete(txq); - max = (elts_n - (elts_head - elts_tail)); + max = (elts_n - (elts_head - txq->elts_tail)); if (max > elts_n) max -= elts_n; assert(max >= 1); @@ -1391,6 +1710,46 @@ stop: return i; } +/** + * DPDK callback for TX in secondary processes. + * + * This function configures all queues from primary process information + * if necessary before reverting to the normal TX burst callback. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static uint16_t +mlx4_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + struct txq *txq = dpdk_txq; + struct priv *priv = mlx4_secondary_data_setup(txq->priv); + struct priv *primary_priv; + unsigned int index; + + if (priv == NULL) + return 0; + primary_priv = + mlx4_secondary_data[priv->dev->data->port_id].primary_priv; + /* Look for queue index in both private structures. */ + for (index = 0; index != priv->txqs_n; ++index) + if (((*primary_priv->txqs)[index] == txq) || + ((*priv->txqs)[index] == txq)) + break; + if (index == priv->txqs_n) + return 0; + txq = (*priv->txqs)[index]; + return priv->dev->tx_pkt_burst(txq, pkts, pkts_n); +} + /** * Configure a TX queue. * @@ -1412,7 +1771,7 @@ static int txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, unsigned int socket, const struct rte_eth_txconf *conf) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct txq tmpl = { .priv = priv, .socket = socket @@ -1428,6 +1787,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, int ret = 0; (void)conf; /* Thresholds configuration (ignored). */ + if (priv == NULL) + return EINVAL; if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) { ERROR("%p: invalid number of TX descriptors (must be a" " multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N); @@ -1571,6 +1932,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, txq_cleanup(txq); *txq = tmpl; DEBUG("%p: txq updated with %p", (void *)txq, (void *)&tmpl); + /* Pre-register known mempools. */ + rte_mempool_walk(txq_mp2mr_iter, txq); assert(ret == 0); return 0; error: @@ -1604,6 +1967,8 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct txq *txq = (*priv->txqs)[idx]; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); @@ -1659,6 +2024,8 @@ mlx4_tx_queue_release(void *dpdk_txq) struct priv *priv; unsigned int i; + if (mlx4_is_secondary()) + return; if (txq == NULL) return; priv = txq->priv; @@ -2957,6 +3324,46 @@ repost: return pkts_ret; } +/** + * DPDK callback for RX in secondary processes. + * + * This function configures all queues from primary process information + * if necessary before reverting to the normal RX burst callback. + * + * @param dpdk_rxq + * Generic pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +static uint16_t +mlx4_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + struct rxq *rxq = dpdk_rxq; + struct priv *priv = mlx4_secondary_data_setup(rxq->priv); + struct priv *primary_priv; + unsigned int index; + + if (priv == NULL) + return 0; + primary_priv = + mlx4_secondary_data[priv->dev->data->port_id].primary_priv; + /* Look for queue index in both private structures. */ + for (index = 0; index != priv->rxqs_n; ++index) + if (((*primary_priv->rxqs)[index] == rxq) || + ((*priv->rxqs)[index] == rxq)) + break; + if (index == priv->rxqs_n) + return 0; + rxq = (*priv->rxqs)[index]; + return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n); +} + /** * Allocate a Queue Pair. * Optionally setup inline receive if supported. @@ -3356,11 +3763,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, DEBUG("%p: %s scattered packets support (%u WRs)", (void *)dev, (tmpl.sp ? "enabling" : "disabling"), desc); /* Use the entire RX mempool as the memory region. */ - tmpl.mr = ibv_reg_mr(priv->pd, - (void *)mp->elt_va_start, - (mp->elt_va_end - mp->elt_va_start), - (IBV_ACCESS_LOCAL_WRITE | - IBV_ACCESS_REMOTE_WRITE)); + tmpl.mr = mlx4_mp2mr(priv->pd, mp); if (tmpl.mr == NULL) { ret = EINVAL; ERROR("%p: MR creation failure: %s", @@ -3535,6 +3938,8 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct rxq *rxq = (*priv->rxqs)[idx]; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); @@ -3593,6 +3998,8 @@ mlx4_rx_queue_release(void *dpdk_rxq) struct priv *priv; unsigned int i; + if (mlx4_is_secondary()) + return; if (rxq == NULL) return; priv = rxq->priv; @@ -3632,6 +4039,8 @@ mlx4_dev_start(struct rte_eth_dev *dev) unsigned int r; struct rxq *rxq; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); if (priv->started) { priv_unlock(priv); @@ -3696,6 +4105,8 @@ mlx4_dev_stop(struct rte_eth_dev *dev) unsigned int r; struct rxq *rxq; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->started) { priv_unlock(priv); @@ -3786,10 +4197,12 @@ priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); static void mlx4_dev_close(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); void *tmp; unsigned int i; + if (priv == NULL) + return; priv_lock(priv); DEBUG("%p: closing device \"%s\"", (void *)dev, @@ -3851,10 +4264,12 @@ mlx4_dev_close(struct rte_eth_dev *dev) static void mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); unsigned int max; char ifname[IF_NAMESIZE]; + if (priv == NULL) + return; priv_lock(priv); /* FIXME: we should ask the device for these values. */ info->min_rx_bufsize = 32; @@ -3900,11 +4315,13 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) static void mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct rte_eth_stats tmp = {0}; unsigned int i; unsigned int idx; + if (priv == NULL) + return; priv_lock(priv); /* Add software counters. */ for (i = 0; (i != priv->rxqs_n); ++i) { @@ -3963,10 +4380,12 @@ mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) static void mlx4_stats_reset(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); unsigned int i; unsigned int idx; + if (priv == NULL) + return; priv_lock(priv); for (i = 0; (i != priv->rxqs_n); ++i) { if ((*priv->rxqs)[i] == NULL) @@ -4001,6 +4420,8 @@ mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) { struct priv *priv = dev->data->dev_private; + if (mlx4_is_secondary()) + return; priv_lock(priv); DEBUG("%p: removing MAC address from index %" PRIu32, (void *)dev, index); @@ -4030,6 +4451,8 @@ mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, { struct priv *priv = dev->data->dev_private; + if (mlx4_is_secondary()) + return; (void)vmdq; priv_lock(priv); DEBUG("%p: adding MAC address at index %" PRIu32, @@ -4044,6 +4467,22 @@ end: priv_unlock(priv); } +/** + * DPDK callback to set the primary MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + */ +static void +mlx4_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + DEBUG("%p: setting primary MAC address", (void *)dev); + mlx4_mac_addr_remove(dev, 0); + mlx4_mac_addr_add(dev, mac_addr, 0, 0); +} + /** * DPDK callback to enable promiscuous mode. * @@ -4057,6 +4496,8 @@ mlx4_promiscuous_enable(struct rte_eth_dev *dev) unsigned int i; int ret; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (priv->promisc) { priv_unlock(priv); @@ -4103,6 +4544,8 @@ mlx4_promiscuous_disable(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; unsigned int i; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->promisc) { priv_unlock(priv); @@ -4133,6 +4576,8 @@ mlx4_allmulticast_enable(struct rte_eth_dev *dev) unsigned int i; int ret; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (priv->allmulti) { priv_unlock(priv); @@ -4179,6 +4624,8 @@ mlx4_allmulticast_disable(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; unsigned int i; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->allmulti) { priv_unlock(priv); @@ -4207,7 +4654,7 @@ end: static int mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct ethtool_cmd edata = { .cmd = ETHTOOL_GSET }; @@ -4215,6 +4662,8 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) struct rte_eth_link dev_link; int link_speed = 0; + if (priv == NULL) + return -EINVAL; (void)wait_to_complete; if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); @@ -4256,9 +4705,11 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) static int mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); int ret; + if (priv == NULL) + return -EINVAL; priv_lock(priv); ret = mlx4_link_update_unlocked(dev, wait_to_complete); priv_unlock(priv); @@ -4291,6 +4742,8 @@ mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) = mlx4_rx_burst; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); /* Set kernel interface MTU first. */ if (priv_set_mtu(priv, mtu)) { @@ -4374,6 +4827,8 @@ mlx4_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; ifr.ifr_data = ðpause; priv_lock(priv); if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { @@ -4422,6 +4877,8 @@ mlx4_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; ifr.ifr_data = ðpause; ethpause.autoneg = fc_conf->autoneg; if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || @@ -4561,6 +5018,8 @@ mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) struct priv *priv = dev->data->dev_private; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); ret = vlan_filter_set(dev, vlan_id, on); priv_unlock(priv); @@ -4597,9 +5056,8 @@ static const struct eth_dev_ops mlx4_dev_ops = { .priority_flow_ctrl_set = NULL, .mac_addr_remove = mlx4_mac_addr_remove, .mac_addr_add = mlx4_mac_addr_add, + .mac_addr_set = mlx4_mac_addr_set, .mtu_set = mlx4_dev_set_mtu, - .udp_tunnel_add = NULL, - .udp_tunnel_del = NULL, }; /** @@ -4980,7 +5438,7 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct ibv_port_attr port_attr; struct ibv_pd *pd = NULL; struct priv *priv = NULL; - struct rte_eth_dev *eth_dev; + struct rte_eth_dev *eth_dev = NULL; #ifdef HAVE_EXP_QUERY_DEVICE struct ibv_exp_device_attr exp_device_attr; #endif /* HAVE_EXP_QUERY_DEVICE */ @@ -5156,18 +5614,46 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) goto port_error; } - eth_dev->data->dev_private = priv; + /* Secondary processes have to use local storage for their + * private data as well as a copy of eth_dev->data, but this + * pointer must not be modified before burst functions are + * actually called. */ + if (mlx4_is_secondary()) { + struct mlx4_secondary_data *sd = + &mlx4_secondary_data[eth_dev->data->port_id]; + + sd->primary_priv = eth_dev->data->dev_private; + if (sd->primary_priv == NULL) { + ERROR("no private data for port %u", + eth_dev->data->port_id); + err = EINVAL; + goto port_error; + } + sd->shared_dev_data = eth_dev->data; + rte_spinlock_init(&sd->lock); + memcpy(sd->data.name, sd->shared_dev_data->name, + sizeof(sd->data.name)); + sd->data.dev_private = priv; + sd->data.rx_mbuf_alloc_failed = 0; + sd->data.mtu = ETHER_MTU; + sd->data.port_id = sd->shared_dev_data->port_id; + sd->data.mac_addrs = priv->mac; + eth_dev->tx_pkt_burst = mlx4_tx_burst_secondary_setup; + eth_dev->rx_pkt_burst = mlx4_rx_burst_secondary_setup; + } else { + eth_dev->data->dev_private = priv; + eth_dev->data->rx_mbuf_alloc_failed = 0; + eth_dev->data->mtu = ETHER_MTU; + eth_dev->data->mac_addrs = priv->mac; + } eth_dev->pci_dev = pci_dev; rte_eth_copy_pci_info(eth_dev, pci_dev); eth_dev->driver = &mlx4_driver; - eth_dev->data->rx_mbuf_alloc_failed = 0; - eth_dev->data->mtu = ETHER_MTU; priv->dev = eth_dev; eth_dev->dev_ops = &mlx4_dev_ops; - eth_dev->data->mac_addrs = priv->mac; TAILQ_INIT(ð_dev->link_intr_cbs); /* Bring Ethernet device up. */ @@ -5181,6 +5667,8 @@ port_error: claim_zero(ibv_dealloc_pd(pd)); if (ctx) claim_zero(ibv_close_device(ctx)); + if (eth_dev) + rte_eth_dev_release_port(eth_dev); break; } @@ -5248,6 +5736,8 @@ rte_mlx4_pmd_init(const char *name, const char *args) { (void)name; (void)args; + + RTE_BUILD_BUG_ON(sizeof(wr_id_t) != sizeof(uint64_t)); /* * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use * huge pages. Calling ibv_fork_init() during init allows