X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4.c;h=0ec66ec8188976f447e671ff3508d5f226296cb7;hb=b84355e23259b52d9e99322230d9dc83afd7c37f;hp=3dff64d4f6add2ede3bba4e998e74ddd8a3b0997;hpb=9980f81dc2623291b89cf1c281a6a9f116fd2394;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index 3dff64d4f6..0ec66ec818 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -57,6 +57,7 @@ #include #include #include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ @@ -72,7 +73,6 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-pedantic" #endif -#include #include #include #include @@ -85,6 +85,7 @@ #include #include #include +#include #ifdef PEDANTIC #pragma GCC diagnostic error "-pedantic" #endif @@ -130,14 +131,11 @@ typedef union { #define WR_ID(o) (((wr_id_t *)&(o))->data) -/* Compile-time check. */ -static inline void wr_id_t_check(void) -{ - wr_id_t check[1 + (2 * -!(sizeof(wr_id_t) == sizeof(uint64_t)))]; - - (void)check; - (void)wr_id_t_check; -} +/* Transpose flags. Useful to convert IBV to DPDK flags. */ +#define TRANSPOSE(val, from, to) \ + (((from) >= (to)) ? \ + (((val) & (from)) / ((from) / (to))) : \ + (((val) & (from)) * ((to) / (from)))) struct mlx4_rxq_stats { unsigned int idx; /**< Mapping index. */ @@ -196,16 +194,17 @@ struct rxq { struct rxq_elt (*no_sp)[]; /* RX elements. */ } elts; unsigned int sp:1; /* Use scattered RX elements. */ + unsigned int csum:1; /* Enable checksum offloading. */ + unsigned int csum_l2tun:1; /* Same for L2 tunnels. */ uint32_t mb_len; /* Length of a mp-issued mbuf. */ struct mlx4_rxq_stats stats; /* RX queue counters. */ unsigned int socket; /* CPU socket ID for allocations. */ + struct ibv_exp_res_domain *rd; /* Resource Domain. */ }; /* TX element. */ struct txq_elt { - struct ibv_send_wr wr; /* Work Request. */ - struct ibv_sge sges[MLX4_PMD_SGE_WR_N]; /* Scatter/Gather Elements. */ - /* mbuf pointer is derived from WR_ID(wr.wr_id).offset. */ + struct rte_mbuf *buf; }; /* Linear buffer type. It is used when transmitting buffers with too many @@ -220,7 +219,7 @@ typedef uint8_t linear_t[16384]; struct txq { struct priv *priv; /* Back pointer to private data. */ struct { - struct rte_mempool *mp; /* Cached Memory Pool. */ + const struct rte_mempool *mp; /* Cached Memory Pool. */ struct ibv_mr *mr; /* Memory Region (for mp). */ uint32_t lkey; /* mr->lkey */ } mp2mr[MLX4_PMD_TX_MP_CACHE]; /* MP to MR translation table. */ @@ -242,6 +241,7 @@ struct txq { linear_t (*elts_linear)[]; /* Linearized buffers. */ struct ibv_mr *mr_linear; /* Memory Region for linearized buffers. */ unsigned int socket; /* CPU socket ID for allocations. */ + struct ibv_exp_res_domain *rd; /* Resource Domain. */ }; struct priv { @@ -270,8 +270,11 @@ struct priv { unsigned int hw_qpg:1; /* QP groups are supported. */ unsigned int hw_tss:1; /* TSS is supported. */ unsigned int hw_rss:1; /* RSS is supported. */ + unsigned int hw_csum:1; /* Checksum offload is supported. */ + unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */ unsigned int rss:1; /* RSS is enabled. */ unsigned int vf:1; /* This is a VF device. */ + unsigned int pending_alarm:1; /* An alarm is pending. */ #ifdef INLINE_RECV unsigned int inl_recv_size; /* Inline recv size */ #endif @@ -282,9 +285,50 @@ struct priv { unsigned int txqs_n; /* TX queues array size. */ struct rxq *(*rxqs)[]; /* RX queues. */ struct txq *(*txqs)[]; /* TX queues. */ + struct rte_intr_handle intr_handle; /* Interrupt handler. */ rte_spinlock_t lock; /* Lock for control functions. */ }; +/* Local storage for secondary process data. */ +struct mlx4_secondary_data { + struct rte_eth_dev_data data; /* Local device data. */ + struct priv *primary_priv; /* Private structure from primary. */ + struct rte_eth_dev_data *shared_dev_data; /* Shared device data. */ + rte_spinlock_t lock; /* Port configuration lock. */ +} mlx4_secondary_data[RTE_MAX_ETHPORTS]; + +/** + * Check if running as a secondary process. + * + * @return + * Nonzero if running as a secondary process. + */ +static inline int +mlx4_is_secondary(void) +{ + return rte_eal_process_type() != RTE_PROC_PRIMARY; +} + +/** + * Return private structure associated with an Ethernet device. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * Pointer to private structure. + */ +static struct priv * +mlx4_get_priv(struct rte_eth_dev *dev) +{ + struct mlx4_secondary_data *sd; + + if (!mlx4_is_secondary()) + return dev->data->dev_private; + sd = &mlx4_secondary_data[dev->data->port_id]; + return sd->data.dev_private; +} + /** * Lock private structure to protect it from concurrent access in the * control path. @@ -644,6 +688,13 @@ priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) /* Device configuration. */ +static int +txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf); + +static void +txq_cleanup(struct txq *txq); + static int rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, @@ -682,6 +733,12 @@ dev_configure(struct rte_eth_dev *dev) } if (rxqs_n == priv->rxqs_n) return 0; + if ((rxqs_n & (rxqs_n - 1)) != 0) { + ERROR("%p: invalid number of RX queues (%u)," + " must be a power of 2", + (void *)dev, rxqs_n); + return EINVAL; + } INFO("%p: RX queues number update: %u -> %u", (void *)dev, priv->rxqs_n, rxqs_n); /* If RSS is enabled, disable it first. */ @@ -742,6 +799,8 @@ mlx4_dev_configure(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); ret = dev_configure(dev); assert(ret >= 0); @@ -749,6 +808,160 @@ mlx4_dev_configure(struct rte_eth_dev *dev) return -ret; } +static uint16_t mlx4_tx_burst(void *, struct rte_mbuf **, uint16_t); +static uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t); + +/** + * Configure secondary process queues from a private data pointer (primary + * or secondary) and update burst callbacks. Can take place only once. + * + * All queues must have been previously created by the primary process to + * avoid undefined behavior. + * + * @param priv + * Private data pointer from either primary or secondary process. + * + * @return + * Private data pointer from secondary process, NULL in case of error. + */ +static struct priv * +mlx4_secondary_data_setup(struct priv *priv) +{ + unsigned int port_id = 0; + struct mlx4_secondary_data *sd; + void **tx_queues; + void **rx_queues; + unsigned int nb_tx_queues; + unsigned int nb_rx_queues; + unsigned int i; + + /* priv must be valid at this point. */ + assert(priv != NULL); + /* priv->dev must also be valid but may point to local memory from + * another process, possibly with the same address and must not + * be dereferenced yet. */ + assert(priv->dev != NULL); + /* Determine port ID by finding out where priv comes from. */ + while (1) { + sd = &mlx4_secondary_data[port_id]; + rte_spinlock_lock(&sd->lock); + /* Primary process? */ + if (sd->primary_priv == priv) + break; + /* Secondary process? */ + if (sd->data.dev_private == priv) + break; + rte_spinlock_unlock(&sd->lock); + if (++port_id == RTE_DIM(mlx4_secondary_data)) + port_id = 0; + } + /* Switch to secondary private structure. If private data has already + * been updated by another thread, there is nothing else to do. */ + priv = sd->data.dev_private; + if (priv->dev->data == &sd->data) + goto end; + /* Sanity checks. Secondary private structure is supposed to point + * to local eth_dev, itself still pointing to the shared device data + * structure allocated by the primary process. */ + assert(sd->shared_dev_data != &sd->data); + assert(sd->data.nb_tx_queues == 0); + assert(sd->data.tx_queues == NULL); + assert(sd->data.nb_rx_queues == 0); + assert(sd->data.rx_queues == NULL); + assert(priv != sd->primary_priv); + assert(priv->dev->data == sd->shared_dev_data); + assert(priv->txqs_n == 0); + assert(priv->txqs == NULL); + assert(priv->rxqs_n == 0); + assert(priv->rxqs == NULL); + nb_tx_queues = sd->shared_dev_data->nb_tx_queues; + nb_rx_queues = sd->shared_dev_data->nb_rx_queues; + /* Allocate local storage for queues. */ + tx_queues = rte_zmalloc("secondary ethdev->tx_queues", + sizeof(sd->data.tx_queues[0]) * nb_tx_queues, + RTE_CACHE_LINE_SIZE); + rx_queues = rte_zmalloc("secondary ethdev->rx_queues", + sizeof(sd->data.rx_queues[0]) * nb_rx_queues, + RTE_CACHE_LINE_SIZE); + if (tx_queues == NULL || rx_queues == NULL) + goto error; + /* Lock to prevent control operations during setup. */ + priv_lock(priv); + /* TX queues. */ + for (i = 0; i != nb_tx_queues; ++i) { + struct txq *primary_txq = (*sd->primary_priv->txqs)[i]; + struct txq *txq; + + if (primary_txq == NULL) + continue; + txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0, + primary_txq->socket); + if (txq != NULL) { + if (txq_setup(priv->dev, + txq, + primary_txq->elts_n * MLX4_PMD_SGE_WR_N, + primary_txq->socket, + NULL) == 0) { + txq->stats.idx = primary_txq->stats.idx; + tx_queues[i] = txq; + continue; + } + rte_free(txq); + } + while (i) { + txq = tx_queues[--i]; + txq_cleanup(txq); + rte_free(txq); + } + goto error; + } + /* RX queues. */ + for (i = 0; i != nb_rx_queues; ++i) { + struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i]; + + if (primary_rxq == NULL) + continue; + /* Not supported yet. */ + rx_queues[i] = NULL; + } + /* Update everything. */ + priv->txqs = (void *)tx_queues; + priv->txqs_n = nb_tx_queues; + priv->rxqs = (void *)rx_queues; + priv->rxqs_n = nb_rx_queues; + sd->data.rx_queues = rx_queues; + sd->data.tx_queues = tx_queues; + sd->data.nb_rx_queues = nb_rx_queues; + sd->data.nb_tx_queues = nb_tx_queues; + sd->data.dev_link = sd->shared_dev_data->dev_link; + sd->data.mtu = sd->shared_dev_data->mtu; + memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state, + sizeof(sd->data.rx_queue_state)); + memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state, + sizeof(sd->data.tx_queue_state)); + sd->data.dev_flags = sd->shared_dev_data->dev_flags; + /* Use local data from now on. */ + rte_mb(); + priv->dev->data = &sd->data; + rte_mb(); + priv->dev->tx_pkt_burst = mlx4_tx_burst; + priv->dev->rx_pkt_burst = removed_rx_burst; + priv_unlock(priv); +end: + /* More sanity checks. */ + assert(priv->dev->tx_pkt_burst == mlx4_tx_burst); + assert(priv->dev->rx_pkt_burst == removed_rx_burst); + assert(priv->dev->data == &sd->data); + rte_spinlock_unlock(&sd->lock); + return priv; +error: + priv_unlock(priv); + rte_free(tx_queues); + rte_free(rx_queues); + rte_spinlock_unlock(&sd->lock); + return NULL; +} + /* TX queues handling. */ /** @@ -790,14 +1003,8 @@ txq_alloc_elts(struct txq *txq, unsigned int elts_n) } for (i = 0; (i != elts_n); ++i) { struct txq_elt *elt = &(*elts)[i]; - struct ibv_send_wr *wr = &elt->wr; - /* Configure WR. */ - WR_ID(wr->wr_id).id = i; - WR_ID(wr->wr_id).offset = 0; - wr->sg_list = &elt->sges[0]; - wr->opcode = IBV_WR_SEND; - /* Other fields are updated during TX. */ + elt->buf = NULL; } DEBUG("%p: allocated and configured %u WRs", (void *)txq, elts_n); txq->elts_n = elts_n; @@ -856,10 +1063,9 @@ txq_free_elts(struct txq *txq) for (i = 0; (i != elemof(*elts)); ++i) { struct txq_elt *elt = &(*elts)[i]; - if (WR_ID(elt->wr.wr_id).offset == 0) + if (elt->buf == NULL) continue; - rte_pktmbuf_free((void *)((uintptr_t)elt->sges[0].addr - - WR_ID(elt->wr.wr_id).offset)); + rte_pktmbuf_free(elt->buf); } rte_free(elts); } @@ -907,6 +1113,17 @@ txq_cleanup(struct txq *txq) claim_zero(ibv_destroy_qp(txq->qp)); if (txq->cq != NULL) claim_zero(ibv_destroy_cq(txq->cq)); + if (txq->rd != NULL) { + struct ibv_exp_destroy_res_domain_attr attr = { + .comp_mask = 0, + }; + + assert(txq->priv != NULL); + assert(txq->priv->ctx != NULL); + claim_zero(ibv_exp_destroy_res_domain(txq->priv->ctx, + txq->rd, + &attr)); + } for (i = 0; (i != elemof(txq->mp2mr)); ++i) { if (txq->mp2mr[i].mp == NULL) break; @@ -966,6 +1183,24 @@ txq_complete(struct txq *txq) return 0; } +/** + * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which + * the cloned mbuf is allocated is returned instead. + * + * @param buf + * Pointer to mbuf. + * + * @return + * Memory pool where data is located for given mbuf. + */ +static struct rte_mempool * +txq_mb2mp(struct rte_mbuf *buf) +{ + if (unlikely(RTE_MBUF_INDIRECT(buf))) + return rte_mbuf_from_indirect(buf)->pool; + return buf->pool; +} + /** * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[]. * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full, @@ -980,7 +1215,7 @@ txq_complete(struct txq *txq) * mr->lkey on success, (uint32_t)-1 on failure. */ static uint32_t -txq_mp2mr(struct txq *txq, struct rte_mempool *mp) +txq_mp2mr(struct txq *txq, const struct rte_mempool *mp) { unsigned int i; struct ibv_mr *mr; @@ -997,7 +1232,8 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) } } /* Add a new entry, register MR first. */ - DEBUG("%p: discovered new memory pool %p", (void *)txq, (void *)mp); + DEBUG("%p: discovered new memory pool \"%s\" (%p)", + (void *)txq, mp->name, (const void *)mp); mr = ibv_reg_mr(txq->priv->pd, (void *)mp->elt_va_start, (mp->elt_va_end - mp->elt_va_start), @@ -1012,7 +1248,7 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) DEBUG("%p: MR <-> MP table full, dropping oldest entry.", (void *)txq); --i; - claim_zero(ibv_dereg_mr(txq->mp2mr[i].mr)); + claim_zero(ibv_dereg_mr(txq->mp2mr[0].mr)); memmove(&txq->mp2mr[0], &txq->mp2mr[1], (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); } @@ -1020,11 +1256,89 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) txq->mp2mr[i].mp = mp; txq->mp2mr[i].mr = mr; txq->mp2mr[i].lkey = mr->lkey; - DEBUG("%p: new MR lkey for MP %p: 0x%08" PRIu32, - (void *)txq, (void *)mp, txq->mp2mr[i].lkey); + DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, + (void *)txq, mp->name, (const void *)mp, txq->mp2mr[i].lkey); return txq->mp2mr[i].lkey; } +struct txq_mp2mr_mbuf_check_data { + const struct rte_mempool *mp; + int ret; +}; + +/** + * Callback function for rte_mempool_obj_iter() to check whether a given + * mempool object looks like a mbuf. + * + * @param[in, out] arg + * Context data (struct txq_mp2mr_mbuf_check_data). Contains mempool pointer + * and return value. + * @param[in] start + * Object start address. + * @param[in] end + * Object end address. + * @param index + * Unused. + * + * @return + * Nonzero value when object is not a mbuf. + */ +static void +txq_mp2mr_mbuf_check(void *arg, void *start, void *end, + uint32_t index __rte_unused) +{ + struct txq_mp2mr_mbuf_check_data *data = arg; + struct rte_mbuf *buf = + (void *)((uintptr_t)start + data->mp->header_size); + + (void)index; + /* Check whether mbuf structure fits element size and whether mempool + * pointer is valid. */ + if (((uintptr_t)end >= (uintptr_t)(buf + 1)) && + (buf->pool == data->mp)) + data->ret = 0; + else + data->ret = -1; +} + +/** + * Iterator function for rte_mempool_walk() to register existing mempools and + * fill the MP to MR cache of a TX queue. + * + * @param[in] mp + * Memory Pool to register. + * @param *arg + * Pointer to TX queue structure. + */ +static void +txq_mp2mr_iter(const struct rte_mempool *mp, void *arg) +{ + struct txq *txq = arg; + struct txq_mp2mr_mbuf_check_data data = { + .mp = mp, + .ret = -1, + }; + + /* Discard empty mempools. */ + if (mp->size == 0) + return; + /* Register mempool only if the first element looks like a mbuf. */ + rte_mempool_obj_iter((void *)mp->elt_va_start, + 1, + mp->header_size + mp->elt_size + mp->trailer_size, + 1, + mp->elt_pa, + mp->pg_num, + mp->pg_shift, + txq_mp2mr_mbuf_check, + &data); + if (data.ret) + return; + txq_mp2mr(txq, mp); +} + +#if MLX4_PMD_SGE_WR_N > 1 + /** * Copy scattered mbuf contents to a single linear buffer. * @@ -1057,6 +1371,122 @@ linearize_mbuf(linear_t *linear, struct rte_mbuf *buf) return size; } +/** + * Handle scattered buffers for mlx4_tx_burst(). + * + * @param txq + * TX queue structure. + * @param segs + * Number of segments in buf. + * @param elt + * TX queue element to fill. + * @param[in] buf + * Buffer to process. + * @param elts_head + * Index of the linear buffer to use if necessary (normally txq->elts_head). + * @param[out] sges + * Array filled with SGEs on success. + * + * @return + * A structure containing the processed packet size in bytes and the + * number of SGEs. Both fields are set to (unsigned int)-1 in case of + * failure. + */ +static struct tx_burst_sg_ret { + unsigned int length; + unsigned int num; +} +tx_burst_sg(struct txq *txq, unsigned int segs, struct txq_elt *elt, + struct rte_mbuf *buf, unsigned int elts_head, + struct ibv_sge (*sges)[MLX4_PMD_SGE_WR_N]) +{ + unsigned int sent_size = 0; + unsigned int j; + int linearize = 0; + + /* When there are too many segments, extra segments are + * linearized in the last SGE. */ + if (unlikely(segs > elemof(*sges))) { + segs = (elemof(*sges) - 1); + linearize = 1; + } + /* Update element. */ + elt->buf = buf; + /* Register segments as SGEs. */ + for (j = 0; (j != segs); ++j) { + struct ibv_sge *sge = &(*sges)[j]; + uint32_t lkey; + + /* Retrieve Memory Region key for this memory pool. */ + lkey = txq_mp2mr(txq, txq_mb2mp(buf)); + if (unlikely(lkey == (uint32_t)-1)) { + /* MR does not exist. */ + DEBUG("%p: unable to get MP <-> MR association", + (void *)txq); + /* Clean up TX element. */ + elt->buf = NULL; + goto stop; + } + /* Update SGE. */ + sge->addr = rte_pktmbuf_mtod(buf, uintptr_t); + if (txq->priv->vf) + rte_prefetch0((volatile void *) + (uintptr_t)sge->addr); + sge->length = DATA_LEN(buf); + sge->lkey = lkey; + sent_size += sge->length; + buf = NEXT(buf); + } + /* If buf is not NULL here and is not going to be linearized, + * nb_segs is not valid. */ + assert(j == segs); + assert((buf == NULL) || (linearize)); + /* Linearize extra segments. */ + if (linearize) { + struct ibv_sge *sge = &(*sges)[segs]; + linear_t *linear = &(*txq->elts_linear)[elts_head]; + unsigned int size = linearize_mbuf(linear, buf); + + assert(segs == (elemof(*sges) - 1)); + if (size == 0) { + /* Invalid packet. */ + DEBUG("%p: packet too large to be linearized.", + (void *)txq); + /* Clean up TX element. */ + elt->buf = NULL; + goto stop; + } + /* If MLX4_PMD_SGE_WR_N is 1, free mbuf immediately. */ + if (elemof(*sges) == 1) { + do { + struct rte_mbuf *next = NEXT(buf); + + rte_pktmbuf_free_seg(buf); + buf = next; + } while (buf != NULL); + elt->buf = NULL; + } + /* Update SGE. */ + sge->addr = (uintptr_t)&(*linear)[0]; + sge->length = size; + sge->lkey = txq->mr_linear->lkey; + sent_size += size; + /* Include last segment. */ + segs++; + } + return (struct tx_burst_sg_ret){ + .length = sent_size, + .num = segs, + }; +stop: + return (struct tx_burst_sg_ret){ + .length = -1, + .num = -1, + }; +} + +#endif /* MLX4_PMD_SGE_WR_N > 1 */ + /** * DPDK callback for TX. * @@ -1074,10 +1504,7 @@ static uint16_t mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct txq *txq = (struct txq *)dpdk_txq; - struct ibv_send_wr head; - struct ibv_send_wr **wr_next = &head.next; unsigned int elts_head = txq->elts_head; - const unsigned int elts_tail = txq->elts_tail; const unsigned int elts_n = txq->elts_n; unsigned int elts_comp_cd = txq->elts_comp_cd; unsigned int elts_comp = 0; @@ -1087,7 +1514,7 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) assert(elts_comp_cd != 0); txq_complete(txq); - max = (elts_n - (elts_head - elts_tail)); + max = (elts_n - (elts_head - txq->elts_tail)); if (max > elts_n) max -= elts_n; assert(max >= 1); @@ -1100,21 +1527,19 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) max = pkts_n; for (i = 0; (i != max); ++i) { struct rte_mbuf *buf = pkts[i]; + unsigned int elts_head_next = + (((elts_head + 1) == elts_n) ? 0 : elts_head + 1); + struct txq_elt *elt_next = &(*txq->elts)[elts_head_next]; struct txq_elt *elt = &(*txq->elts)[elts_head]; - struct ibv_send_wr *wr = &elt->wr; unsigned int segs = NB_SEGS(buf); #ifdef MLX4_PMD_SOFT_COUNTERS unsigned int sent_size = 0; #endif - unsigned int j; - int linearize = 0; uint32_t send_flags = 0; /* Clean up old buffer. */ - if (likely(WR_ID(wr->wr_id).offset != 0)) { - struct rte_mbuf *tmp = (void *) - ((uintptr_t)elt->sges[0].addr - - WR_ID(wr->wr_id).offset); + if (likely(elt->buf != NULL)) { + struct rte_mbuf *tmp = elt->buf; /* Faster than rte_pktmbuf_free(). */ do { @@ -1124,171 +1549,100 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) tmp = next; } while (tmp != NULL); } -#ifndef NDEBUG - /* For assert(). */ - WR_ID(wr->wr_id).offset = 0; - for (j = 0; ((int)j < wr->num_sge); ++j) { - elt->sges[j].addr = 0; - elt->sges[j].length = 0; - elt->sges[j].lkey = 0; + /* Request TX completion. */ + if (unlikely(--elts_comp_cd == 0)) { + elts_comp_cd = txq->elts_comp_cd_init; + ++elts_comp; + send_flags |= IBV_EXP_QP_BURST_SIGNALED; } - wr->next = NULL; - wr->num_sge = 0; -#endif - /* Sanity checks, most of which are only relevant with - * debugging enabled. */ - assert(WR_ID(wr->wr_id).id == elts_head); - assert(WR_ID(wr->wr_id).offset == 0); - assert(wr->next == NULL); - assert(wr->sg_list == &elt->sges[0]); - assert(wr->num_sge == 0); - assert(wr->opcode == IBV_WR_SEND); - /* When there are too many segments, extra segments are - * linearized in the last SGE. */ - if (unlikely(segs > elemof(elt->sges))) { - segs = (elemof(elt->sges) - 1); - linearize = 1; + /* Should we enable HW CKSUM offload */ + if (buf->ol_flags & + (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { + send_flags |= IBV_EXP_QP_BURST_IP_CSUM; + /* HW does not support checksum offloads at arbitrary + * offsets but automatically recognizes the packet + * type. For inner L3/L4 checksums, only VXLAN (UDP) + * tunnels are currently supported. */ + if (RTE_ETH_IS_TUNNEL_PKT(buf->packet_type)) + send_flags |= IBV_EXP_QP_BURST_TUNNEL; } - /* Set WR fields. */ - assert((rte_pktmbuf_mtod(buf, uintptr_t) - - (uintptr_t)buf) <= 0xffff); - WR_ID(wr->wr_id).offset = - (rte_pktmbuf_mtod(buf, uintptr_t) - - (uintptr_t)buf); - wr->num_sge = segs; - /* Register segments as SGEs. */ - for (j = 0; (j != segs); ++j) { - struct ibv_sge *sge = &elt->sges[j]; + if (likely(segs == 1)) { + uintptr_t addr; + uint32_t length; uint32_t lkey; + /* Retrieve buffer information. */ + addr = rte_pktmbuf_mtod(buf, uintptr_t); + length = DATA_LEN(buf); /* Retrieve Memory Region key for this memory pool. */ - lkey = txq_mp2mr(txq, buf->pool); + lkey = txq_mp2mr(txq, txq_mb2mp(buf)); if (unlikely(lkey == (uint32_t)-1)) { /* MR does not exist. */ DEBUG("%p: unable to get MP <-> MR" " association", (void *)txq); /* Clean up TX element. */ - WR_ID(elt->wr.wr_id).offset = 0; -#ifndef NDEBUG - /* For assert(). */ - while (j) { - --j; - --sge; - sge->addr = 0; - sge->length = 0; - sge->lkey = 0; - } - wr->num_sge = 0; -#endif + elt->buf = NULL; goto stop; } - /* Sanity checks, only relevant with debugging - * enabled. */ - assert(sge->addr == 0); - assert(sge->length == 0); - assert(sge->lkey == 0); - /* Update SGE. */ - sge->addr = rte_pktmbuf_mtod(buf, uintptr_t); + /* Update element. */ + elt->buf = buf; if (txq->priv->vf) rte_prefetch0((volatile void *) - (uintptr_t)sge->addr); - sge->length = DATA_LEN(buf); - sge->lkey = lkey; -#ifdef MLX4_PMD_SOFT_COUNTERS - sent_size += sge->length; -#endif - buf = NEXT(buf); - } - /* If buf is not NULL here and is not going to be linearized, - * nb_segs is not valid. */ - assert(j == segs); - assert((buf == NULL) || (linearize)); - /* Linearize extra segments. */ - if (linearize) { - struct ibv_sge *sge = &elt->sges[segs]; - linear_t *linear = &(*txq->elts_linear)[elts_head]; - unsigned int size = linearize_mbuf(linear, buf); - - assert(segs == (elemof(elt->sges) - 1)); - if (size == 0) { - /* Invalid packet. */ - DEBUG("%p: packet too large to be linearized.", - (void *)txq); - /* Clean up TX element. */ - WR_ID(elt->wr.wr_id).offset = 0; -#ifndef NDEBUG - /* For assert(). */ - while (j) { - --j; - --sge; - sge->addr = 0; - sge->length = 0; - sge->lkey = 0; - } - wr->num_sge = 0; + (uintptr_t)addr); + RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf); + /* Put packet into send queue. */ +#if MLX4_PMD_MAX_INLINE > 0 + if (length <= txq->max_inline) + err = txq->if_qp->send_pending_inline + (txq->qp, + (void *)addr, + length, + send_flags); + else #endif + err = txq->if_qp->send_pending + (txq->qp, + addr, + length, + lkey, + send_flags); + if (unlikely(err)) goto stop; - } - /* If MLX4_PMD_SGE_WR_N is 1, free mbuf immediately - * and clear offset from WR ID. */ - if (elemof(elt->sges) == 1) { - do { - struct rte_mbuf *next = NEXT(buf); +#ifdef MLX4_PMD_SOFT_COUNTERS + sent_size += length; +#endif + } else { +#if MLX4_PMD_SGE_WR_N > 1 + struct ibv_sge sges[MLX4_PMD_SGE_WR_N]; + struct tx_burst_sg_ret ret; - rte_pktmbuf_free_seg(buf); - buf = next; - } while (buf != NULL); - WR_ID(wr->wr_id).offset = 0; - } - /* Set WR fields and fill SGE with linear buffer. */ - ++wr->num_sge; - /* Sanity checks, only relevant with debugging - * enabled. */ - assert(sge->addr == 0); - assert(sge->length == 0); - assert(sge->lkey == 0); - /* Update SGE. */ - sge->addr = (uintptr_t)&(*linear)[0]; - sge->length = size; - sge->lkey = txq->mr_linear->lkey; + ret = tx_burst_sg(txq, segs, elt, buf, elts_head, + &sges); + if (ret.length == (unsigned int)-1) + goto stop; + RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf); + /* Put SG list into send queue. */ + err = txq->if_qp->send_pending_sg_list + (txq->qp, + sges, + ret.num, + send_flags); + if (unlikely(err)) + goto stop; #ifdef MLX4_PMD_SOFT_COUNTERS - sent_size += size; + sent_size += ret.length; #endif +#else /* MLX4_PMD_SGE_WR_N > 1 */ + DEBUG("%p: TX scattered buffers support not" + " compiled in", (void *)txq); + goto stop; +#endif /* MLX4_PMD_SGE_WR_N > 1 */ } - /* Link WRs together for ibv_post_send(). */ - *wr_next = wr; - wr_next = &wr->next; - assert(wr->send_flags == 0); - /* Request TX completion. */ - if (unlikely(--elts_comp_cd == 0)) { - elts_comp_cd = txq->elts_comp_cd_init; - ++elts_comp; - send_flags |= IBV_EXP_QP_BURST_SIGNALED; - } - if (++elts_head >= elts_n) - elts_head = 0; + elts_head = elts_head_next; #ifdef MLX4_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += sent_size; #endif - /* Put SG list into send queue and ask for completion event. */ -#if MLX4_PMD_MAX_INLINE > 0 - if ((segs == 1) && - (elt->sges[0].length <= txq->max_inline)) - err = txq->if_qp->send_pending_inline - (txq->qp, - (void *)(uintptr_t)elt->sges[0].addr, - elt->sges[0].length, - send_flags); - else -#endif - err = txq->if_qp->send_pending_sg_list - (txq->qp, - elt->sges, - segs, - send_flags); - if (unlikely(err)) - goto stop; } stop: /* Take a shortcut if nothing must be sent. */ @@ -1298,7 +1652,6 @@ stop: /* Increment sent packets counter. */ txq->stats.opackets += i; #endif - *wr_next = NULL; /* Ring QP doorbell. */ err = txq->if_qp->send_flush(txq->qp); if (unlikely(err)) { @@ -1313,6 +1666,46 @@ stop: return i; } +/** + * DPDK callback for TX in secondary processes. + * + * This function configures all queues from primary process information + * if necessary before reverting to the normal TX burst callback. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static uint16_t +mlx4_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + struct txq *txq = dpdk_txq; + struct priv *priv = mlx4_secondary_data_setup(txq->priv); + struct priv *primary_priv; + unsigned int index; + + if (priv == NULL) + return 0; + primary_priv = + mlx4_secondary_data[priv->dev->data->port_id].primary_priv; + /* Look for queue index in both private structures. */ + for (index = 0; index != priv->txqs_n; ++index) + if (((*primary_priv->txqs)[index] == txq) || + ((*priv->txqs)[index] == txq)) + break; + if (index == priv->txqs_n) + return 0; + txq = (*priv->txqs)[index]; + return priv->dev->tx_pkt_burst(txq, pkts, pkts_n); +} + /** * Configure a TX queue. * @@ -1334,20 +1727,24 @@ static int txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, unsigned int socket, const struct rte_eth_txconf *conf) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct txq tmpl = { .priv = priv, .socket = socket }; union { struct ibv_exp_query_intf_params params; - struct ibv_qp_init_attr init; + struct ibv_exp_qp_init_attr init; + struct ibv_exp_res_domain_init_attr rd; + struct ibv_exp_cq_init_attr cq; struct ibv_exp_qp_attr mod; } attr; enum ibv_exp_query_intf_status status; int ret = 0; (void)conf; /* Thresholds configuration (ignored). */ + if (priv == NULL) + return EINVAL; if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) { ERROR("%p: invalid number of TX descriptors (must be a" " multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N); @@ -1355,7 +1752,24 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, } desc /= MLX4_PMD_SGE_WR_N; /* MRs will be registered in mp2mr[] later. */ - tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0); + attr.rd = (struct ibv_exp_res_domain_init_attr){ + .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | + IBV_EXP_RES_DOMAIN_MSG_MODEL), + .thread_model = IBV_EXP_THREAD_SINGLE, + .msg_model = IBV_EXP_MSG_HIGH_BW, + }; + tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); + if (tmpl.rd == NULL) { + ret = ENOMEM; + ERROR("%p: RD creation failure: %s", + (void *)dev, strerror(ret)); + goto error; + } + attr.cq = (struct ibv_exp_cq_init_attr){ + .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, + .res_domain = tmpl.rd, + }; + tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -1366,7 +1780,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, priv->device_attr.max_qp_wr); DEBUG("priv->device_attr.max_sge is %d", priv->device_attr.max_sge); - attr.init = (struct ibv_qp_init_attr){ + attr.init = (struct ibv_exp_qp_init_attr){ /* CQ to be associated with the send queue. */ .send_cq = tmpl.cq, /* CQ to be associated with the receive queue. */ @@ -1388,9 +1802,13 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, .qp_type = IBV_QPT_RAW_PACKET, /* Do *NOT* enable this, completions events are managed per * TX burst. */ - .sq_sig_all = 0 + .sq_sig_all = 0, + .pd = priv->pd, + .res_domain = tmpl.rd, + .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | + IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), }; - tmpl.qp = ibv_create_qp(priv->pd, &attr.init); + tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); if (tmpl.qp == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: QP creation failure: %s", @@ -1451,6 +1869,13 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, .intf_scope = IBV_EXP_INTF_GLOBAL, .intf = IBV_EXP_INTF_QP_BURST, .obj = tmpl.qp, +#ifdef HAVE_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK + /* MC loopback must be disabled when not using a VF. */ + .family_flags = + (!priv->vf ? + IBV_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK : + 0), +#endif }; tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); if (tmpl.if_qp == NULL) { @@ -1463,6 +1888,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, txq_cleanup(txq); *txq = tmpl; DEBUG("%p: txq updated with %p", (void *)txq, (void *)&tmpl); + /* Pre-register known mempools. */ + rte_mempool_walk(txq_mp2mr_iter, txq); assert(ret == 0); return 0; error: @@ -1496,6 +1923,8 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct txq *txq = (*priv->txqs)[idx]; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); @@ -1551,6 +1980,8 @@ mlx4_tx_queue_release(void *dpdk_txq) struct priv *priv; unsigned int i; + if (mlx4_is_secondary()) + return; if (txq == NULL) return; priv = txq->priv; @@ -2379,11 +2810,95 @@ rxq_cleanup(struct rxq *rxq) } if (rxq->cq != NULL) claim_zero(ibv_destroy_cq(rxq->cq)); + if (rxq->rd != NULL) { + struct ibv_exp_destroy_res_domain_attr attr = { + .comp_mask = 0, + }; + + assert(rxq->priv != NULL); + assert(rxq->priv->ctx != NULL); + claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx, + rxq->rd, + &attr)); + } if (rxq->mr != NULL) claim_zero(ibv_dereg_mr(rxq->mr)); memset(rxq, 0, sizeof(*rxq)); } +/** + * Translate RX completion flags to packet type. + * + * @param flags + * RX completion flags returned by poll_length_flags(). + * + * @return + * Packet type for struct rte_mbuf. + */ +static inline uint32_t +rxq_cq_to_pkt_type(uint32_t flags) +{ + uint32_t pkt_type; + + if (flags & IBV_EXP_CQ_RX_TUNNEL_PACKET) + pkt_type = + TRANSPOSE(flags, + IBV_EXP_CQ_RX_OUTER_IPV4_PACKET, RTE_PTYPE_L3_IPV4) | + TRANSPOSE(flags, + IBV_EXP_CQ_RX_OUTER_IPV6_PACKET, RTE_PTYPE_L3_IPV6) | + TRANSPOSE(flags, + IBV_EXP_CQ_RX_IPV4_PACKET, RTE_PTYPE_INNER_L3_IPV4) | + TRANSPOSE(flags, + IBV_EXP_CQ_RX_IPV6_PACKET, RTE_PTYPE_INNER_L3_IPV6); + else + pkt_type = + TRANSPOSE(flags, + IBV_EXP_CQ_RX_IPV4_PACKET, RTE_PTYPE_L3_IPV4) | + TRANSPOSE(flags, + IBV_EXP_CQ_RX_IPV6_PACKET, RTE_PTYPE_L3_IPV6); + return pkt_type; +} + +/** + * Translate RX completion flags to offload flags. + * + * @param[in] rxq + * Pointer to RX queue structure. + * @param flags + * RX completion flags returned by poll_length_flags(). + * + * @return + * Offload flags (ol_flags) for struct rte_mbuf. + */ +static inline uint32_t +rxq_cq_to_ol_flags(const struct rxq *rxq, uint32_t flags) +{ + uint32_t ol_flags = 0; + + if (rxq->csum) + ol_flags |= + TRANSPOSE(~flags, + IBV_EXP_CQ_RX_IP_CSUM_OK, + PKT_RX_IP_CKSUM_BAD) | + TRANSPOSE(~flags, + IBV_EXP_CQ_RX_TCP_UDP_CSUM_OK, + PKT_RX_L4_CKSUM_BAD); + /* + * PKT_RX_IP_CKSUM_BAD and PKT_RX_L4_CKSUM_BAD are used in place + * of PKT_RX_EIP_CKSUM_BAD because the latter is not functional + * (its value is 0). + */ + if ((flags & IBV_EXP_CQ_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) + ol_flags |= + TRANSPOSE(~flags, + IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK, + PKT_RX_IP_CKSUM_BAD) | + TRANSPOSE(~flags, + IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK, + PKT_RX_L4_CKSUM_BAD); + return ol_flags; +} + static uint16_t mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); @@ -2428,6 +2943,7 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) struct rte_mbuf **pkt_buf_next = &pkt_buf; unsigned int seg_headroom = RTE_PKTMBUF_HEADROOM; unsigned int j = 0; + uint32_t flags; /* Sanity checks. */ #ifdef NDEBUG @@ -2438,7 +2954,8 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) assert(wr->num_sge == elemof(elt->sges)); assert(elts_head < rxq->elts_n); assert(rxq->elts_head < rxq->elts_n); - ret = rxq->if_cq->poll_length(rxq->cq, NULL, NULL); + ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL, + &flags); if (unlikely(ret < 0)) { struct ibv_wc wc; int wcs_n; @@ -2564,7 +3081,8 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) NB_SEGS(pkt_buf) = j; PORT(pkt_buf) = rxq->port_id; PKT_LEN(pkt_buf) = pkt_buf_len; - pkt_buf->ol_flags = 0; + pkt_buf->packet_type = rxq_cq_to_pkt_type(flags); + pkt_buf->ol_flags = rxq_cq_to_ol_flags(rxq, flags); /* Return packet. */ *(pkts++) = pkt_buf; @@ -2641,6 +3159,7 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) struct rte_mbuf *seg = (void *)((uintptr_t)elt->sge.addr - WR_ID(wr_id).offset); struct rte_mbuf *rep; + uint32_t flags; /* Sanity checks. */ assert(WR_ID(wr_id).id < rxq->elts_n); @@ -2648,7 +3167,14 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) assert(wr->num_sge == 1); assert(elts_head < rxq->elts_n); assert(rxq->elts_head < rxq->elts_n); - ret = rxq->if_cq->poll_length(rxq->cq, NULL, NULL); + /* + * Fetch initial bytes of packet descriptor into a + * cacheline while allocating rep. + */ + rte_prefetch0(seg); + rte_prefetch0(&seg->cacheline1); + ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL, + &flags); if (unlikely(ret < 0)) { struct ibv_wc wc; int wcs_n; @@ -2684,11 +3210,6 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) if (ret == 0) break; len = ret; - /* - * Fetch initial bytes of packet descriptor into a - * cacheline while allocating rep. - */ - rte_prefetch0(seg); rep = __rte_mbuf_raw_alloc(rxq->mp); if (unlikely(rep == NULL)) { /* @@ -2722,7 +3243,8 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) NEXT(seg) = NULL; PKT_LEN(seg) = len; DATA_LEN(seg) = len; - seg->ol_flags = 0; + seg->packet_type = rxq_cq_to_pkt_type(flags); + seg->ol_flags = rxq_cq_to_ol_flags(rxq, flags); /* Return packet. */ *(pkts++) = seg; @@ -2758,6 +3280,46 @@ repost: return pkts_ret; } +/** + * DPDK callback for RX in secondary processes. + * + * This function configures all queues from primary process information + * if necessary before reverting to the normal RX burst callback. + * + * @param dpdk_rxq + * Generic pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +static uint16_t +mlx4_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + struct rxq *rxq = dpdk_rxq; + struct priv *priv = mlx4_secondary_data_setup(rxq->priv); + struct priv *primary_priv; + unsigned int index; + + if (priv == NULL) + return 0; + primary_priv = + mlx4_secondary_data[priv->dev->data->port_id].primary_priv; + /* Look for queue index in both private structures. */ + for (index = 0; index != priv->rxqs_n; ++index) + if (((*primary_priv->rxqs)[index] == rxq) || + ((*priv->rxqs)[index] == rxq)) + break; + if (index == priv->rxqs_n) + return 0; + rxq = (*priv->rxqs)[index]; + return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n); +} + /** * Allocate a Queue Pair. * Optionally setup inline receive if supported. @@ -2773,7 +3335,8 @@ repost: * QP pointer or NULL in case of error. */ static struct ibv_qp * -rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc) +rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc, + struct ibv_exp_res_domain *rd) { struct ibv_exp_qp_init_attr attr = { /* CQ to be associated with the send queue. */ @@ -2792,8 +3355,10 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc) MLX4_PMD_SGE_WR_N), }, .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = IBV_EXP_QP_INIT_ATTR_PD, + .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | + IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), .pd = priv->pd, + .res_domain = rd, }; #ifdef INLINE_RECV @@ -2823,7 +3388,7 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc) */ static struct ibv_qp * rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, - int parent) + int parent, struct ibv_exp_res_domain *rd) { struct ibv_exp_qp_init_attr attr = { /* CQ to be associated with the send queue. */ @@ -2843,8 +3408,10 @@ rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, }, .qp_type = IBV_QPT_RAW_PACKET, .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | + IBV_EXP_QP_INIT_ATTR_RES_DOMAIN | IBV_EXP_QP_INIT_ATTR_QPG), - .pd = priv->pd + .pd = priv->pd, + .res_domain = rd, }; #ifdef INLINE_RECV @@ -2905,6 +3472,15 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) /* Number of descriptors and mbufs currently allocated. */ desc_n = (tmpl.elts_n * (tmpl.sp ? MLX4_PMD_SGE_WR_N : 1)); mbuf_n = desc_n; + /* Toggle RX checksum offload if hardware supports it. */ + if (priv->hw_csum) { + tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; + rxq->csum = tmpl.csum; + } + if (priv->hw_csum_l2tun) { + tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum; + rxq->csum_l2tun = tmpl.csum_l2tun; + } /* Enable scattered packets support for this queue if necessary. */ if ((dev->data->dev_conf.rxmode.jumbo_frame) && (dev->data->dev_conf.rxmode.max_rx_pkt_len > @@ -3091,6 +3667,8 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, struct ibv_exp_qp_attr mod; union { struct ibv_exp_query_intf_params params; + struct ibv_exp_cq_init_attr cq; + struct ibv_exp_res_domain_init_attr rd; } attr; enum ibv_exp_query_intf_status status; struct ibv_recv_wr *bad_wr; @@ -3126,6 +3704,11 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, rte_pktmbuf_tailroom(buf)) == tmpl.mb_len); assert(rte_pktmbuf_headroom(buf) == RTE_PKTMBUF_HEADROOM); rte_pktmbuf_free(buf); + /* Toggle RX checksum offload if hardware supports it. */ + if (priv->hw_csum) + tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; + if (priv->hw_csum_l2tun) + tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum; /* Enable scattered packets support for this queue if necessary. */ if ((dev->data->dev_conf.rxmode.jumbo_frame) && (dev->data->dev_conf.rxmode.max_rx_pkt_len > @@ -3148,7 +3731,24 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, goto error; } skip_mr: - tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0); + attr.rd = (struct ibv_exp_res_domain_init_attr){ + .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | + IBV_EXP_RES_DOMAIN_MSG_MODEL), + .thread_model = IBV_EXP_THREAD_SINGLE, + .msg_model = IBV_EXP_MSG_HIGH_BW, + }; + tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); + if (tmpl.rd == NULL) { + ret = ENOMEM; + ERROR("%p: RD creation failure: %s", + (void *)dev, strerror(ret)); + goto error; + } + attr.cq = (struct ibv_exp_cq_init_attr){ + .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, + .res_domain = tmpl.rd, + }; + tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -3161,10 +3761,11 @@ skip_mr: priv->device_attr.max_sge); #ifdef RSS_SUPPORT if (priv->rss) - tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent); + tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent, + tmpl.rd); else #endif /* RSS_SUPPORT */ - tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc); + tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd); if (tmpl.qp == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: QP creation failure: %s", @@ -3297,6 +3898,8 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct rxq *rxq = (*priv->rxqs)[idx]; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); @@ -3355,6 +3958,8 @@ mlx4_rx_queue_release(void *dpdk_rxq) struct priv *priv; unsigned int i; + if (mlx4_is_secondary()) + return; if (rxq == NULL) return; priv = rxq->priv; @@ -3372,6 +3977,9 @@ mlx4_rx_queue_release(void *dpdk_rxq) priv_unlock(priv); } +static void +priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *); + /** * DPDK callback to start the device. * @@ -3391,6 +3999,8 @@ mlx4_dev_start(struct rte_eth_dev *dev) unsigned int r; struct rxq *rxq; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); if (priv->started) { priv_unlock(priv); @@ -3431,8 +4041,10 @@ mlx4_dev_start(struct rte_eth_dev *dev) } } priv->started = 0; + priv_unlock(priv); return -ret; } while ((--r) && ((rxq = (*priv->rxqs)[++i]), i)); + priv_dev_interrupt_handler_install(priv, dev); priv_unlock(priv); return 0; } @@ -3453,6 +4065,8 @@ mlx4_dev_stop(struct rte_eth_dev *dev) unsigned int r; struct rxq *rxq; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->started) { priv_unlock(priv); @@ -3529,6 +4143,9 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) return 0; } +static void +priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); + /** * DPDK callback to close the device. * @@ -3540,10 +4157,12 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) static void mlx4_dev_close(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); void *tmp; unsigned int i; + if (priv == NULL) + return; priv_lock(priv); DEBUG("%p: closing device \"%s\"", (void *)dev, @@ -3589,6 +4208,7 @@ mlx4_dev_close(struct rte_eth_dev *dev) claim_zero(ibv_close_device(priv->ctx)); } else assert(priv->ctx == NULL); + priv_dev_interrupt_handler_uninstall(priv, dev); priv_unlock(priv); memset(priv, 0, sizeof(*priv)); } @@ -3604,9 +4224,12 @@ mlx4_dev_close(struct rte_eth_dev *dev) static void mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); unsigned int max; + char ifname[IF_NAMESIZE]; + if (priv == NULL) + return; priv_lock(priv); /* FIXME: we should ask the device for these values. */ info->min_rx_bufsize = 32; @@ -3622,7 +4245,22 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) max = 65535; info->max_rx_queues = max; info->max_tx_queues = max; - info->max_mac_addrs = elemof(priv->mac); + /* Last array entry is reserved for broadcast. */ + info->max_mac_addrs = (elemof(priv->mac) - 1); + info->rx_offload_capa = + (priv->hw_csum ? + (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM) : + 0); + info->tx_offload_capa = + (priv->hw_csum ? + (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM) : + 0); + if (priv_get_ifname(priv, &ifname) == 0) + info->if_index = if_nametoindex(ifname); priv_unlock(priv); } @@ -3637,11 +4275,13 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) static void mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct rte_eth_stats tmp = {0}; unsigned int i; unsigned int idx; + if (priv == NULL) + return; priv_lock(priv); /* Add software counters. */ for (i = 0; (i != priv->rxqs_n); ++i) { @@ -3700,10 +4340,12 @@ mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) static void mlx4_stats_reset(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); unsigned int i; unsigned int idx; + if (priv == NULL) + return; priv_lock(priv); for (i = 0; (i != priv->rxqs_n); ++i) { if ((*priv->rxqs)[i] == NULL) @@ -3715,7 +4357,7 @@ mlx4_stats_reset(struct rte_eth_dev *dev) for (i = 0; (i != priv->txqs_n); ++i) { if ((*priv->txqs)[i] == NULL) continue; - idx = (*priv->rxqs)[i]->stats.idx; + idx = (*priv->txqs)[i]->stats.idx; (*priv->txqs)[i]->stats = (struct mlx4_txq_stats){ .idx = idx }; } @@ -3738,14 +4380,13 @@ mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) { struct priv *priv = dev->data->dev_private; + if (mlx4_is_secondary()) + return; priv_lock(priv); DEBUG("%p: removing MAC address from index %" PRIu32, (void *)dev, index); - if (index >= MLX4_MAX_MAC_ADDRESSES) - goto end; - /* Refuse to remove the broadcast address, this one is special. */ - if (!memcmp(priv->mac[index].addr_bytes, "\xff\xff\xff\xff\xff\xff", - ETHER_ADDR_LEN)) + /* Last array entry is reserved for broadcast. */ + if (index >= (elemof(priv->mac) - 1)) goto end; priv_mac_addr_del(priv, index); end: @@ -3770,15 +4411,14 @@ mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, { struct priv *priv = dev->data->dev_private; + if (mlx4_is_secondary()) + return; (void)vmdq; priv_lock(priv); DEBUG("%p: adding MAC address at index %" PRIu32, (void *)dev, index); - if (index >= MLX4_MAX_MAC_ADDRESSES) - goto end; - /* Refuse to add the broadcast address, this one is special. */ - if (!memcmp(mac_addr->addr_bytes, "\xff\xff\xff\xff\xff\xff", - ETHER_ADDR_LEN)) + /* Last array entry is reserved for broadcast. */ + if (index >= (elemof(priv->mac) - 1)) goto end; priv_mac_addr_add(priv, index, (const uint8_t (*)[ETHER_ADDR_LEN]) @@ -3787,6 +4427,22 @@ end: priv_unlock(priv); } +/** + * DPDK callback to set the primary MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + */ +static void +mlx4_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + DEBUG("%p: setting primary MAC address", (void *)dev); + mlx4_mac_addr_remove(dev, 0); + mlx4_mac_addr_add(dev, mac_addr, 0, 0); +} + /** * DPDK callback to enable promiscuous mode. * @@ -3800,6 +4456,8 @@ mlx4_promiscuous_enable(struct rte_eth_dev *dev) unsigned int i; int ret; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (priv->promisc) { priv_unlock(priv); @@ -3846,6 +4504,8 @@ mlx4_promiscuous_disable(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; unsigned int i; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->promisc) { priv_unlock(priv); @@ -3876,6 +4536,8 @@ mlx4_allmulticast_enable(struct rte_eth_dev *dev) unsigned int i; int ret; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (priv->allmulti) { priv_unlock(priv); @@ -3922,6 +4584,8 @@ mlx4_allmulticast_disable(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; unsigned int i; + if (mlx4_is_secondary()) + return; priv_lock(priv); if (!priv->allmulti) { priv_unlock(priv); @@ -3950,7 +4614,7 @@ end: static int mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); struct ethtool_cmd edata = { .cmd = ETHTOOL_GSET }; @@ -3958,6 +4622,8 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) struct rte_eth_link dev_link; int link_speed = 0; + if (priv == NULL) + return -EINVAL; (void)wait_to_complete; if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); @@ -3999,9 +4665,11 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) static int mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = dev->data->dev_private; + struct priv *priv = mlx4_get_priv(dev); int ret; + if (priv == NULL) + return -EINVAL; priv_lock(priv); ret = mlx4_link_update_unlocked(dev, wait_to_complete); priv_unlock(priv); @@ -4034,6 +4702,8 @@ mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) = mlx4_rx_burst; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); /* Set kernel interface MTU first. */ if (priv_set_mtu(priv, mtu)) { @@ -4117,6 +4787,8 @@ mlx4_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; ifr.ifr_data = ðpause; priv_lock(priv); if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { @@ -4165,6 +4837,8 @@ mlx4_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; ifr.ifr_data = ðpause; ethpause.autoneg = fc_conf->autoneg; if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || @@ -4304,6 +4978,8 @@ mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) struct priv *priv = dev->data->dev_private; int ret; + if (mlx4_is_secondary()) + return -E_RTE_SECONDARY; priv_lock(priv); ret = vlan_filter_set(dev, vlan_id, on); priv_unlock(priv); @@ -4340,14 +5016,8 @@ static const struct eth_dev_ops mlx4_dev_ops = { .priority_flow_ctrl_set = NULL, .mac_addr_remove = mlx4_mac_addr_remove, .mac_addr_add = mlx4_mac_addr_add, + .mac_addr_set = mlx4_mac_addr_set, .mtu_set = mlx4_dev_set_mtu, - .fdir_add_signature_filter = NULL, - .fdir_update_signature_filter = NULL, - .fdir_remove_signature_filter = NULL, - .fdir_add_perfect_filter = NULL, - .fdir_update_perfect_filter = NULL, - .fdir_remove_perfect_filter = NULL, - .fdir_set_masks = NULL }; /** @@ -4475,6 +5145,158 @@ mlx4_getenv_int(const char *name) return atoi(val); } +static void +mlx4_dev_link_status_handler(void *); +static void +mlx4_dev_interrupt_handler(struct rte_intr_handle *, void *); + +/** + * Link status handler. + * + * @param priv + * Pointer to private structure. + * @param dev + * Pointer to the rte_eth_dev structure. + * + * @return + * Nonzero if the callback process can be called immediately. + */ +static int +priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) +{ + struct ibv_async_event event; + int port_change = 0; + int ret = 0; + + /* Read all message and acknowledge them. */ + for (;;) { + if (ibv_get_async_event(priv->ctx, &event)) + break; + + if (event.event_type == IBV_EVENT_PORT_ACTIVE || + event.event_type == IBV_EVENT_PORT_ERR) + port_change = 1; + else + DEBUG("event type %d on port %d not handled", + event.event_type, event.element.port_num); + ibv_ack_async_event(&event); + } + + if (port_change ^ priv->pending_alarm) { + struct rte_eth_link *link = &dev->data->dev_link; + + priv->pending_alarm = 0; + mlx4_link_update_unlocked(dev, 0); + if (((link->link_speed == 0) && link->link_status) || + ((link->link_speed != 0) && !link->link_status)) { + /* Inconsistent status, check again later. */ + priv->pending_alarm = 1; + rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US, + mlx4_dev_link_status_handler, + dev); + } else + ret = 1; + } + return ret; +} + +/** + * Handle delayed link status event. + * + * @param arg + * Registered argument. + */ +static void +mlx4_dev_link_status_handler(void *arg) +{ + struct rte_eth_dev *dev = arg; + struct priv *priv = dev->data->dev_private; + int ret; + + priv_lock(priv); + assert(priv->pending_alarm == 1); + ret = priv_dev_link_status_handler(priv, dev); + priv_unlock(priv); + if (ret) + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); +} + +/** + * Handle interrupts from the NIC. + * + * @param[in] intr_handle + * Interrupt handler. + * @param cb_arg + * Callback argument. + */ +static void +mlx4_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg) +{ + struct rte_eth_dev *dev = cb_arg; + struct priv *priv = dev->data->dev_private; + int ret; + + (void)intr_handle; + priv_lock(priv); + ret = priv_dev_link_status_handler(priv, dev); + priv_unlock(priv); + if (ret) + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); +} + +/** + * Uninstall interrupt handler. + * + * @param priv + * Pointer to private structure. + * @param dev + * Pointer to the rte_eth_dev structure. + */ +static void +priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) +{ + if (!dev->data->dev_conf.intr_conf.lsc) + return; + rte_intr_callback_unregister(&priv->intr_handle, + mlx4_dev_interrupt_handler, + dev); + if (priv->pending_alarm) + rte_eal_alarm_cancel(mlx4_dev_link_status_handler, dev); + priv->pending_alarm = 0; + priv->intr_handle.fd = 0; + priv->intr_handle.type = 0; +} + +/** + * Install interrupt handler. + * + * @param priv + * Pointer to private structure. + * @param dev + * Pointer to the rte_eth_dev structure. + */ +static void +priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) +{ + int rc, flags; + + if (!dev->data->dev_conf.intr_conf.lsc) + return; + assert(priv->ctx->async_fd > 0); + flags = fcntl(priv->ctx->async_fd, F_GETFL); + rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); + if (rc < 0) { + INFO("failed to change file descriptor async event queue"); + dev->data->dev_conf.intr_conf.lsc = 0; + } else { + priv->intr_handle.fd = priv->ctx->async_fd; + priv->intr_handle.type = RTE_INTR_HANDLE_EXT; + rte_intr_callback_register(&priv->intr_handle, + mlx4_dev_interrupt_handler, + dev); + } +} + static struct eth_driver mlx4_driver; /** @@ -4576,7 +5398,7 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct ibv_port_attr port_attr; struct ibv_pd *pd = NULL; struct priv *priv = NULL; - struct rte_eth_dev *eth_dev; + struct rte_eth_dev *eth_dev = NULL; #ifdef HAVE_EXP_QUERY_DEVICE struct ibv_exp_device_attr exp_device_attr; #endif /* HAVE_EXP_QUERY_DEVICE */ @@ -4602,9 +5424,9 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) goto port_error; } if (port_attr.state != IBV_PORT_ACTIVE) - WARN("bad state for port %d: \"%s\" (%d)", - port, ibv_port_state_str(port_attr.state), - port_attr.state); + DEBUG("port %d is not active: \"%s\" (%d)", + port, ibv_port_state_str(port_attr.state), + port_attr.state); /* Allocate protection domain. */ pd = ibv_alloc_pd(ctx); @@ -4663,6 +5485,19 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) exp_device_attr.max_rss_tbl_sz); #endif /* RSS_SUPPORT */ + priv->hw_csum = + ((exp_device_attr.exp_device_cap_flags & + IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) && + (exp_device_attr.exp_device_cap_flags & + IBV_EXP_DEVICE_RX_CSUM_IP_PKT)); + DEBUG("checksum offloading is %ssupported", + (priv->hw_csum ? "" : "not ")); + + priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags & + IBV_EXP_DEVICE_VXLAN_SUPPORT); + DEBUG("L2 tunnel checksum offloads are %ssupported", + (priv->hw_csum_l2tun ? "" : "not ")); + #ifdef INLINE_RECV priv->inl_recv_size = mlx4_getenv_int("MLX4_INLINE_RECV_SIZE"); @@ -4707,7 +5542,7 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) claim_zero(priv_mac_addr_add(priv, 0, (const uint8_t (*)[ETHER_ADDR_LEN]) mac.addr_bytes)); - claim_zero(priv_mac_addr_add(priv, 1, + claim_zero(priv_mac_addr_add(priv, (elemof(priv->mac) - 1), &(const uint8_t [ETHER_ADDR_LEN]) { "\xff\xff\xff\xff\xff\xff" })); #ifndef NDEBUG @@ -4739,15 +5574,47 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) goto port_error; } - eth_dev->data->dev_private = priv; + /* Secondary processes have to use local storage for their + * private data as well as a copy of eth_dev->data, but this + * pointer must not be modified before burst functions are + * actually called. */ + if (mlx4_is_secondary()) { + struct mlx4_secondary_data *sd = + &mlx4_secondary_data[eth_dev->data->port_id]; + + sd->primary_priv = eth_dev->data->dev_private; + if (sd->primary_priv == NULL) { + ERROR("no private data for port %u", + eth_dev->data->port_id); + err = EINVAL; + goto port_error; + } + sd->shared_dev_data = eth_dev->data; + rte_spinlock_init(&sd->lock); + memcpy(sd->data.name, sd->shared_dev_data->name, + sizeof(sd->data.name)); + sd->data.dev_private = priv; + sd->data.rx_mbuf_alloc_failed = 0; + sd->data.mtu = ETHER_MTU; + sd->data.port_id = sd->shared_dev_data->port_id; + sd->data.mac_addrs = priv->mac; + eth_dev->tx_pkt_burst = mlx4_tx_burst_secondary_setup; + eth_dev->rx_pkt_burst = mlx4_rx_burst_secondary_setup; + } else { + eth_dev->data->dev_private = priv; + eth_dev->data->rx_mbuf_alloc_failed = 0; + eth_dev->data->mtu = ETHER_MTU; + eth_dev->data->mac_addrs = priv->mac; + } eth_dev->pci_dev = pci_dev; + + rte_eth_copy_pci_info(eth_dev, pci_dev); + eth_dev->driver = &mlx4_driver; - eth_dev->data->rx_mbuf_alloc_failed = 0; - eth_dev->data->mtu = ETHER_MTU; priv->dev = eth_dev; eth_dev->dev_ops = &mlx4_dev_ops; - eth_dev->data->mac_addrs = priv->mac; + TAILQ_INIT(ð_dev->link_intr_cbs); /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); @@ -4760,6 +5627,8 @@ port_error: claim_zero(ibv_dealloc_pd(pd)); if (ctx) claim_zero(ibv_close_device(ctx)); + if (eth_dev) + rte_eth_dev_release_port(eth_dev); break; } @@ -4814,6 +5683,7 @@ static struct eth_driver mlx4_driver = { .name = MLX4_DRIVER_NAME, .id_table = mlx4_pci_id_map, .devinit = mlx4_pci_devinit, + .drv_flags = RTE_PCI_DRV_INTR_LSC, }, .dev_private_size = sizeof(struct priv) }; @@ -4826,6 +5696,8 @@ rte_mlx4_pmd_init(const char *name, const char *args) { (void)name; (void)args; + + RTE_BUILD_BUG_ON(sizeof(wr_id_t) != sizeof(uint64_t)); /* * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use * huge pages. Calling ibv_fork_init() during init allows