X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rxq.c;h=35c5cb42e951411bfbea1c9e3dadbe082b906c5f;hb=9f9bebae55309c1230d2b9a81ece2f94f1eeda69;hp=a82c415cd0b7aa5138b32ea5f85d24e92dc526d8;hpb=a0edafe4099b1ef139242abb7baa2c2a48b83fd2;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index a82c415cd0..35c5cb42e9 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -50,19 +50,12 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include #include #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5.h" #include "mlx5_rxtx.h" @@ -633,41 +626,6 @@ priv_rehash_flows(struct priv *priv) return 0; } -/** - * Unlike regular Rx function, vPMD Rx doesn't replace mbufs immediately when - * receiving packets. Instead it replaces later in bulk. In rxq->elts[], entries - * from rq_pi to rq_ci are owned by device but the rest is already delivered to - * application. In order not to reuse those mbufs by rxq_alloc_elts(), this - * function must be called to replace used mbufs. - * - * @param rxq - * Pointer to RX queue structure. - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_trim_elts(struct rxq *rxq) -{ - const uint16_t q_n = (1 << rxq->elts_n); - const uint16_t q_mask = q_n - 1; - uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi); - uint16_t i; - - if (!rxq->trim_elts) - return 0; - for (i = 0; i < used; ++i) { - struct rte_mbuf *buf; - buf = rte_pktmbuf_alloc(rxq->mp); - if (!buf) - return ENOMEM; - (*rxq->elts)[(rxq->rq_ci + i) & q_mask] = buf; - } - rxq->rq_pi = rxq->rq_ci; - rxq->trim_elts = 0; - return 0; -} - /** * Allocate RX queue elements. * @@ -675,16 +633,12 @@ rxq_trim_elts(struct rxq *rxq) * Pointer to RX queue structure. * @param elts_n * Number of elements to allocate. - * @param[in] pool - * If not NULL, fetch buffers from this array instead of allocating them - * with rte_pktmbuf_alloc(). * * @return * 0 on success, errno value on failure. */ static int -rxq_alloc_elts(struct rxq_ctrl *rxq_ctrl, unsigned int elts_n, - struct rte_mbuf *(*pool)[]) +rxq_alloc_elts(struct rxq_ctrl *rxq_ctrl, unsigned int elts_n) { const unsigned int sges_n = 1 << rxq_ctrl->rxq.sges_n; unsigned int i; @@ -696,15 +650,8 @@ rxq_alloc_elts(struct rxq_ctrl *rxq_ctrl, unsigned int elts_n, volatile struct mlx5_wqe_data_seg *scat = &(*rxq_ctrl->rxq.wqes)[i]; - if (pool != NULL) { - buf = (*pool)[i]; - assert(buf != NULL); - rte_pktmbuf_reset(buf); - rte_pktmbuf_refcnt_update(buf, 1); - } else - buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp); + buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp); if (buf == NULL) { - assert(pool == NULL); ERROR("%p: empty mbuf pool", (void *)rxq_ctrl); ret = ENOMEM; goto error; @@ -731,12 +678,31 @@ rxq_alloc_elts(struct rxq_ctrl *rxq_ctrl, unsigned int elts_n, }; (*rxq_ctrl->rxq.elts)[i] = buf; } + if (rxq_check_vec_support(&rxq_ctrl->rxq) > 0) { + struct rxq *rxq = &rxq_ctrl->rxq; + struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; + + assert(rxq->elts_n == rxq->cqe_n); + /* Initialize default rearm_data for vPMD. */ + mbuf_init->data_off = RTE_PKTMBUF_HEADROOM; + rte_mbuf_refcnt_set(mbuf_init, 1); + mbuf_init->nb_segs = 1; + mbuf_init->port = rxq->port_id; + /* + * prevent compiler reordering: + * rearm_data covers previous fields. + */ + rte_compiler_barrier(); + rxq->mbuf_initializer = *(uint64_t *)&mbuf_init->rearm_data; + /* Padding with a fake mbuf for vectorized Rx. */ + for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) + (*rxq->elts)[elts_n + i] = &rxq->fake_mbuf; + } DEBUG("%p: allocated and configured %u segments (max %u packets)", (void *)rxq_ctrl, elts_n, elts_n / (1 << rxq_ctrl->rxq.sges_n)); assert(ret == 0); return 0; error: - assert(pool == NULL); elts_n = i; for (i = 0; (i != elts_n); ++i) { if ((*rxq_ctrl->rxq.elts)[i] != NULL) @@ -757,16 +723,28 @@ error: static void rxq_free_elts(struct rxq_ctrl *rxq_ctrl) { - unsigned int i; + struct rxq *rxq = &rxq_ctrl->rxq; + const uint16_t q_n = (1 << rxq->elts_n); + const uint16_t q_mask = q_n - 1; + uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi); + uint16_t i; DEBUG("%p: freeing WRs", (void *)rxq_ctrl); - if (rxq_ctrl->rxq.elts == NULL) + if (rxq->elts == NULL) return; - - for (i = 0; (i != (1u << rxq_ctrl->rxq.elts_n)); ++i) { - if ((*rxq_ctrl->rxq.elts)[i] != NULL) - rte_pktmbuf_free_seg((*rxq_ctrl->rxq.elts)[i]); - (*rxq_ctrl->rxq.elts)[i] = NULL; + /** + * Some mbuf in the Ring belongs to the application. They cannot be + * freed. + */ + if (rxq_check_vec_support(rxq) > 0) { + for (i = 0; i < used; ++i) + (*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL; + rxq->rq_pi = rxq->rq_ci; + } + for (i = 0; (i != (1u << rxq->elts_n)); ++i) { + if ((*rxq->elts)[i] != NULL) + rte_pktmbuf_free_seg((*rxq->elts)[i]); + (*rxq->elts)[i] = NULL; } } @@ -811,9 +789,11 @@ rxq_setup(struct rxq_ctrl *tmpl) struct ibv_cq *ibcq = tmpl->cq; struct ibv_mlx5_cq_info cq_info; struct mlx5_rwq *rwq = container_of(tmpl->wq, struct mlx5_rwq, wq); - struct rte_mbuf *(*elts)[1 << tmpl->rxq.elts_n] = + const uint16_t desc_n = + (1 << tmpl->rxq.elts_n) + tmpl->priv->rx_vec_en * + MLX5_VPMD_DESCS_PER_LOOP; + struct rte_mbuf *(*elts)[desc_n] = rte_calloc_socket("RXQ", 1, sizeof(*elts), 0, tmpl->socket); - if (ibv_mlx5_exp_get_cq_info(ibcq, &cq_info)) { ERROR("Unable to query CQ info. check your OFED."); return ENOTSUP; @@ -883,7 +863,9 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, } attr; unsigned int mb_len = rte_pktmbuf_data_room_size(mp); unsigned int cqe_n = desc - 1; - struct rte_mbuf *(*elts)[desc] = NULL; + const uint16_t desc_n = + desc + priv->rx_vec_en * MLX5_VPMD_DESCS_PER_LOOP; + struct rte_mbuf *(*elts)[desc_n] = NULL; int ret = 0; (void)conf; /* Thresholds configuration (ignored). */ @@ -1074,15 +1056,7 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, (void *)dev, strerror(ret)); goto error; } - /* Reuse buffers from original queue if possible. */ - if (rxq_ctrl->rxq.elts_n) { - assert(1 << rxq_ctrl->rxq.elts_n == desc); - assert(rxq_ctrl->rxq.elts != tmpl.rxq.elts); - ret = rxq_trim_elts(&rxq_ctrl->rxq); - if (!ret) - ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts); - } else - ret = rxq_alloc_elts(&tmpl, desc, NULL); + ret = rxq_alloc_elts(&tmpl, desc); if (ret) { ERROR("%p: RXQ allocation failed: %s", (void *)dev, strerror(ret)); @@ -1142,7 +1116,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct priv *priv = dev->data->dev_private; struct rxq *rxq = (*priv->rxqs)[idx]; struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - const uint16_t desc_pad = MLX5_VPMD_DESCS_PER_LOOP; /* For vPMD. */ + const uint16_t desc_n = + desc + priv->rx_vec_en * MLX5_VPMD_DESCS_PER_LOOP; int ret; if (mlx5_is_secondary()) @@ -1175,9 +1150,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, /* Resize if rxq size is changed. */ if (rxq_ctrl->rxq.elts_n != log2above(desc)) { rxq_ctrl = rte_realloc(rxq_ctrl, - sizeof(*rxq_ctrl) + - (desc + desc_pad) * - sizeof(struct rte_mbuf *), + sizeof(*rxq_ctrl) + desc_n * + sizeof(struct rte_mbuf *), RTE_CACHE_LINE_SIZE); if (!rxq_ctrl) { ERROR("%p: unable to reallocate queue index %u", @@ -1188,8 +1162,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, } } else { rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl) + - (desc + desc_pad) * - sizeof(struct rte_mbuf *), + desc_n * + sizeof(struct rte_mbuf *), 0, socket); if (rxq_ctrl == NULL) { ERROR("%p: unable to allocate queue index %u", @@ -1248,47 +1222,6 @@ mlx5_rx_queue_release(void *dpdk_rxq) priv_unlock(priv); } -/** - * DPDK callback for RX in secondary processes. - * - * This function configures all queues from primary process information - * if necessary before reverting to the normal RX burst callback. - * - * @param dpdk_rxq - * Generic pointer to RX queue structure. - * @param[out] pkts - * Array to store received packets. - * @param pkts_n - * Maximum number of packets in array. - * - * @return - * Number of packets successfully received (<= pkts_n). - */ -uint16_t -mlx5_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts, - uint16_t pkts_n) -{ - struct rxq *rxq = dpdk_rxq; - struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - struct priv *priv = mlx5_secondary_data_setup(rxq_ctrl->priv); - struct priv *primary_priv; - unsigned int index; - - if (priv == NULL) - return 0; - primary_priv = - mlx5_secondary_data[priv->dev->data->port_id].primary_priv; - /* Look for queue index in both private structures. */ - for (index = 0; index != priv->rxqs_n; ++index) - if (((*primary_priv->rxqs)[index] == rxq) || - ((*priv->rxqs)[index] == rxq)) - break; - if (index == priv->rxqs_n) - return 0; - rxq = (*priv->rxqs)[index]; - return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n); -} - /** * Allocate queue vector and fill epoll fd list for Rx interrupts. *