X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4_txq.c;h=0db2e55befd348ff66713a463577bffcd592b93e;hb=f96789aeaf1851071f5b2ab532d7687c66e1ee41;hp=352700820dd679ab834c30c31a5c5a0fe977864a;hpb=099c2c5376131b3d352d70f8904e586c0e84651e;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c index 352700820d..0db2e55bef 100644 --- a/drivers/net/mlx4/mlx4_txq.c +++ b/drivers/net/mlx4/mlx4_txq.c @@ -8,12 +8,13 @@ * Tx queues configuration for mlx4 driver. */ -#include #include #include #include #include +#include #include +#include /* Verbs headers do not support -pedantic. */ #ifdef PEDANTIC @@ -26,7 +27,7 @@ #include #include -#include +#include #include #include #include @@ -37,6 +38,165 @@ #include "mlx4_rxtx.h" #include "mlx4_utils.h" +/** + * Initialize Tx UAR registers for primary process. + * + * @param txq + * Pointer to Tx queue structure. + */ +static void +txq_uar_init(struct txq *txq) +{ + struct mlx4_priv *priv = txq->priv; + struct mlx4_proc_priv *ppriv = MLX4_PROC_PRIV(PORT_ID(priv)); + + MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); + MLX4_ASSERT(ppriv); + ppriv->uar_table[txq->stats.idx] = txq->msq.db; +} + +#ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET +/** + * Remap UAR register of a Tx queue for secondary process. + * + * Remapped address is stored at the table in the process private structure of + * the device, indexed by queue index. + * + * @param txq + * Pointer to Tx queue structure. + * @param fd + * Verbs file descriptor to map UAR pages. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +txq_uar_init_secondary(struct txq *txq, int fd) +{ + struct mlx4_priv *priv = txq->priv; + struct mlx4_proc_priv *ppriv = MLX4_PROC_PRIV(PORT_ID(priv)); + void *addr; + uintptr_t uar_va; + uintptr_t offset; + const size_t page_size = sysconf(_SC_PAGESIZE); + + MLX4_ASSERT(ppriv); + /* + * As rdma-core, UARs are mapped in size of OS page + * size. Ref to libmlx4 function: mlx4_init_context() + */ + uar_va = (uintptr_t)txq->msq.db; + offset = uar_va & (page_size - 1); /* Offset in page. */ + addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, + txq->msq.uar_mmap_offset); + if (addr == MAP_FAILED) { + ERROR("port %u mmap failed for BF reg of txq %u", + txq->port_id, txq->stats.idx); + rte_errno = ENXIO; + return -rte_errno; + } + addr = RTE_PTR_ADD(addr, offset); + ppriv->uar_table[txq->stats.idx] = addr; + return 0; +} + +/** + * Unmap UAR register of a Tx queue for secondary process. + * + * @param txq + * Pointer to Tx queue structure. + */ +static void +txq_uar_uninit_secondary(struct txq *txq) +{ + struct mlx4_proc_priv *ppriv = MLX4_PROC_PRIV(PORT_ID(txq->priv)); + const size_t page_size = sysconf(_SC_PAGESIZE); + void *addr; + + addr = ppriv->uar_table[txq->stats.idx]; + munmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size); +} + +/** + * Initialize Tx UAR registers for secondary process. + * + * @param dev + * Pointer to Ethernet device. + * @param fd + * Verbs file descriptor to map UAR pages. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx4_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd) +{ + const unsigned int txqs_n = dev->data->nb_tx_queues; + struct txq *txq; + unsigned int i; + int ret; + + MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY); + for (i = 0; i != txqs_n; ++i) { + txq = dev->data->tx_queues[i]; + if (!txq) + continue; + MLX4_ASSERT(txq->stats.idx == (uint16_t)i); + ret = txq_uar_init_secondary(txq, fd); + if (ret) + goto error; + } + return 0; +error: + /* Rollback. */ + do { + txq = dev->data->tx_queues[i]; + if (!txq) + continue; + txq_uar_uninit_secondary(txq); + } while (i--); + return -rte_errno; +} + +void +mlx4_tx_uar_uninit_secondary(struct rte_eth_dev *dev) +{ + struct mlx4_proc_priv *ppriv = + (struct mlx4_proc_priv *)dev->process_private; + const size_t page_size = sysconf(_SC_PAGESIZE); + void *addr; + size_t i; + + if (page_size == (size_t)-1) { + ERROR("Failed to get mem page size"); + return; + } + for (i = 0; i < ppriv->uar_table_sz; i++) { + addr = ppriv->uar_table[i]; + if (addr) + munmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size); + } +} + +#else +int +mlx4_tx_uar_init_secondary(struct rte_eth_dev *dev __rte_unused, + int fd __rte_unused) +{ + MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY); + ERROR("UAR remap is not supported"); + rte_errno = ENOTSUP; + return -rte_errno; +} + +void +mlx4_tx_uar_uninit_secondary(struct rte_eth_dev *dev __rte_unused) +{ + assert(rte_eal_process_type() == RTE_PROC_SECONDARY); + ERROR("UAR remap is not supported"); +} +#endif + /** * Free Tx queue elements. * @@ -46,19 +206,18 @@ static void mlx4_txq_free_elts(struct txq *txq) { - unsigned int elts_head = txq->elts_head; - unsigned int elts_tail = txq->elts_tail; struct txq_elt (*elts)[txq->elts_n] = txq->elts; - unsigned int elts_m = txq->elts_n - 1; + unsigned int n = txq->elts_n; - DEBUG("%p: freeing WRs", (void *)txq); - while (elts_tail != elts_head) { - struct txq_elt *elt = &(*elts)[elts_tail++ & elts_m]; + DEBUG("%p: freeing WRs, %u", (void *)txq, n); + while (n--) { + struct txq_elt *elt = &(*elts)[n]; - assert(elt->buf != NULL); - rte_pktmbuf_free(elt->buf); - elt->buf = NULL; - elt->wqe = NULL; + if (elt->buf) { + rte_pktmbuf_free(elt->buf); + elt->buf = NULL; + elt->wqe = NULL; + } } txq->elts_tail = txq->elts_head; } @@ -89,6 +248,11 @@ mlx4_txq_fill_dv_obj_info(struct txq *txq, struct mlx4dv_obj *mlxdv) sq->owner_opcode = MLX4_OPCODE_SEND | (0u << MLX4_SQ_OWNER_BIT); sq->stamp = rte_cpu_to_be_32(MLX4_SQ_STAMP_VAL | (0u << MLX4_SQ_OWNER_BIT)); +#ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET + sq->uar_mmap_offset = dqp->uar_mmap_offset; +#else + sq->uar_mmap_offset = -1; /* Make mmap() fail. */ +#endif sq->db = dqp->sdb; sq->doorbell_qpn = dqp->doorbell_qpn; cq->buf = dcq->buf.buf; @@ -109,20 +273,20 @@ mlx4_txq_fill_dv_obj_info(struct txq *txq, struct mlx4dv_obj *mlxdv) uint64_t mlx4_get_tx_port_offloads(struct mlx4_priv *priv) { - uint64_t offloads = DEV_TX_OFFLOAD_MULTI_SEGS; + uint64_t offloads = RTE_ETH_TX_OFFLOAD_MULTI_SEGS; if (priv->hw_csum) { - offloads |= (DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM); + offloads |= (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | + RTE_ETH_TX_OFFLOAD_UDP_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_CKSUM); } if (priv->tso) - offloads |= DEV_TX_OFFLOAD_TCP_TSO; + offloads |= RTE_ETH_TX_OFFLOAD_TCP_TSO; if (priv->hw_csum_l2tun) { - offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + offloads |= RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM; if (priv->tso) - offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | - DEV_TX_OFFLOAD_GRE_TNL_TSO); + offloads |= (RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | + RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO); } return offloads; } @@ -177,10 +341,8 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, uint64_t offloads; offloads = conf->offloads | dev->data->dev_conf.txmode.offloads; - DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); - if (idx >= dev->data->nb_tx_queues) { rte_errno = EOVERFLOW; ERROR("%p: queue index out of range (%u >= %u)", @@ -214,6 +376,7 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, } *txq = (struct txq){ .priv = priv, + .port_id = dev->data->port_id, .stats = { .idx = idx, }, @@ -231,16 +394,19 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, .elts_comp_cd_init = RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4), .csum = priv->hw_csum && - (offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM)), + (offloads & (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | + RTE_ETH_TX_OFFLOAD_UDP_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_CKSUM)), .csum_l2tun = priv->hw_csum_l2tun && (offloads & - DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM), + RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM), /* Enable Tx loopback for VF devices. */ .lb = !!priv->vf, .bounce_buf = bounce_buf, }; + dev->data->tx_queues[idx] = txq; + priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_TX_QUEUE; + priv->verbs_alloc_ctx.obj = txq; txq->cq = mlx4_glue->create_cq(priv->ctx, desc, NULL, NULL, 0); if (!txq->cq) { rte_errno = ENOMEM; @@ -307,6 +473,11 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, goto error; } /* Retrieve device queue information. */ +#ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET + dv_qp = (struct mlx4dv_qp){ + .comp_mask = MLX4DV_QP_MASK_UAR_MMAP_OFFSET, + }; +#endif mlxdv.cq.in = txq->cq; mlxdv.cq.out = &dv_cq; mlxdv.qp.in = txq->qp; @@ -318,7 +489,14 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, " accessing the device queues", (void *)dev); goto error; } +#ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET + if (!(dv_qp.comp_mask & MLX4DV_QP_MASK_UAR_MMAP_OFFSET)) { + WARN("%p: failed to obtain UAR mmap offset", (void *)dev); + dv_qp.uar_mmap_offset = -1; /* Make mmap() fail. */ + } +#endif mlx4_txq_fill_dv_obj_info(txq, &mlxdv); + txq_uar_init(txq); /* Save first wqe pointer in the first element. */ (&(*txq->elts)[0])->wqe = (volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf; @@ -330,40 +508,34 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, /* Save pointer of global generation number to check memory event. */ txq->mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen; DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq); - dev->data->tx_queues[idx] = txq; + priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_NONE; return 0; error: - dev->data->tx_queues[idx] = NULL; ret = rte_errno; - mlx4_tx_queue_release(txq); + mlx4_tx_queue_release(dev, idx); rte_errno = ret; - assert(rte_errno > 0); + MLX4_ASSERT(rte_errno > 0); + priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_NONE; return -rte_errno; } /** * DPDK callback to release a Tx queue. * - * @param dpdk_txq - * Generic Tx queue pointer. + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * Transmit queue index. */ void -mlx4_tx_queue_release(void *dpdk_txq) +mlx4_tx_queue_release(struct rte_eth_dev *dev, uint16_t idx) { - struct txq *txq = (struct txq *)dpdk_txq; - struct mlx4_priv *priv; - unsigned int i; + struct txq *txq = dev->data->tx_queues[idx]; if (txq == NULL) return; - priv = txq->priv; - for (i = 0; i != ETH_DEV(priv)->data->nb_tx_queues; ++i) - if (ETH_DEV(priv)->data->tx_queues[i] == txq) { - DEBUG("%p: removing Tx queue %p from list", - (void *)ETH_DEV(priv), (void *)txq); - ETH_DEV(priv)->data->tx_queues[i] = NULL; - break; - } + DEBUG("%p: removing Tx queue %hu from list", (void *)dev, idx); + dev->data->tx_queues[idx] = NULL; mlx4_txq_free_elts(txq); if (txq->qp) claim_zero(mlx4_glue->destroy_qp(txq->qp));