X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_mr.c;h=fdbe7986fdddd5c1075362f2c210c362a5103590;hb=c2450e933f01d4d31448240f7304730292db7ee8;hp=89c43fc9e91b497553c895980d20a196cd209b12;hpb=1c196da274b82e6f1642bde7b1c407a2066caee1;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index 89c43fc9e9..fdbe7986fd 100644 --- a/drivers/net/mlx5/mlx5_mr.c +++ b/drivers/net/mlx5/mlx5_mr.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -66,30 +65,6 @@ mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr, } } -/** - * Bottom-half of LKey search on Rx. - * - * @param rxq - * Pointer to Rx queue structure. - * @param addr - * Search key. - * - * @return - * Searched LKey on success, UINT32_MAX on no match. - */ -uint32_t -mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr) -{ - struct mlx5_rxq_ctrl *rxq_ctrl = - container_of(rxq, struct mlx5_rxq_ctrl, rxq); - struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl; - struct mlx5_priv *priv = rxq_ctrl->priv; - - return mlx5_mr_addr2mr_bh(priv->sh->pd, &priv->mp_id, - &priv->sh->share_cache, mr_ctrl, addr, - priv->config.mr_ext_memseg_en); -} - /** * Bottom-half of LKey search on Tx. * @@ -129,9 +104,36 @@ mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr) uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb) { + struct mlx5_txq_ctrl *txq_ctrl = + container_of(txq, struct mlx5_txq_ctrl, txq); + struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl; + struct mlx5_priv *priv = txq_ctrl->priv; uintptr_t addr = (uintptr_t)mb->buf_addr; uint32_t lkey; + if (priv->config.mr_mempool_reg_en) { + struct rte_mempool *mp = NULL; + struct mlx5_mprq_buf *buf; + + if (!RTE_MBUF_HAS_EXTBUF(mb)) { + mp = mlx5_mb2mp(mb); + } else if (mb->shinfo->free_cb == mlx5_mprq_buf_free_cb) { + /* Recover MPRQ mempool. */ + buf = mb->shinfo->fcb_opaque; + mp = buf->mp; + } + if (mp != NULL) { + lkey = mlx5_mr_mempool2mr_bh(&priv->sh->share_cache, + mr_ctrl, mp, addr); + /* + * Lookup can only fail on invalid input, e.g. "addr" + * is not from "mp" or "mp" has RTE_MEMPOOL_F_NON_IO set. + */ + if (lkey != UINT32_MAX) + return lkey; + } + /* Fallback for generic mechanism in corner cases. */ + } lkey = mlx5_tx_addr2mr_bh(txq, addr); if (lkey == UINT32_MAX && rte_errno == ENXIO) { /* Mempool may have externally allocated memory. */ @@ -199,33 +201,33 @@ mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque, } /** - * Finds the first ethdev that match the pci device. + * Finds the first ethdev that match the device. * The existence of multiple ethdev per pci device is only with representors. * On such case, it is enough to get only one of the ports as they all share * the same ibv context. * - * @param pdev - * Pointer to the PCI device. + * @param dev + * Pointer to the device. * * @return * Pointer to the ethdev if found, NULL otherwise. */ static struct rte_eth_dev * -pci_dev_to_eth_dev(struct rte_pci_device *pdev) +dev_to_eth_dev(struct rte_device *dev) { uint16_t port_id; - port_id = rte_eth_find_next_of(0, &pdev->device); + port_id = rte_eth_find_next_of(0, dev); if (port_id == RTE_MAX_ETHPORTS) return NULL; return &rte_eth_devices[port_id]; } /** - * DPDK callback to DMA map external memory to a PCI device. + * Callback to DMA map external memory to a device. * - * @param pdev - * Pointer to the PCI device. + * @param rte_dev + * Pointer to the generic device. * @param addr * Starting virtual address of memory to be mapped. * @param iova @@ -237,18 +239,18 @@ pci_dev_to_eth_dev(struct rte_pci_device *pdev) * 0 on success, negative value on error. */ int -mlx5_dma_map(struct rte_pci_device *pdev, void *addr, - uint64_t iova __rte_unused, size_t len) +mlx5_net_dma_map(struct rte_device *rte_dev, void *addr, + uint64_t iova __rte_unused, size_t len) { struct rte_eth_dev *dev; struct mlx5_mr *mr; struct mlx5_priv *priv; struct mlx5_dev_ctx_shared *sh; - dev = pci_dev_to_eth_dev(pdev); + dev = dev_to_eth_dev(rte_dev); if (!dev) { DRV_LOG(WARNING, "unable to find matching ethdev " - "to PCI device %p", (void *)pdev); + "to device %s", rte_dev->name); rte_errno = ENODEV; return -1; } @@ -271,10 +273,10 @@ mlx5_dma_map(struct rte_pci_device *pdev, void *addr, } /** - * DPDK callback to DMA unmap external memory to a PCI device. + * Callback to DMA unmap external memory to a device. * - * @param pdev - * Pointer to the PCI device. + * @param rte_dev + * Pointer to the generic device. * @param addr * Starting virtual address of memory to be unmapped. * @param iova @@ -286,8 +288,8 @@ mlx5_dma_map(struct rte_pci_device *pdev, void *addr, * 0 on success, negative value on error. */ int -mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, - uint64_t iova __rte_unused, size_t len __rte_unused) +mlx5_net_dma_unmap(struct rte_device *rte_dev, void *addr, + uint64_t iova __rte_unused, size_t len __rte_unused) { struct rte_eth_dev *dev; struct mlx5_priv *priv; @@ -295,10 +297,10 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, struct mlx5_mr *mr; struct mr_cache_entry entry; - dev = pci_dev_to_eth_dev(pdev); + dev = dev_to_eth_dev(rte_dev); if (!dev) { - DRV_LOG(WARNING, "unable to find matching ethdev " - "to PCI device %p", (void *)pdev); + DRV_LOG(WARNING, "unable to find matching ethdev to device %s", + rte_dev->name); rte_errno = ENODEV; return -1; } @@ -308,16 +310,15 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, mr = mlx5_mr_lookup_list(&sh->share_cache, &entry, (uintptr_t)addr); if (!mr) { rte_rwlock_write_unlock(&sh->share_cache.rwlock); - DRV_LOG(WARNING, "address 0x%" PRIxPTR " wasn't registered " - "to PCI device %p", (uintptr_t)addr, - (void *)pdev); + DRV_LOG(WARNING, "address 0x%" PRIxPTR " wasn't registered to device %s", + (uintptr_t)addr, rte_dev->name); rte_errno = EINVAL; return -1; } LIST_REMOVE(mr, mr); - mlx5_mr_free(mr, sh->share_cache.dereg_mr_cb); DRV_LOG(DEBUG, "port %u remove MR(%p) from list", dev->data->port_id, (void *)mr); + mlx5_mr_free(mr, sh->share_cache.dereg_mr_cb); mlx5_mr_rebuild_cache(&sh->share_cache); /* * No explicit wmb is needed after updating dev_gen due to @@ -394,72 +395,3 @@ mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr, mlx5_mr_update_ext_mp(ETH_DEV(priv), mr_ctrl, mp); return mlx5_tx_addr2mr_bh(txq, addr); } - -/* Called during rte_mempool_mem_iter() by mlx5_mr_update_mp(). */ -static void -mlx5_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque, - struct rte_mempool_memhdr *memhdr, - unsigned mem_idx __rte_unused) -{ - struct mr_update_mp_data *data = opaque; - struct rte_eth_dev *dev = data->dev; - struct mlx5_priv *priv = dev->data->dev_private; - - uint32_t lkey; - - /* Stop iteration if failed in the previous walk. */ - if (data->ret < 0) - return; - /* Register address of the chunk and update local caches. */ - lkey = mlx5_mr_addr2mr_bh(priv->sh->pd, &priv->mp_id, - &priv->sh->share_cache, data->mr_ctrl, - (uintptr_t)memhdr->addr, - priv->config.mr_ext_memseg_en); - if (lkey == UINT32_MAX) - data->ret = -1; -} - -/** - * Register entire memory chunks in a Mempool. - * - * @param dev - * Pointer to Ethernet device. - * @param mr_ctrl - * Pointer to per-queue MR control structure. - * @param mp - * Pointer to registering Mempool. - * - * @return - * 0 on success, -1 on failure. - */ -int -mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl, - struct rte_mempool *mp) -{ - struct mr_update_mp_data data = { - .dev = dev, - .mr_ctrl = mr_ctrl, - .ret = 0, - }; - uint32_t flags = rte_pktmbuf_priv_flags(mp); - - if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) { - /* - * The pinned external buffer should be registered for DMA - * operations by application. The mem_list of the pool contains - * the list of chunks with mbuf structures w/o built-in data - * buffers and DMA actually does not happen there, no need - * to create MR for these chunks. - */ - return 0; - } - DRV_LOG(DEBUG, "Port %u Rx queue registering mp %s " - "having %u chunks.", dev->data->port_id, - mp->name, mp->nb_mem_chunks); - rte_mempool_mem_iter(mp, mlx5_mr_update_mp_cb, &data); - if (data.ret < 0 && rte_errno == ENXIO) { - /* Mempool may have externally allocated memory. */ - return mlx5_mr_update_ext_mp(dev, mr_ctrl, mp); - } - return data.ret; -}