From 7e95c11df4f18eb1155ce5d5fa4bfb6a1316d2a0 Mon Sep 17 00:00:00 2001 From: Nithin Dabilpuram Date: Wed, 23 Jun 2021 10:16:23 +0530 Subject: [PATCH] net/cnxk: add multi-segment Tx for CN9K Add Tx burst multi-segment version for CN9K. Signed-off-by: Nithin Dabilpuram Signed-off-by: Pavan Nikhilesh --- drivers/net/cnxk/cn9k_tx.c | 14 +++ drivers/net/cnxk/cn9k_tx.h | 150 ++++++++++++++++++++++++++++++++ drivers/net/cnxk/cn9k_tx_mseg.c | 25 ++++++ drivers/net/cnxk/cnxk_ethdev.h | 4 + drivers/net/cnxk/meson.build | 1 + 5 files changed, 194 insertions(+) create mode 100644 drivers/net/cnxk/cn9k_tx_mseg.c diff --git a/drivers/net/cnxk/cn9k_tx.c b/drivers/net/cnxk/cn9k_tx.c index a0b022a117..8f1d5f59b0 100644 --- a/drivers/net/cnxk/cn9k_tx.c +++ b/drivers/net/cnxk/cn9k_tx.c @@ -21,6 +21,7 @@ NIX_TX_FASTPATH_MODES #undef T + static inline void pick_tx_func(struct rte_eth_dev *eth_dev, const eth_tx_burst_t tx_burst[2][2][2][2][2]) @@ -39,6 +40,8 @@ pick_tx_func(struct rte_eth_dev *eth_dev, void cn9k_eth_set_tx_function(struct rte_eth_dev *eth_dev) { + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + const eth_tx_burst_t nix_eth_tx_burst[2][2][2][2][2] = { #define T(name, f4, f3, f2, f1, f0, sz, flags) \ [f4][f3][f2][f1][f0] = cn9k_nix_xmit_pkts_##name, @@ -47,7 +50,18 @@ cn9k_eth_set_tx_function(struct rte_eth_dev *eth_dev) #undef T }; + const eth_tx_burst_t nix_eth_tx_burst_mseg[2][2][2][2][2] = { +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + [f4][f3][f2][f1][f0] = cn9k_nix_xmit_pkts_mseg_##name, + + NIX_TX_FASTPATH_MODES +#undef T + }; + pick_tx_func(eth_dev, nix_eth_tx_burst); + if (dev->tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) + pick_tx_func(eth_dev, nix_eth_tx_burst_mseg); + rte_mb(); } diff --git a/drivers/net/cnxk/cn9k_tx.h b/drivers/net/cnxk/cn9k_tx.h index 7acecc6f22..d9aa4063c1 100644 --- a/drivers/net/cnxk/cn9k_tx.h +++ b/drivers/net/cnxk/cn9k_tx.h @@ -310,6 +310,111 @@ cn9k_nix_xmit_submit_lmt_release(const rte_iova_t io_addr) return roc_lmt_submit_ldeorl(io_addr); } +static __rte_always_inline uint16_t +cn9k_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags) +{ + struct nix_send_hdr_s *send_hdr; + union nix_send_sg_s *sg; + struct rte_mbuf *m_next; + uint64_t *slist, sg_u; + uint64_t nb_segs; + uint64_t segdw; + uint8_t off, i; + + send_hdr = (struct nix_send_hdr_s *)cmd; + send_hdr->w0.total = m->pkt_len; + send_hdr->w0.aura = roc_npa_aura_handle_to_aura(m->pool->pool_id); + + if (flags & NIX_TX_NEED_EXT_HDR) + off = 2; + else + off = 0; + + sg = (union nix_send_sg_s *)&cmd[2 + off]; + /* Clear sg->u header before use */ + sg->u &= 0xFC00000000000000; + sg_u = sg->u; + slist = &cmd[3 + off]; + + i = 0; + nb_segs = m->nb_segs; + + /* Fill mbuf segments */ + do { + m_next = m->next; + sg_u = sg_u | ((uint64_t)m->data_len << (i << 4)); + *slist = rte_mbuf_data_iova(m); + /* Set invert df if buffer is not to be freed by H/W */ + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { + sg_u |= (cnxk_nix_prefree_seg(m) << (i + 55)); + /* Commit changes to mbuf */ + rte_io_wmb(); + } + /* Mark mempool object as "put" since it is freed by NIX */ +#ifdef RTE_LIBRTE_MEMPOOL_DEBUG + if (!(sg_u & (1ULL << (i + 55)))) + __mempool_check_cookies(m->pool, (void **)&m, 1, 0); + rte_io_wmb(); +#endif + slist++; + i++; + nb_segs--; + if (i > 2 && nb_segs) { + i = 0; + /* Next SG subdesc */ + *(uint64_t *)slist = sg_u & 0xFC00000000000000; + sg->u = sg_u; + sg->segs = 3; + sg = (union nix_send_sg_s *)slist; + sg_u = sg->u; + slist++; + } + m = m_next; + } while (nb_segs); + + sg->u = sg_u; + sg->segs = i; + segdw = (uint64_t *)slist - (uint64_t *)&cmd[2 + off]; + /* Roundup extra dwords to multiple of 2 */ + segdw = (segdw >> 1) + (segdw & 0x1); + /* Default dwords */ + segdw += (off >> 1) + 1; + send_hdr->w0.sizem1 = segdw - 1; + + return segdw; +} + +static __rte_always_inline void +cn9k_nix_xmit_mseg_prep_lmt(uint64_t *cmd, void *lmt_addr, uint16_t segdw) +{ + roc_lmt_mov_seg(lmt_addr, (const void *)cmd, segdw); +} + +static __rte_always_inline void +cn9k_nix_xmit_mseg_one(uint64_t *cmd, void *lmt_addr, rte_iova_t io_addr, + uint16_t segdw) +{ + uint64_t lmt_status; + + do { + roc_lmt_mov_seg(lmt_addr, (const void *)cmd, segdw); + lmt_status = roc_lmt_submit_ldeor(io_addr); + } while (lmt_status == 0); +} + +static __rte_always_inline void +cn9k_nix_xmit_mseg_one_release(uint64_t *cmd, void *lmt_addr, + rte_iova_t io_addr, uint16_t segdw) +{ + uint64_t lmt_status; + + rte_io_wmb(); + do { + roc_lmt_mov_seg(lmt_addr, (const void *)cmd, segdw); + lmt_status = roc_lmt_submit_ldeor(io_addr); + } while (lmt_status == 0); +} + static __rte_always_inline uint16_t cn9k_nix_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts, uint64_t *cmd, const uint16_t flags) @@ -349,6 +454,48 @@ cn9k_nix_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts, return pkts; } +static __rte_always_inline uint16_t +cn9k_nix_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t pkts, uint64_t *cmd, const uint16_t flags) +{ + struct cn9k_eth_txq *txq = tx_queue; + const rte_iova_t io_addr = txq->io_addr; + void *lmt_addr = txq->lmt_addr; + uint64_t lso_tun_fmt; + uint16_t segdw; + uint64_t i; + + NIX_XMIT_FC_OR_RETURN(txq, pkts); + + roc_lmt_mov(cmd, &txq->cmd[0], cn9k_nix_tx_ext_subs(flags)); + + /* Perform header writes before barrier for TSO */ + if (flags & NIX_TX_OFFLOAD_TSO_F) { + lso_tun_fmt = txq->lso_tun_fmt; + + for (i = 0; i < pkts; i++) + cn9k_nix_xmit_prepare_tso(tx_pkts[i], flags); + } + + /* Lets commit any changes in the packet here as no further changes + * to the packet will be done unless no fast free is enabled. + */ + if (!(flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)) + rte_io_wmb(); + + for (i = 0; i < pkts; i++) { + cn9k_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt); + segdw = cn9k_nix_prepare_mseg(tx_pkts[i], cmd, flags); + cn9k_nix_xmit_mseg_one(cmd, lmt_addr, io_addr, segdw); + } + + /* Reduce the cached count */ + txq->fc_cache_pkts -= pkts; + + return pkts; +} + + #define L3L4CSUM_F NIX_TX_OFFLOAD_L3_L4_CSUM_F #define OL3OL4CSUM_F NIX_TX_OFFLOAD_OL3_OL4_CSUM_F #define VLAN_F NIX_TX_OFFLOAD_VLAN_QINQ_F @@ -424,6 +571,9 @@ T(tso_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 1, 6, \ #define T(name, f4, f3, f2, f1, f0, sz, flags) \ uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_##name( \ + void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); \ + \ + uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_mseg_##name( \ void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); NIX_TX_FASTPATH_MODES diff --git a/drivers/net/cnxk/cn9k_tx_mseg.c b/drivers/net/cnxk/cn9k_tx_mseg.c new file mode 100644 index 0000000000..65c5f362ab --- /dev/null +++ b/drivers/net/cnxk/cn9k_tx_mseg.c @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2021 Marvell. + */ + +#include "cn9k_ethdev.h" +#include "cn9k_tx.h" + +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + uint16_t __rte_noinline __rte_hot \ + cn9k_nix_xmit_pkts_mseg_##name(void *tx_queue, \ + struct rte_mbuf **tx_pkts, \ + uint16_t pkts) \ + { \ + uint64_t cmd[(sz) + CNXK_NIX_TX_MSEG_SG_DWORDS - 2]; \ + \ + /* For TSO inner checksum is a must */ \ + if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \ + !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \ + return 0; \ + return cn9k_nix_xmit_pkts_mseg(tx_queue, tx_pkts, pkts, cmd, \ + (flags) | NIX_TX_MULTI_SEG_F); \ + } + +NIX_TX_FASTPATH_MODES +#undef T diff --git a/drivers/net/cnxk/cnxk_ethdev.h b/drivers/net/cnxk/cnxk_ethdev.h index 58cc6b77ea..276e569420 100644 --- a/drivers/net/cnxk/cnxk_ethdev.h +++ b/drivers/net/cnxk/cnxk_ethdev.h @@ -46,6 +46,10 @@ #define CNXK_NIX_TX_NB_SEG_MAX 9 #endif +#define CNXK_NIX_TX_MSEG_SG_DWORDS \ + ((RTE_ALIGN_MUL_CEIL(CNXK_NIX_TX_NB_SEG_MAX, 3) / 3) + \ + CNXK_NIX_TX_NB_SEG_MAX) + #define CNXK_NIX_RSS_L3_L4_SRC_DST \ (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY | \ ETH_RSS_L4_DST_ONLY) diff --git a/drivers/net/cnxk/meson.build b/drivers/net/cnxk/meson.build index 94520022a7..c0f019c33d 100644 --- a/drivers/net/cnxk/meson.build +++ b/drivers/net/cnxk/meson.build @@ -23,6 +23,7 @@ sources += files( 'cn9k_rx_mseg.c', 'cn9k_rx_vec.c', 'cn9k_tx.c', + 'cn9k_tx_mseg.c', ) # CN10K sources += files( -- 2.20.1