From fb3ae0951abd05ff0dc9616a650d81b89c5ac33b Mon Sep 17 00:00:00 2001 From: Jerin Jacob Date: Sun, 2 Jun 2019 16:45:37 +0530 Subject: [PATCH] net/octeontx2: support Tx Add Tx burst support. Signed-off-by: Jerin Jacob Signed-off-by: Nithin Dabilpuram Signed-off-by: Pavan Nikhilesh Signed-off-by: Harman Kalra --- doc/guides/nics/features/octeontx2.ini | 5 + doc/guides/nics/features/octeontx2_vec.ini | 5 + doc/guides/nics/features/octeontx2_vf.ini | 5 + doc/guides/nics/octeontx2.rst | 1 + drivers/net/octeontx2/Makefile | 1 + drivers/net/octeontx2/meson.build | 1 + drivers/net/octeontx2/otx2_ethdev.c | 6 - drivers/net/octeontx2/otx2_ethdev.h | 1 + drivers/net/octeontx2/otx2_tx.c | 94 ++++++++ drivers/net/octeontx2/otx2_tx.h | 261 +++++++++++++++++++++ 10 files changed, 374 insertions(+), 6 deletions(-) create mode 100644 drivers/net/octeontx2/otx2_tx.c diff --git a/doc/guides/nics/features/octeontx2.ini b/doc/guides/nics/features/octeontx2.ini index 3280cba78f..1856d99243 100644 --- a/doc/guides/nics/features/octeontx2.ini +++ b/doc/guides/nics/features/octeontx2.ini @@ -12,6 +12,7 @@ Link status = Y Link status event = Y Runtime Rx queue setup = Y Runtime Tx queue setup = Y +Fast mbuf free = Y Free Tx mbuf on demand = Y Queue start/stop = Y Promiscuous mode = Y @@ -28,6 +29,10 @@ Jumbo frame = Y Scattered Rx = Y VLAN offload = Y QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y Packet type parsing = Y Timesync = Y Timestamp offload = Y diff --git a/doc/guides/nics/features/octeontx2_vec.ini b/doc/guides/nics/features/octeontx2_vec.ini index 315722e60e..053fca2880 100644 --- a/doc/guides/nics/features/octeontx2_vec.ini +++ b/doc/guides/nics/features/octeontx2_vec.ini @@ -12,6 +12,7 @@ Link status = Y Link status event = Y Runtime Rx queue setup = Y Runtime Tx queue setup = Y +Fast mbuf free = Y Free Tx mbuf on demand = Y Queue start/stop = Y Promiscuous mode = Y @@ -27,6 +28,10 @@ Flow API = Y Jumbo frame = Y VLAN offload = Y QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y Packet type parsing = Y Rx descriptor status = Y Basic stats = Y diff --git a/doc/guides/nics/features/octeontx2_vf.ini b/doc/guides/nics/features/octeontx2_vf.ini index 17b223221c..bef451d01d 100644 --- a/doc/guides/nics/features/octeontx2_vf.ini +++ b/doc/guides/nics/features/octeontx2_vf.ini @@ -11,6 +11,7 @@ Link status = Y Link status event = Y Runtime Rx queue setup = Y Runtime Tx queue setup = Y +Fast mbuf free = Y Free Tx mbuf on demand = Y Queue start/stop = Y RSS hash = Y @@ -23,6 +24,10 @@ Jumbo frame = Y Scattered Rx = Y VLAN offload = Y QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y Packet type parsing = Y Rx descriptor status = Y Basic stats = Y diff --git a/doc/guides/nics/octeontx2.rst b/doc/guides/nics/octeontx2.rst index 97054d11dd..e92631057e 100644 --- a/doc/guides/nics/octeontx2.rst +++ b/doc/guides/nics/octeontx2.rst @@ -25,6 +25,7 @@ Features of the OCTEON TX2 Ethdev PMD are: - Receiver Side Scaling (RSS) - MAC/VLAN filtering - Generic flow API +- Inner and Outer Checksum offload - VLAN/QinQ stripping and insertion - Port hardware statistics - Link state information diff --git a/drivers/net/octeontx2/Makefile b/drivers/net/octeontx2/Makefile index b8c92a8140..d08d3d854b 100644 --- a/drivers/net/octeontx2/Makefile +++ b/drivers/net/octeontx2/Makefile @@ -35,6 +35,7 @@ LIBABIVER := 1 # SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX2_PMD) += \ otx2_rx.c \ + otx2_tx.c \ otx2_tm.c \ otx2_rss.c \ otx2_mac.c \ diff --git a/drivers/net/octeontx2/meson.build b/drivers/net/octeontx2/meson.build index 9d151f88dc..94bf09a788 100644 --- a/drivers/net/octeontx2/meson.build +++ b/drivers/net/octeontx2/meson.build @@ -3,6 +3,7 @@ # sources = files('otx2_rx.c', + 'otx2_tx.c', 'otx2_tm.c', 'otx2_rss.c', 'otx2_mac.c', diff --git a/drivers/net/octeontx2/otx2_ethdev.c b/drivers/net/octeontx2/otx2_ethdev.c index 3217169459..1081d070ac 100644 --- a/drivers/net/octeontx2/otx2_ethdev.c +++ b/drivers/net/octeontx2/otx2_ethdev.c @@ -14,12 +14,6 @@ #include "otx2_ethdev.h" -static inline void -otx2_eth_set_tx_function(struct rte_eth_dev *eth_dev) -{ - RTE_SET_USED(eth_dev); -} - static inline uint64_t nix_get_rx_offload_capa(struct otx2_eth_dev *dev) { diff --git a/drivers/net/octeontx2/otx2_ethdev.h b/drivers/net/octeontx2/otx2_ethdev.h index 22cf86981c..1f9323fe32 100644 --- a/drivers/net/octeontx2/otx2_ethdev.h +++ b/drivers/net/octeontx2/otx2_ethdev.h @@ -484,6 +484,7 @@ int otx2_ethdev_parse_devargs(struct rte_devargs *devargs, /* Rx and Tx routines */ void otx2_eth_set_rx_function(struct rte_eth_dev *eth_dev); +void otx2_eth_set_tx_function(struct rte_eth_dev *eth_dev); void otx2_nix_form_default_desc(struct otx2_eth_txq *txq); /* Timesync - PTP routines */ diff --git a/drivers/net/octeontx2/otx2_tx.c b/drivers/net/octeontx2/otx2_tx.c new file mode 100644 index 0000000000..16d69b74fc --- /dev/null +++ b/drivers/net/octeontx2/otx2_tx.c @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2019 Marvell International Ltd. + */ + +#include + +#include "otx2_ethdev.h" + +#define NIX_XMIT_FC_OR_RETURN(txq, pkts) do { \ + /* Cached value is low, Update the fc_cache_pkts */ \ + if (unlikely((txq)->fc_cache_pkts < (pkts))) { \ + /* Multiply with sqe_per_sqb to express in pkts */ \ + (txq)->fc_cache_pkts = \ + ((txq)->nb_sqb_bufs_adj - *(txq)->fc_mem) << \ + (txq)->sqes_per_sqb_log2; \ + /* Check it again for the room */ \ + if (unlikely((txq)->fc_cache_pkts < (pkts))) \ + return 0; \ + } \ +} while (0) + + +static __rte_always_inline uint16_t +nix_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t pkts, uint64_t *cmd, const uint16_t flags) +{ + struct otx2_eth_txq *txq = tx_queue; uint16_t i; + const rte_iova_t io_addr = txq->io_addr; + void *lmt_addr = txq->lmt_addr; + + NIX_XMIT_FC_OR_RETURN(txq, pkts); + + otx2_lmt_mov(cmd, &txq->cmd[0], otx2_nix_tx_ext_subs(flags)); + + /* Lets commit any changes in the packet */ + rte_cio_wmb(); + + for (i = 0; i < pkts; i++) { + otx2_nix_xmit_prepare(tx_pkts[i], cmd, flags); + /* Passing no of segdw as 4: HDR + EXT + SG + SMEM */ + otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0], + tx_pkts[i]->ol_flags, 4, flags); + otx2_nix_xmit_one(cmd, lmt_addr, io_addr, flags); + } + + /* Reduce the cached count */ + txq->fc_cache_pkts -= pkts; + + return pkts; +} + +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ +static uint16_t __rte_noinline __hot \ +otx2_nix_xmit_pkts_ ## name(void *tx_queue, \ + struct rte_mbuf **tx_pkts, uint16_t pkts) \ +{ \ + uint64_t cmd[sz]; \ + \ + return nix_xmit_pkts(tx_queue, tx_pkts, pkts, cmd, flags); \ +} + +NIX_TX_FASTPATH_MODES +#undef T + +static inline void +pick_tx_func(struct rte_eth_dev *eth_dev, + const eth_tx_burst_t tx_burst[2][2][2][2][2]) +{ + struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); + + /* [TSTMP] [NOFF] [VLAN] [OL3_OL4_CSUM] [IL3_IL4_CSUM] */ + eth_dev->tx_pkt_burst = tx_burst + [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_TSTAMP_F)] + [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)] + [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_VLAN_QINQ_F)] + [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)] + [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F)]; +} + +void +otx2_eth_set_tx_function(struct rte_eth_dev *eth_dev) +{ + const eth_tx_burst_t nix_eth_tx_burst[2][2][2][2][2] = { +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + [f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_ ## name, + +NIX_TX_FASTPATH_MODES +#undef T + }; + + pick_tx_func(eth_dev, nix_eth_tx_burst); + + rte_mb(); +} diff --git a/drivers/net/octeontx2/otx2_tx.h b/drivers/net/octeontx2/otx2_tx.h index 4d0993f879..db4c1f70f0 100644 --- a/drivers/net/octeontx2/otx2_tx.h +++ b/drivers/net/octeontx2/otx2_tx.h @@ -25,4 +25,265 @@ #define NIX_TX_NEED_EXT_HDR \ (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSTAMP_F) +/* Function to determine no of tx subdesc required in case ext + * sub desc is enabled. + */ +static __rte_always_inline int +otx2_nix_tx_ext_subs(const uint16_t flags) +{ + return (flags & NIX_TX_OFFLOAD_TSTAMP_F) ? 2 : + ((flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) ? 1 : 0); +} + +static __rte_always_inline void +otx2_nix_xmit_prepare_tstamp(uint64_t *cmd, const uint64_t *send_mem_desc, + const uint64_t ol_flags, const uint16_t no_segdw, + const uint16_t flags) +{ + if (flags & NIX_TX_OFFLOAD_TSTAMP_F) { + struct nix_send_mem_s *send_mem; + uint16_t off = (no_segdw - 1) << 1; + + send_mem = (struct nix_send_mem_s *)(cmd + off); + if (flags & NIX_TX_MULTI_SEG_F) + /* Retrieving the default desc values */ + cmd[off] = send_mem_desc[6]; + + /* Packets for which PKT_TX_IEEE1588_TMST is not set, tx tstamp + * should not be updated at tx tstamp registered address, rather + * a dummy address which is eight bytes ahead would be updated + */ + send_mem->addr = (rte_iova_t)((uint64_t *)send_mem_desc[7] + + !(ol_flags & PKT_TX_IEEE1588_TMST)); + } +} + +static inline void +otx2_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags) +{ + struct nix_send_ext_s *send_hdr_ext; + struct nix_send_hdr_s *send_hdr; + uint64_t ol_flags = 0, mask; + union nix_send_hdr_w1_u w1; + union nix_send_sg_s *sg; + + send_hdr = (struct nix_send_hdr_s *)cmd; + if (flags & NIX_TX_NEED_EXT_HDR) { + send_hdr_ext = (struct nix_send_ext_s *)(cmd + 2); + sg = (union nix_send_sg_s *)(cmd + 4); + /* Clear previous markings */ + send_hdr_ext->w0.lso = 0; + send_hdr_ext->w1.u = 0; + } else { + sg = (union nix_send_sg_s *)(cmd + 2); + } + + if (flags & NIX_TX_NEED_SEND_HDR_W1) { + ol_flags = m->ol_flags; + w1.u = 0; + } + + if (!(flags & NIX_TX_MULTI_SEG_F)) { + send_hdr->w0.total = m->data_len; + send_hdr->w0.aura = + npa_lf_aura_handle_to_aura(m->pool->pool_id); + } + + /* + * L3type: 2 => IPV4 + * 3 => IPV4 with csum + * 4 => IPV6 + * L3type and L3ptr needs to be set for either + * L3 csum or L4 csum or LSO + * + */ + + if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) && + (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) { + const uint8_t csum = !!(ol_flags & PKT_TX_OUTER_UDP_CKSUM); + const uint8_t ol3type = + ((!!(ol_flags & PKT_TX_OUTER_IPV4)) << 1) + + ((!!(ol_flags & PKT_TX_OUTER_IPV6)) << 2) + + !!(ol_flags & PKT_TX_OUTER_IP_CKSUM); + + /* Outer L3 */ + w1.ol3type = ol3type; + mask = 0xffffull << ((!!ol3type) << 4); + w1.ol3ptr = ~mask & m->outer_l2_len; + w1.ol4ptr = ~mask & (w1.ol3ptr + m->outer_l3_len); + + /* Outer L4 */ + w1.ol4type = csum + (csum << 1); + + /* Inner L3 */ + w1.il3type = ((!!(ol_flags & PKT_TX_IPV4)) << 1) + + ((!!(ol_flags & PKT_TX_IPV6)) << 2); + w1.il3ptr = w1.ol4ptr + m->l2_len; + w1.il4ptr = w1.il3ptr + m->l3_len; + /* Increment it by 1 if it is IPV4 as 3 is with csum */ + w1.il3type = w1.il3type + !!(ol_flags & PKT_TX_IP_CKSUM); + + /* Inner L4 */ + w1.il4type = (ol_flags & PKT_TX_L4_MASK) >> 52; + + /* In case of no tunnel header use only + * shift IL3/IL4 fields a bit to use + * OL3/OL4 for header checksum + */ + mask = !ol3type; + w1.u = ((w1.u & 0xFFFFFFFF00000000) >> (mask << 3)) | + ((w1.u & 0X00000000FFFFFFFF) >> (mask << 4)); + + } else if (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) { + const uint8_t csum = !!(ol_flags & PKT_TX_OUTER_UDP_CKSUM); + const uint8_t outer_l2_len = m->outer_l2_len; + + /* Outer L3 */ + w1.ol3ptr = outer_l2_len; + w1.ol4ptr = outer_l2_len + m->outer_l3_len; + /* Increment it by 1 if it is IPV4 as 3 is with csum */ + w1.ol3type = ((!!(ol_flags & PKT_TX_OUTER_IPV4)) << 1) + + ((!!(ol_flags & PKT_TX_OUTER_IPV6)) << 2) + + !!(ol_flags & PKT_TX_OUTER_IP_CKSUM); + + /* Outer L4 */ + w1.ol4type = csum + (csum << 1); + + } else if (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) { + const uint8_t l2_len = m->l2_len; + + /* Always use OLXPTR and OLXTYPE when only + * when one header is present + */ + + /* Inner L3 */ + w1.ol3ptr = l2_len; + w1.ol4ptr = l2_len + m->l3_len; + /* Increment it by 1 if it is IPV4 as 3 is with csum */ + w1.ol3type = ((!!(ol_flags & PKT_TX_IPV4)) << 1) + + ((!!(ol_flags & PKT_TX_IPV6)) << 2) + + !!(ol_flags & PKT_TX_IP_CKSUM); + + /* Inner L4 */ + w1.ol4type = (ol_flags & PKT_TX_L4_MASK) >> 52; + } + + if (flags & NIX_TX_NEED_EXT_HDR && + flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) { + send_hdr_ext->w1.vlan1_ins_ena = !!(ol_flags & PKT_TX_VLAN); + /* HW will update ptr after vlan0 update */ + send_hdr_ext->w1.vlan1_ins_ptr = 12; + send_hdr_ext->w1.vlan1_ins_tci = m->vlan_tci; + + send_hdr_ext->w1.vlan0_ins_ena = !!(ol_flags & PKT_TX_QINQ); + /* 2B before end of l2 header */ + send_hdr_ext->w1.vlan0_ins_ptr = 12; + send_hdr_ext->w1.vlan0_ins_tci = m->vlan_tci_outer; + } + + if (flags & NIX_TX_NEED_SEND_HDR_W1) + send_hdr->w1.u = w1.u; + + if (!(flags & NIX_TX_MULTI_SEG_F)) { + sg->seg1_size = m->data_len; + *(rte_iova_t *)(++sg) = rte_mbuf_data_iova(m); + + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { + /* Set don't free bit if reference count > 1 */ + if (rte_pktmbuf_prefree_seg(m) == NULL) + send_hdr->w0.df = 1; /* SET DF */ + } + /* Mark mempool object as "put" since it is freed by NIX */ + if (!send_hdr->w0.df) + __mempool_check_cookies(m->pool, (void **)&m, 1, 0); + } +} + + +static __rte_always_inline void +otx2_nix_xmit_one(uint64_t *cmd, void *lmt_addr, + const rte_iova_t io_addr, const uint32_t flags) +{ + uint64_t lmt_status; + + do { + otx2_lmt_mov(lmt_addr, cmd, otx2_nix_tx_ext_subs(flags)); + lmt_status = otx2_lmt_submit(io_addr); + } while (lmt_status == 0); +} + + +#define L3L4CSUM_F NIX_TX_OFFLOAD_L3_L4_CSUM_F +#define OL3OL4CSUM_F NIX_TX_OFFLOAD_OL3_OL4_CSUM_F +#define VLAN_F NIX_TX_OFFLOAD_VLAN_QINQ_F +#define NOFF_F NIX_TX_OFFLOAD_MBUF_NOFF_F +#define TSP_F NIX_TX_OFFLOAD_TSTAMP_F + +/* [TSTMP] [NOFF] [VLAN] [OL3OL4CSUM] [L3L4CSUM] */ +#define NIX_TX_FASTPATH_MODES \ +T(no_offload, 0, 0, 0, 0, 0, 4, \ + NIX_TX_OFFLOAD_NONE) \ +T(l3l4csum, 0, 0, 0, 0, 1, 4, \ + L3L4CSUM_F) \ +T(ol3ol4csum, 0, 0, 0, 1, 0, 4, \ + OL3OL4CSUM_F) \ +T(ol3ol4csum_l3l4csum, 0, 0, 0, 1, 1, 4, \ + OL3OL4CSUM_F | L3L4CSUM_F) \ +T(vlan, 0, 0, 1, 0, 0, 6, \ + VLAN_F) \ +T(vlan_l3l4csum, 0, 0, 1, 0, 1, 6, \ + VLAN_F | L3L4CSUM_F) \ +T(vlan_ol3ol4csum, 0, 0, 1, 1, 0, 6, \ + VLAN_F | OL3OL4CSUM_F) \ +T(vlan_ol3ol4csum_l3l4csum, 0, 0, 1, 1, 1, 6, \ + VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ +T(noff, 0, 1, 0, 0, 0, 4, \ + NOFF_F) \ +T(noff_l3l4csum, 0, 1, 0, 0, 1, 4, \ + NOFF_F | L3L4CSUM_F) \ +T(noff_ol3ol4csum, 0, 1, 0, 1, 0, 4, \ + NOFF_F | OL3OL4CSUM_F) \ +T(noff_ol3ol4csum_l3l4csum, 0, 1, 0, 1, 1, 4, \ + NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ +T(noff_vlan, 0, 1, 1, 0, 0, 6, \ + NOFF_F | VLAN_F) \ +T(noff_vlan_l3l4csum, 0, 1, 1, 0, 1, 6, \ + NOFF_F | VLAN_F | L3L4CSUM_F) \ +T(noff_vlan_ol3ol4csum, 0, 1, 1, 1, 0, 6, \ + NOFF_F | VLAN_F | OL3OL4CSUM_F) \ +T(noff_vlan_ol3ol4csum_l3l4csum, 0, 1, 1, 1, 1, 6, \ + NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ +T(ts, 1, 0, 0, 0, 0, 8, \ + TSP_F) \ +T(ts_l3l4csum, 1, 0, 0, 0, 1, 8, \ + TSP_F | L3L4CSUM_F) \ +T(ts_ol3ol4csum, 1, 0, 0, 1, 0, 8, \ + TSP_F | OL3OL4CSUM_F) \ +T(ts_ol3ol4csum_l3l4csum, 1, 0, 0, 1, 1, 8, \ + TSP_F | OL3OL4CSUM_F | L3L4CSUM_F) \ +T(ts_vlan, 1, 0, 1, 0, 0, 8, \ + TSP_F | VLAN_F) \ +T(ts_vlan_l3l4csum, 1, 0, 1, 0, 1, 8, \ + TSP_F | VLAN_F | L3L4CSUM_F) \ +T(ts_vlan_ol3ol4csum, 1, 0, 1, 1, 0, 8, \ + TSP_F | VLAN_F | OL3OL4CSUM_F) \ +T(ts_vlan_ol3ol4csum_l3l4csum, 1, 0, 1, 1, 1, 8, \ + TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ +T(ts_noff, 1, 1, 0, 0, 0, 8, \ + TSP_F | NOFF_F) \ +T(ts_noff_l3l4csum, 1, 1, 0, 0, 1, 8, \ + TSP_F | NOFF_F | L3L4CSUM_F) \ +T(ts_noff_ol3ol4csum, 1, 1, 0, 1, 0, 8, \ + TSP_F | NOFF_F | OL3OL4CSUM_F) \ +T(ts_noff_ol3ol4csum_l3l4csum, 1, 1, 0, 1, 1, 8, \ + TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ +T(ts_noff_vlan, 1, 1, 1, 0, 0, 8, \ + TSP_F | NOFF_F | VLAN_F) \ +T(ts_noff_vlan_l3l4csum, 1, 1, 1, 0, 1, 8, \ + TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ +T(ts_noff_vlan_ol3ol4csum, 1, 1, 1, 1, 0, 8, \ + TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ +T(ts_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 1, 8, \ + TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) + #endif /* __OTX2_TX_H__ */ -- 2.20.1