#define NIX_TX_OFFLOAD_VLAN_QINQ_F BIT(2)
#define NIX_TX_OFFLOAD_MBUF_NOFF_F BIT(3)
#define NIX_TX_OFFLOAD_TSO_F BIT(4)
+#define NIX_TX_OFFLOAD_TSTAMP_F BIT(5)
/* Flags to control xmit_prepare function.
* Defining it from backwards to denote its been
NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F)
#define NIX_TX_NEED_EXT_HDR \
- (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F)
+ (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSTAMP_F | \
+ NIX_TX_OFFLOAD_TSO_F)
#define NIX_XMIT_FC_OR_RETURN(txq, pkts) \
do { \
static __rte_always_inline int
cn10k_nix_tx_ext_subs(const uint16_t flags)
{
- return (flags &
- (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F)) ? 1 : 0;
+ return (flags & NIX_TX_OFFLOAD_TSTAMP_F)
+ ? 2
+ : ((flags &
+ (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F))
+ ? 1
+ : 0);
}
static __rte_always_inline uint8_t
cn10k_nix_pkts_per_vec_brst(const uint16_t flags)
{
- RTE_SET_USED(flags);
- /* We can pack up to 4 packets per LMTLINE if there are no offloads. */
- return 4 << ROC_LMT_LINES_PER_CORE_LOG2;
+ return ((flags & NIX_TX_NEED_EXT_HDR) ? 2 : 4)
+ << ROC_LMT_LINES_PER_CORE_LOG2;
+}
+
+static __rte_always_inline uint8_t
+cn10k_nix_tx_dwords_per_line(const uint16_t flags)
+{
+ return (flags & NIX_TX_NEED_EXT_HDR) ?
+ ((flags & NIX_TX_OFFLOAD_TSTAMP_F) ? 8 : 6) :
+ 8;
}
static __rte_always_inline uint64_t
static __rte_always_inline uint64_t
cn10k_nix_tx_steor_vec_data(const uint16_t flags)
{
- const uint64_t dw_m1 = 0x7;
+ const uint64_t dw_m1 = cn10k_nix_tx_dwords_per_line(flags) - 1;
uint64_t data;
- RTE_SET_USED(flags);
/* This will be moved to addr area */
data = dw_m1;
/* 15 vector sizes for single seg */
*(rte_iova_t *)(lmt_addr + 8) = *(rte_iova_t *)(sg + 1);
}
+static __rte_always_inline void
+cn10k_nix_xmit_prepare_tstamp(uintptr_t lmt_addr, const uint64_t *cmd,
+ const uint64_t ol_flags, const uint16_t no_segdw,
+ const uint16_t flags)
+{
+ if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
+ const uint8_t is_ol_tstamp = !(ol_flags & PKT_TX_IEEE1588_TMST);
+ struct nix_send_ext_s *send_hdr_ext =
+ (struct nix_send_ext_s *)lmt_addr + 16;
+ uint64_t *lmt = (uint64_t *)lmt_addr;
+ uint16_t off = (no_segdw - 1) << 1;
+ struct nix_send_mem_s *send_mem;
+
+ send_mem = (struct nix_send_mem_s *)(lmt + off);
+ send_hdr_ext->w0.subdc = NIX_SUBDC_EXT;
+ send_hdr_ext->w0.tstmp = 1;
+ if (flags & NIX_TX_MULTI_SEG_F) {
+ /* Retrieving the default desc values */
+ lmt[off] = cmd[2];
+
+ /* Using compiler barier to avoid voilation of C
+ * aliasing rules.
+ */
+ rte_compiler_barrier();
+ }
+
+ /* Packets for which PKT_TX_IEEE1588_TMST is not set, tx tstamp
+ * should not be recorded, hence changing the alg type to
+ * NIX_SENDMEMALG_SET and also changing send mem addr field to
+ * next 8 bytes as it corrpt the actual tx tstamp registered
+ * address.
+ */
+ send_mem->w0.subdc = NIX_SUBDC_MEM;
+ send_mem->w0.alg = NIX_SENDMEMALG_SETTSTMP - (is_ol_tstamp);
+ send_mem->addr =
+ (rte_iova_t)(((uint64_t *)cmd[3]) + is_ol_tstamp);
+ }
+}
+
static __rte_always_inline uint16_t
cn10k_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
{
/* Roundup extra dwords to multiple of 2 */
segdw = (segdw >> 1) + (segdw & 0x1);
/* Default dwords */
- segdw += (off >> 1) + 1;
+ segdw += (off >> 1) + 1 + !!(flags & NIX_TX_OFFLOAD_TSTAMP_F);
send_hdr->w0.sizem1 = segdw - 1;
return segdw;
cn10k_nix_xmit_prepare(tx_pkts[i], cmd, lmt_addr, flags,
lso_tun_fmt);
+ cn10k_nix_xmit_prepare_tstamp(lmt_addr, &txq->cmd[0],
+ tx_pkts[i]->ol_flags, 4, flags);
lmt_addr += (1ULL << ROC_LMT_LINE_SIZE_LOG2);
}
/* Store sg list directly on lmt line */
segdw = cn10k_nix_prepare_mseg(tx_pkts[i], (uint64_t *)lmt_addr,
flags);
+ cn10k_nix_xmit_prepare_tstamp(lmt_addr, &txq->cmd[0],
+ tx_pkts[i]->ol_flags, segdw,
+ flags);
lmt_addr += (1ULL << ROC_LMT_LINE_SIZE_LOG2);
data128 |= (((__uint128_t)(segdw - 1)) << shft);
shft += 3;
#if defined(RTE_ARCH_ARM64)
+static __rte_always_inline void
+cn10k_nix_prepare_tso(struct rte_mbuf *m, union nix_send_hdr_w1_u *w1,
+ union nix_send_ext_w0_u *w0, uint64_t ol_flags,
+ const uint64_t flags, const uint64_t lso_tun_fmt)
+{
+ uint16_t lso_sb;
+ uint64_t mask;
+
+ if (!(ol_flags & PKT_TX_TCP_SEG))
+ return;
+
+ mask = -(!w1->il3type);
+ lso_sb = (mask & w1->ol4ptr) + (~mask & w1->il4ptr) + m->l4_len;
+
+ w0->u |= BIT(14);
+ w0->lso_sb = lso_sb;
+ w0->lso_mps = m->tso_segsz;
+ w0->lso_format = NIX_LSO_FORMAT_IDX_TSOV4 + !!(ol_flags & PKT_TX_IPV6);
+ w1->ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
+
+ /* Handle tunnel tso */
+ if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
+ (ol_flags & PKT_TX_TUNNEL_MASK)) {
+ const uint8_t is_udp_tun =
+ (CNXK_NIX_UDP_TUN_BITMASK >>
+ ((ol_flags & PKT_TX_TUNNEL_MASK) >> 45)) &
+ 0x1;
+ uint8_t shift = is_udp_tun ? 32 : 0;
+
+ shift += (!!(ol_flags & PKT_TX_OUTER_IPV6) << 4);
+ shift += (!!(ol_flags & PKT_TX_IPV6) << 3);
+
+ w1->il4type = NIX_SENDL4TYPE_TCP_CKSUM;
+ w1->ol4type = is_udp_tun ? NIX_SENDL4TYPE_UDP_CKSUM : 0;
+ /* Update format for UDP tunneled packet */
+
+ w0->lso_format = (lso_tun_fmt >> shift);
+ }
+}
+
#define NIX_DESCS_PER_LOOP 4
static __rte_always_inline uint16_t
cn10k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
{
uint64x2_t dataoff_iova0, dataoff_iova1, dataoff_iova2, dataoff_iova3;
uint64x2_t len_olflags0, len_olflags1, len_olflags2, len_olflags3;
- uint64x2_t cmd0[NIX_DESCS_PER_LOOP], cmd1[NIX_DESCS_PER_LOOP];
+ uint64x2_t cmd0[NIX_DESCS_PER_LOOP], cmd1[NIX_DESCS_PER_LOOP],
+ cmd2[NIX_DESCS_PER_LOOP], cmd3[NIX_DESCS_PER_LOOP];
uint64_t *mbuf0, *mbuf1, *mbuf2, *mbuf3, data, pa;
uint64x2_t senddesc01_w0, senddesc23_w0;
uint64x2_t senddesc01_w1, senddesc23_w1;
uint16_t left, scalar, burst, i, lmt_id;
+ uint64x2_t sendext01_w0, sendext23_w0;
+ uint64x2_t sendext01_w1, sendext23_w1;
+ uint64x2_t sendmem01_w0, sendmem23_w0;
+ uint64x2_t sendmem01_w1, sendmem23_w1;
uint64x2_t sgdesc01_w0, sgdesc23_w0;
uint64x2_t sgdesc01_w1, sgdesc23_w1;
struct cn10k_eth_txq *txq = tx_queue;
/* Reduce the cached count */
txq->fc_cache_pkts -= pkts;
+ /* Perform header writes before barrier for TSO */
+ if (flags & NIX_TX_OFFLOAD_TSO_F) {
+ for (i = 0; i < pkts; i++)
+ cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags);
+ }
senddesc01_w0 = vld1q_dup_u64(&txq->send_hdr_w0);
senddesc23_w0 = senddesc01_w0;
sgdesc01_w0 = vld1q_dup_u64(&txq->sg_w0);
sgdesc23_w0 = sgdesc01_w0;
+ /* Load command defaults into vector variables. */
+ if (flags & NIX_TX_NEED_EXT_HDR) {
+ sendext01_w0 = vld1q_dup_u64(&txq->cmd[0]);
+ sendext23_w0 = sendext01_w0;
+ sendext01_w1 = vdupq_n_u64(12 | 12U << 24);
+ sendext23_w1 = sendext01_w1;
+ if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
+ sendmem01_w0 = vld1q_dup_u64(&txq->cmd[2]);
+ sendmem23_w0 = sendmem01_w0;
+ sendmem01_w1 = vld1q_dup_u64(&txq->cmd[3]);
+ sendmem23_w1 = sendmem01_w1;
+ }
+ }
+
/* Get LMT base address and LMT ID as lcore id */
ROC_LMT_BASE_ID_GET(laddr, lmt_id);
left = pkts;
senddesc23_w0 = senddesc01_w0;
sgdesc23_w0 = sgdesc01_w0;
+ /* Clear vlan enables. */
+ if (flags & NIX_TX_NEED_EXT_HDR) {
+ sendext01_w1 = vbicq_u64(sendext01_w1,
+ vdupq_n_u64(0x3FFFF00FFFF00));
+ sendext23_w1 = sendext01_w1;
+ }
+
+ if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
+ /* Reset send mem alg to SETTSTMP from SUB*/
+ sendmem01_w0 = vbicq_u64(sendmem01_w0,
+ vdupq_n_u64(BIT_ULL(59)));
+ /* Reset send mem address to default. */
+ sendmem01_w1 =
+ vbicq_u64(sendmem01_w1, vdupq_n_u64(0xF));
+ sendmem23_w0 = sendmem01_w0;
+ sendmem23_w1 = sendmem01_w1;
+ }
+
+ if (flags & NIX_TX_OFFLOAD_TSO_F) {
+ /* Clear the LSO enable bit. */
+ sendext01_w0 = vbicq_u64(sendext01_w0,
+ vdupq_n_u64(BIT_ULL(14)));
+ sendext23_w0 = sendext01_w0;
+ }
+
/* Move mbufs to iova */
mbuf0 = (uint64_t *)tx_pkts[0];
mbuf1 = (uint64_t *)tx_pkts[1];
senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
+ if (flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
+ /* Tx ol_flag for vlan. */
+ const uint64x2_t olv = {PKT_TX_VLAN, PKT_TX_VLAN};
+ /* Bit enable for VLAN1 */
+ const uint64x2_t mlv = {BIT_ULL(49), BIT_ULL(49)};
+ /* Tx ol_flag for QnQ. */
+ const uint64x2_t olq = {PKT_TX_QINQ, PKT_TX_QINQ};
+ /* Bit enable for VLAN0 */
+ const uint64x2_t mlq = {BIT_ULL(48), BIT_ULL(48)};
+ /* Load vlan values from packet. outer is VLAN 0 */
+ uint64x2_t ext01 = {
+ ((uint32_t)tx_pkts[0]->vlan_tci_outer) << 8 |
+ ((uint64_t)tx_pkts[0]->vlan_tci) << 32,
+ ((uint32_t)tx_pkts[1]->vlan_tci_outer) << 8 |
+ ((uint64_t)tx_pkts[1]->vlan_tci) << 32,
+ };
+ uint64x2_t ext23 = {
+ ((uint32_t)tx_pkts[2]->vlan_tci_outer) << 8 |
+ ((uint64_t)tx_pkts[2]->vlan_tci) << 32,
+ ((uint32_t)tx_pkts[3]->vlan_tci_outer) << 8 |
+ ((uint64_t)tx_pkts[3]->vlan_tci) << 32,
+ };
+
+ /* Get ol_flags of the packets. */
+ xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
+ ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
+
+ /* ORR vlan outer/inner values into cmd. */
+ sendext01_w1 = vorrq_u64(sendext01_w1, ext01);
+ sendext23_w1 = vorrq_u64(sendext23_w1, ext23);
+
+ /* Test for offload enable bits and generate masks. */
+ xtmp128 = vorrq_u64(vandq_u64(vtstq_u64(xtmp128, olv),
+ mlv),
+ vandq_u64(vtstq_u64(xtmp128, olq),
+ mlq));
+ ytmp128 = vorrq_u64(vandq_u64(vtstq_u64(ytmp128, olv),
+ mlv),
+ vandq_u64(vtstq_u64(ytmp128, olq),
+ mlq));
+
+ /* Set vlan enable bits into cmd based on mask. */
+ sendext01_w1 = vorrq_u64(sendext01_w1, xtmp128);
+ sendext23_w1 = vorrq_u64(sendext23_w1, ytmp128);
+ }
+
+ if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
+ /* Tx ol_flag for timestam. */
+ const uint64x2_t olf = {PKT_TX_IEEE1588_TMST,
+ PKT_TX_IEEE1588_TMST};
+ /* Set send mem alg to SUB. */
+ const uint64x2_t alg = {BIT_ULL(59), BIT_ULL(59)};
+ /* Increment send mem address by 8. */
+ const uint64x2_t addr = {0x8, 0x8};
+
+ xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
+ ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
+
+ /* Check if timestamp is requested and generate inverted
+ * mask as we need not make any changes to default cmd
+ * value.
+ */
+ xtmp128 = vmvnq_u32(vtstq_u64(olf, xtmp128));
+ ytmp128 = vmvnq_u32(vtstq_u64(olf, ytmp128));
+
+ /* Change send mem address to an 8 byte offset when
+ * TSTMP is disabled.
+ */
+ sendmem01_w1 = vaddq_u64(sendmem01_w1,
+ vandq_u64(xtmp128, addr));
+ sendmem23_w1 = vaddq_u64(sendmem23_w1,
+ vandq_u64(ytmp128, addr));
+ /* Change send mem alg to SUB when TSTMP is disabled. */
+ sendmem01_w0 = vorrq_u64(sendmem01_w0,
+ vandq_u64(xtmp128, alg));
+ sendmem23_w0 = vorrq_u64(sendmem23_w0,
+ vandq_u64(ytmp128, alg));
+
+ cmd3[0] = vzip1q_u64(sendmem01_w0, sendmem01_w1);
+ cmd3[1] = vzip2q_u64(sendmem01_w0, sendmem01_w1);
+ cmd3[2] = vzip1q_u64(sendmem23_w0, sendmem23_w1);
+ cmd3[3] = vzip2q_u64(sendmem23_w0, sendmem23_w1);
+ }
+
+ if (flags & NIX_TX_OFFLOAD_TSO_F) {
+ const uint64_t lso_fmt = txq->lso_tun_fmt;
+ uint64_t sx_w0[NIX_DESCS_PER_LOOP];
+ uint64_t sd_w1[NIX_DESCS_PER_LOOP];
+
+ /* Extract SD W1 as we need to set L4 types. */
+ vst1q_u64(sd_w1, senddesc01_w1);
+ vst1q_u64(sd_w1 + 2, senddesc23_w1);
+
+ /* Extract SX W0 as we need to set LSO fields. */
+ vst1q_u64(sx_w0, sendext01_w0);
+ vst1q_u64(sx_w0 + 2, sendext23_w0);
+
+ /* Extract ol_flags. */
+ xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
+ ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
+
+ /* Prepare individual mbufs. */
+ cn10k_nix_prepare_tso(tx_pkts[0],
+ (union nix_send_hdr_w1_u *)&sd_w1[0],
+ (union nix_send_ext_w0_u *)&sx_w0[0],
+ vgetq_lane_u64(xtmp128, 0), flags, lso_fmt);
+
+ cn10k_nix_prepare_tso(tx_pkts[1],
+ (union nix_send_hdr_w1_u *)&sd_w1[1],
+ (union nix_send_ext_w0_u *)&sx_w0[1],
+ vgetq_lane_u64(xtmp128, 1), flags, lso_fmt);
+
+ cn10k_nix_prepare_tso(tx_pkts[2],
+ (union nix_send_hdr_w1_u *)&sd_w1[2],
+ (union nix_send_ext_w0_u *)&sx_w0[2],
+ vgetq_lane_u64(ytmp128, 0), flags, lso_fmt);
+
+ cn10k_nix_prepare_tso(tx_pkts[3],
+ (union nix_send_hdr_w1_u *)&sd_w1[3],
+ (union nix_send_ext_w0_u *)&sx_w0[3],
+ vgetq_lane_u64(ytmp128, 1), flags, lso_fmt);
+
+ senddesc01_w1 = vld1q_u64(sd_w1);
+ senddesc23_w1 = vld1q_u64(sd_w1 + 2);
+
+ sendext01_w0 = vld1q_u64(sx_w0);
+ sendext23_w0 = vld1q_u64(sx_w0 + 2);
+ }
+
if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {
/* Set don't free bit if reference count > 1 */
xmask01 = vdupq_n_u64(0);
cmd1[2] = vzip1q_u64(sgdesc23_w0, sgdesc23_w1);
cmd1[3] = vzip2q_u64(sgdesc23_w0, sgdesc23_w1);
- /* Store the prepared send desc to LMT lines */
- vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[0]);
- vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd1[0]);
- vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd0[1]);
- vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd1[1]);
- vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd0[2]);
- vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd1[2]);
- vst1q_u64(LMT_OFF(laddr, lnum, 96), cmd0[3]);
- vst1q_u64(LMT_OFF(laddr, lnum, 112), cmd1[3]);
- lnum += 1;
+ if (flags & NIX_TX_NEED_EXT_HDR) {
+ cmd2[0] = vzip1q_u64(sendext01_w0, sendext01_w1);
+ cmd2[1] = vzip2q_u64(sendext01_w0, sendext01_w1);
+ cmd2[2] = vzip1q_u64(sendext23_w0, sendext23_w1);
+ cmd2[3] = vzip2q_u64(sendext23_w0, sendext23_w1);
+ }
+
+ if (flags & NIX_TX_NEED_EXT_HDR) {
+ /* Store the prepared send desc to LMT lines */
+ if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
+ vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd2[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd1[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd3[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd0[1]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd2[1]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 96), cmd1[1]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 112), cmd3[1]);
+ lnum += 1;
+ vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd2[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd1[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd3[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd0[3]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd2[3]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 96), cmd1[3]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 112), cmd3[3]);
+ } else {
+ vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd2[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd1[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd0[1]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd2[1]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd1[1]);
+ lnum += 1;
+ vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd2[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd1[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd0[3]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd2[3]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd1[3]);
+ }
+ lnum += 1;
+ } else {
+ /* Store the prepared send desc to LMT lines */
+ vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd1[0]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd0[1]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd1[1]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd0[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd1[2]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 96), cmd0[3]);
+ vst1q_u64(LMT_OFF(laddr, lnum, 112), cmd1[3]);
+ lnum += 1;
+ }
tx_pkts = tx_pkts + NIX_DESCS_PER_LOOP;
}
#define VLAN_F NIX_TX_OFFLOAD_VLAN_QINQ_F
#define NOFF_F NIX_TX_OFFLOAD_MBUF_NOFF_F
#define TSO_F NIX_TX_OFFLOAD_TSO_F
+#define TSP_F NIX_TX_OFFLOAD_TSTAMP_F
-/* [TSO] [NOFF] [VLAN] [OL3OL4CSUM] [L3L4CSUM] */
+/* [TSP] [TSO] [NOFF] [VLAN] [OL3OL4CSUM] [L3L4CSUM] */
#define NIX_TX_FASTPATH_MODES \
-T(no_offload, 0, 0, 0, 0, 0, 4, \
+T(no_offload, 0, 0, 0, 0, 0, 0, 4, \
NIX_TX_OFFLOAD_NONE) \
-T(l3l4csum, 0, 0, 0, 0, 1, 4, \
+T(l3l4csum, 0, 0, 0, 0, 0, 1, 4, \
L3L4CSUM_F) \
-T(ol3ol4csum, 0, 0, 0, 1, 0, 4, \
+T(ol3ol4csum, 0, 0, 0, 0, 1, 0, 4, \
OL3OL4CSUM_F) \
-T(ol3ol4csum_l3l4csum, 0, 0, 0, 1, 1, 4, \
+T(ol3ol4csum_l3l4csum, 0, 0, 0, 0, 1, 1, 4, \
OL3OL4CSUM_F | L3L4CSUM_F) \
-T(vlan, 0, 0, 1, 0, 0, 6, \
+T(vlan, 0, 0, 0, 1, 0, 0, 6, \
VLAN_F) \
-T(vlan_l3l4csum, 0, 0, 1, 0, 1, 6, \
+T(vlan_l3l4csum, 0, 0, 0, 1, 0, 1, 6, \
VLAN_F | L3L4CSUM_F) \
-T(vlan_ol3ol4csum, 0, 0, 1, 1, 0, 6, \
+T(vlan_ol3ol4csum, 0, 0, 0, 1, 1, 0, 6, \
VLAN_F | OL3OL4CSUM_F) \
-T(vlan_ol3ol4csum_l3l4csum, 0, 0, 1, 1, 1, 6, \
+T(vlan_ol3ol4csum_l3l4csum, 0, 0, 0, 1, 1, 1, 6, \
VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
-T(noff, 0, 1, 0, 0, 0, 4, \
+T(noff, 0, 0, 1, 0, 0, 0, 4, \
NOFF_F) \
-T(noff_l3l4csum, 0, 1, 0, 0, 1, 4, \
+T(noff_l3l4csum, 0, 0, 1, 0, 0, 1, 4, \
NOFF_F | L3L4CSUM_F) \
-T(noff_ol3ol4csum, 0, 1, 0, 1, 0, 4, \
+T(noff_ol3ol4csum, 0, 0, 1, 0, 1, 0, 4, \
NOFF_F | OL3OL4CSUM_F) \
-T(noff_ol3ol4csum_l3l4csum, 0, 1, 0, 1, 1, 4, \
+T(noff_ol3ol4csum_l3l4csum, 0, 0, 1, 0, 1, 1, 4, \
NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
-T(noff_vlan, 0, 1, 1, 0, 0, 6, \
+T(noff_vlan, 0, 0, 1, 1, 0, 0, 6, \
NOFF_F | VLAN_F) \
-T(noff_vlan_l3l4csum, 0, 1, 1, 0, 1, 6, \
+T(noff_vlan_l3l4csum, 0, 0, 1, 1, 0, 1, 6, \
NOFF_F | VLAN_F | L3L4CSUM_F) \
-T(noff_vlan_ol3ol4csum, 0, 1, 1, 1, 0, 6, \
+T(noff_vlan_ol3ol4csum, 0, 0, 1, 1, 1, 0, 6, \
NOFF_F | VLAN_F | OL3OL4CSUM_F) \
-T(noff_vlan_ol3ol4csum_l3l4csum, 0, 1, 1, 1, 1, 6, \
+T(noff_vlan_ol3ol4csum_l3l4csum, 0, 0, 1, 1, 1, 1, 6, \
NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
-T(tso, 1, 0, 0, 0, 0, 6, \
+T(tso, 0, 1, 0, 0, 0, 0, 6, \
TSO_F) \
-T(tso_l3l4csum, 1, 0, 0, 0, 1, 6, \
+T(tso_l3l4csum, 0, 1, 0, 0, 0, 1, 6, \
TSO_F | L3L4CSUM_F) \
-T(tso_ol3ol4csum, 1, 0, 0, 1, 0, 6, \
+T(tso_ol3ol4csum, 0, 1, 0, 0, 1, 0, 6, \
TSO_F | OL3OL4CSUM_F) \
-T(tso_ol3ol4csum_l3l4csum, 1, 0, 0, 1, 1, 6, \
+T(tso_ol3ol4csum_l3l4csum, 0, 1, 0, 0, 1, 1, 6, \
TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \
-T(tso_vlan, 1, 0, 1, 0, 0, 6, \
+T(tso_vlan, 0, 1, 0, 1, 0, 0, 6, \
TSO_F | VLAN_F) \
-T(tso_vlan_l3l4csum, 1, 0, 1, 0, 1, 6, \
+T(tso_vlan_l3l4csum, 0, 1, 0, 1, 0, 1, 6, \
TSO_F | VLAN_F | L3L4CSUM_F) \
-T(tso_vlan_ol3ol4csum, 1, 0, 1, 1, 0, 6, \
+T(tso_vlan_ol3ol4csum, 0, 1, 0, 1, 1, 0, 6, \
TSO_F | VLAN_F | OL3OL4CSUM_F) \
-T(tso_vlan_ol3ol4csum_l3l4csum, 1, 0, 1, 1, 1, 6, \
+T(tso_vlan_ol3ol4csum_l3l4csum, 0, 1, 0, 1, 1, 1, 6, \
TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
-T(tso_noff, 1, 1, 0, 0, 0, 6, \
+T(tso_noff, 0, 1, 1, 0, 0, 0, 6, \
TSO_F | NOFF_F) \
-T(tso_noff_l3l4csum, 1, 1, 0, 0, 1, 6, \
+T(tso_noff_l3l4csum, 0, 1, 1, 0, 0, 1, 6, \
TSO_F | NOFF_F | L3L4CSUM_F) \
-T(tso_noff_ol3ol4csum, 1, 1, 0, 1, 0, 6, \
+T(tso_noff_ol3ol4csum, 0, 1, 1, 0, 1, 0, 6, \
TSO_F | NOFF_F | OL3OL4CSUM_F) \
-T(tso_noff_ol3ol4csum_l3l4csum, 1, 1, 0, 1, 1, 6, \
+T(tso_noff_ol3ol4csum_l3l4csum, 0, 1, 1, 0, 1, 1, 6, \
TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
-T(tso_noff_vlan, 1, 1, 1, 0, 0, 6, \
+T(tso_noff_vlan, 0, 1, 1, 1, 0, 0, 6, \
TSO_F | NOFF_F | VLAN_F) \
-T(tso_noff_vlan_l3l4csum, 1, 1, 1, 0, 1, 6, \
+T(tso_noff_vlan_l3l4csum, 0, 1, 1, 1, 0, 1, 6, \
TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
-T(tso_noff_vlan_ol3ol4csum, 1, 1, 1, 1, 0, 6, \
+T(tso_noff_vlan_ol3ol4csum, 0, 1, 1, 1, 1, 0, 6, \
TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
-T(tso_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 1, 6, \
- TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)
-
-#define T(name, f4, f3, f2, f1, f0, sz, flags) \
+T(tso_noff_vlan_ol3ol4csum_l3l4csum, 0, 1, 1, 1, 1, 1, 6, \
+ TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+T(ts, 1, 0, 0, 0, 0, 0, 8, \
+ TSP_F) \
+T(ts_l3l4csum, 1, 0, 0, 0, 0, 1, 8, \
+ TSP_F | L3L4CSUM_F) \
+T(ts_ol3ol4csum, 1, 0, 0, 0, 1, 0, 8, \
+ TSP_F | OL3OL4CSUM_F) \
+T(ts_ol3ol4csum_l3l4csum, 1, 0, 0, 0, 1, 1, 8, \
+ TSP_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+T(ts_vlan, 1, 0, 0, 1, 0, 0, 8, \
+ TSP_F | VLAN_F) \
+T(ts_vlan_l3l4csum, 1, 0, 0, 1, 0, 1, 8, \
+ TSP_F | VLAN_F | L3L4CSUM_F) \
+T(ts_vlan_ol3ol4csum, 1, 0, 0, 1, 1, 0, 8, \
+ TSP_F | VLAN_F | OL3OL4CSUM_F) \
+T(ts_vlan_ol3ol4csum_l3l4csum, 1, 0, 0, 1, 1, 1, 8, \
+ TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+T(ts_noff, 1, 0, 1, 0, 0, 0, 8, \
+ TSP_F | NOFF_F) \
+T(ts_noff_l3l4csum, 1, 0, 1, 0, 0, 1, 8, \
+ TSP_F | NOFF_F | L3L4CSUM_F) \
+T(ts_noff_ol3ol4csum, 1, 0, 1, 0, 1, 0, 8, \
+ TSP_F | NOFF_F | OL3OL4CSUM_F) \
+T(ts_noff_ol3ol4csum_l3l4csum, 1, 0, 1, 0, 1, 1, 8, \
+ TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+T(ts_noff_vlan, 1, 0, 1, 1, 0, 0, 8, \
+ TSP_F | NOFF_F | VLAN_F) \
+T(ts_noff_vlan_l3l4csum, 1, 0, 1, 1, 0, 1, 8, \
+ TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
+T(ts_noff_vlan_ol3ol4csum, 1, 0, 1, 1, 1, 0, 8, \
+ TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
+T(ts_noff_vlan_ol3ol4csum_l3l4csum, 1, 0, 1, 1, 1, 1, 8, \
+ TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+T(ts_tso, 1, 1, 0, 0, 0, 0, 8, \
+ TSP_F | TSO_F) \
+T(ts_tso_l3l4csum, 1, 1, 0, 0, 0, 1, 8, \
+ TSP_F | TSO_F | L3L4CSUM_F) \
+T(ts_tso_ol3ol4csum, 1, 1, 0, 0, 1, 0, 8, \
+ TSP_F | TSO_F | OL3OL4CSUM_F) \
+T(ts_tso_ol3ol4csum_l3l4csum, 1, 1, 0, 0, 1, 1, 8, \
+ TSP_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+T(ts_tso_vlan, 1, 1, 0, 1, 0, 0, 8, \
+ TSP_F | TSO_F | VLAN_F) \
+T(ts_tso_vlan_l3l4csum, 1, 1, 0, 1, 0, 1, 8, \
+ TSP_F | TSO_F | VLAN_F | L3L4CSUM_F) \
+T(ts_tso_vlan_ol3ol4csum, 1, 1, 0, 1, 1, 0, 8, \
+ TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \
+T(ts_tso_vlan_ol3ol4csum_l3l4csum, 1, 1, 0, 1, 1, 1, 8, \
+ TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+T(ts_tso_noff, 1, 1, 1, 0, 0, 0, 8, \
+ TSP_F | TSO_F | NOFF_F) \
+T(ts_tso_noff_l3l4csum, 1, 1, 1, 0, 0, 1, 8, \
+ TSP_F | TSO_F | NOFF_F | L3L4CSUM_F) \
+T(ts_tso_noff_ol3ol4csum, 1, 1, 1, 0, 1, 0, 8, \
+ TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \
+T(ts_tso_noff_ol3ol4csum_l3l4csum, 1, 1, 1, 0, 1, 1, 8, \
+ TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+T(ts_tso_noff_vlan, 1, 1, 1, 1, 0, 0, 8, \
+ TSP_F | TSO_F | NOFF_F | VLAN_F) \
+T(ts_tso_noff_vlan_l3l4csum, 1, 1, 1, 1, 0, 1, 8, \
+ TSP_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
+T(ts_tso_noff_vlan_ol3ol4csum, 1, 1, 1, 1, 1, 0, 8, \
+ TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
+T(ts_tso_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 1, 1, 8, \
+ TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)
+
+#define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
uint16_t __rte_noinline __rte_hot cn10k_nix_xmit_pkts_##name( \
void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); \
\
git.droids-corp.org / dpdk.git / blobdiff