X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fcnxk%2Fcn10k_tx.h;h=feaa41fec709465de0d1054faaa0ee5082270a4f;hb=1e1bfd078e581c155cd5a2b5c9191d87714dfc93;hp=4ae6bbf517331a2c9910701a01d1bfcdc152874c;hpb=ae2c2cb606354d445cfabaacfa6ec8e1bc74fe38;p=dpdk.git diff --git a/drivers/net/cnxk/cn10k_tx.h b/drivers/net/cnxk/cn10k_tx.h index 4ae6bbf517..feaa41fec7 100644 --- a/drivers/net/cnxk/cn10k_tx.h +++ b/drivers/net/cnxk/cn10k_tx.h @@ -246,6 +246,7 @@ cn10k_nix_prep_sec_vec(struct rte_mbuf *m, uint64x2_t *cmd0, uint64x2_t *cmd1, { struct cn10k_sec_sess_priv sess_priv; uint32_t pkt_len, dlen_adj, rlen; + uint8_t l3l4type, chksum; uint64x2_t cmd01, cmd23; uintptr_t dptr, nixtx; uint64_t ucode_cmd[4]; @@ -256,10 +257,23 @@ cn10k_nix_prep_sec_vec(struct rte_mbuf *m, uint64x2_t *cmd0, uint64x2_t *cmd1, sess_priv.u64 = *rte_security_dynfield(m); - if (flags & NIX_TX_NEED_SEND_HDR_W1) + if (flags & NIX_TX_NEED_SEND_HDR_W1) { l2_len = vgetq_lane_u8(*cmd0, 8); - else + /* Extract l3l4type either from il3il4type or ol3ol4type */ + if (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F && + flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) + l3l4type = vgetq_lane_u8(*cmd0, 13); + else + l3l4type = vgetq_lane_u8(*cmd0, 12); + + chksum = (l3l4type & 0x1) << 1 | !!(l3l4type & 0x30); + chksum = ~chksum; + sess_priv.chksum = sess_priv.chksum & chksum; + /* Clear SEND header flags */ + *cmd0 = vsetq_lane_u16(0, *cmd0, 6); + } else { l2_len = m->l2_len; + } /* Retrieve DPTR */ dptr = vgetq_lane_u64(*cmd1, 1); @@ -291,8 +305,8 @@ cn10k_nix_prep_sec_vec(struct rte_mbuf *m, uint64x2_t *cmd0, uint64x2_t *cmd1, sa_base &= ~0xFFFFUL; sa = (uintptr_t)roc_nix_inl_ot_ipsec_outb_sa(sa_base, sess_priv.sa_idx); ucode_cmd[3] = (ROC_CPT_DFLT_ENG_GRP_SE_IE << 61 | 1UL << 60 | sa); - ucode_cmd[0] = - (ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC << 48 | pkt_len); + ucode_cmd[0] = (ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC << 48 | + ((uint64_t)sess_priv.chksum) << 32 | pkt_len); /* CPT Word 0 and Word 1 */ cmd01 = vdupq_n_u64((nixtx + 16) | (cn10k_nix_tx_ext_subs(flags) + 1)); @@ -343,6 +357,7 @@ cn10k_nix_prep_sec(struct rte_mbuf *m, uint64_t *cmd, uintptr_t *nixtx_addr, struct cn10k_sec_sess_priv sess_priv; uint32_t pkt_len, dlen_adj, rlen; struct nix_send_hdr_s *send_hdr; + uint8_t l3l4type, chksum; uint64x2_t cmd01, cmd23; union nix_send_sg_s *sg; uintptr_t dptr, nixtx; @@ -360,10 +375,23 @@ cn10k_nix_prep_sec(struct rte_mbuf *m, uint64_t *cmd, uintptr_t *nixtx_addr, else sg = (union nix_send_sg_s *)&cmd[2]; - if (flags & NIX_TX_NEED_SEND_HDR_W1) + if (flags & NIX_TX_NEED_SEND_HDR_W1) { l2_len = cmd[1] & 0xFF; - else + /* Extract l3l4type either from il3il4type or ol3ol4type */ + if (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F && + flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) + l3l4type = (cmd[1] >> 40) & 0xFF; + else + l3l4type = (cmd[1] >> 32) & 0xFF; + + chksum = (l3l4type & 0x1) << 1 | !!(l3l4type & 0x30); + chksum = ~chksum; + sess_priv.chksum = sess_priv.chksum & chksum; + /* Clear SEND header flags */ + cmd[1] &= ~(0xFFFFUL << 32); + } else { l2_len = m->l2_len; + } /* Retrieve DPTR */ dptr = *(uint64_t *)(sg + 1); @@ -395,8 +423,8 @@ cn10k_nix_prep_sec(struct rte_mbuf *m, uint64_t *cmd, uintptr_t *nixtx_addr, sa_base &= ~0xFFFFUL; sa = (uintptr_t)roc_nix_inl_ot_ipsec_outb_sa(sa_base, sess_priv.sa_idx); ucode_cmd[3] = (ROC_CPT_DFLT_ENG_GRP_SE_IE << 61 | 1UL << 60 | sa); - ucode_cmd[0] = - (ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC << 48 | pkt_len); + ucode_cmd[0] = (ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC << 48 | + ((uint64_t)sess_priv.chksum) << 32 | pkt_len); /* CPT Word 0 and Word 1. Assume no multi-seg support */ cmd01 = vdupq_n_u64((nixtx + 16) | (cn10k_nix_tx_ext_subs(flags) + 1)); @@ -511,13 +539,16 @@ cn10k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags) static __rte_always_inline void cn10k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags, - const uint64_t lso_tun_fmt, bool *sec) + const uint64_t lso_tun_fmt, bool *sec, uint8_t mark_flag, + uint64_t mark_fmt) { + uint8_t mark_off = 0, mark_vlan = 0, markptr = 0; struct nix_send_ext_s *send_hdr_ext; struct nix_send_hdr_s *send_hdr; uint64_t ol_flags = 0, mask; union nix_send_hdr_w1_u w1; union nix_send_sg_s *sg; + uint16_t mark_form = 0; send_hdr = (struct nix_send_hdr_s *)cmd; if (flags & NIX_TX_NEED_EXT_HDR) { @@ -525,7 +556,9 @@ cn10k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags, sg = (union nix_send_sg_s *)(cmd + 4); /* Clear previous markings */ send_hdr_ext->w0.lso = 0; + send_hdr_ext->w0.mark_en = 0; send_hdr_ext->w1.u = 0; + ol_flags = m->ol_flags; } else { sg = (union nix_send_sg_s *)(cmd + 2); } @@ -621,6 +654,10 @@ cn10k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags, } if (flags & NIX_TX_NEED_EXT_HDR && flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) { + const uint8_t ipv6 = !!(ol_flags & RTE_MBUF_F_TX_IPV6); + const uint8_t ip = !!(ol_flags & (RTE_MBUF_F_TX_IPV4 | + RTE_MBUF_F_TX_IPV6)); + send_hdr_ext->w1.vlan1_ins_ena = !!(ol_flags & RTE_MBUF_F_TX_VLAN); /* HW will update ptr after vlan0 update */ send_hdr_ext->w1.vlan1_ins_ptr = 12; @@ -630,6 +667,22 @@ cn10k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags, /* 2B before end of l2 header */ send_hdr_ext->w1.vlan0_ins_ptr = 12; send_hdr_ext->w1.vlan0_ins_tci = m->vlan_tci_outer; + /* Fill for VLAN marking only when VLAN insertion enabled */ + mark_vlan = ((mark_flag & CNXK_TM_MARK_VLAN_DEI) & + (send_hdr_ext->w1.vlan1_ins_ena || + send_hdr_ext->w1.vlan0_ins_ena)); + + /* Mask requested flags with packet data information */ + mark_off = mark_flag & ((ip << 2) | (ip << 1) | mark_vlan); + mark_off = ffs(mark_off & CNXK_TM_MARK_MASK); + + mark_form = (mark_fmt >> ((mark_off - !!mark_off) << 4)); + mark_form = (mark_form >> (ipv6 << 3)) & 0xFF; + markptr = m->l2_len + (mark_form >> 7) - (mark_vlan << 2); + + send_hdr_ext->w0.mark_en = !!mark_off; + send_hdr_ext->w0.markform = mark_form & 0x7F; + send_hdr_ext->w0.markptr = markptr; } if (flags & NIX_TX_OFFLOAD_TSO_F && (ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { @@ -841,8 +894,10 @@ cn10k_nix_xmit_pkts(void *tx_queue, uint64_t *ws, struct rte_mbuf **tx_pkts, uintptr_t pa, lbase = txq->lmt_base; uint16_t lmt_id, burst, left, i; uintptr_t c_lbase = lbase; + uint64_t lso_tun_fmt = 0; + uint64_t mark_fmt = 0; + uint8_t mark_flag = 0; rte_iova_t c_io_addr; - uint64_t lso_tun_fmt; uint16_t c_lmt_id; uint64_t sa_base; uintptr_t laddr; @@ -860,6 +915,11 @@ cn10k_nix_xmit_pkts(void *tx_queue, uint64_t *ws, struct rte_mbuf **tx_pkts, if (flags & NIX_TX_OFFLOAD_TSO_F) lso_tun_fmt = txq->lso_tun_fmt; + if (flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) { + mark_fmt = txq->mark_fmt; + mark_flag = txq->mark_flag; + } + /* Get LMT base address and LMT ID as lcore id */ ROC_LMT_BASE_ID_GET(lbase, lmt_id); if (flags & NIX_TX_OFFLOAD_SECURITY_F) { @@ -887,7 +947,7 @@ again: cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags); cn10k_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt, - &sec); + &sec, mark_flag, mark_fmt); laddr = (uintptr_t)LMT_OFF(lbase, lnum, 0); @@ -905,8 +965,8 @@ again: lnum++; } - if (flags & NIX_TX_VWQE_F) - roc_sso_hws_head_wait(ws[0]); + if ((flags & NIX_TX_VWQE_F) && !(ws[1] & BIT_ULL(35))) + ws[1] = roc_sso_hws_head_wait(ws[0]); left -= burst; tx_pkts += burst; @@ -967,9 +1027,11 @@ cn10k_nix_xmit_pkts_mseg(void *tx_queue, uint64_t *ws, uint16_t segdw, lmt_id, burst, left, i; uint8_t lnum, c_lnum, c_loff; uintptr_t c_lbase = lbase; + uint64_t lso_tun_fmt = 0; + uint64_t mark_fmt = 0; + uint8_t mark_flag = 0; uint64_t data0, data1; rte_iova_t c_io_addr; - uint64_t lso_tun_fmt; uint8_t shft, c_shft; __uint128_t data128; uint16_t c_lmt_id; @@ -988,6 +1050,11 @@ cn10k_nix_xmit_pkts_mseg(void *tx_queue, uint64_t *ws, if (flags & NIX_TX_OFFLOAD_TSO_F) lso_tun_fmt = txq->lso_tun_fmt; + if (flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) { + mark_fmt = txq->mark_fmt; + mark_flag = txq->mark_flag; + } + /* Get LMT base address and LMT ID as lcore id */ ROC_LMT_BASE_ID_GET(lbase, lmt_id); if (flags & NIX_TX_OFFLOAD_SECURITY_F) { @@ -1017,7 +1084,7 @@ again: cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags); cn10k_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt, - &sec); + &sec, mark_flag, mark_fmt); laddr = (uintptr_t)LMT_OFF(lbase, lnum, 0); @@ -1041,8 +1108,8 @@ again: } } - if (flags & NIX_TX_VWQE_F) - roc_sso_hws_head_wait(ws[0]); + if ((flags & NIX_TX_VWQE_F) && !(ws[1] & BIT_ULL(35))) + ws[1] = roc_sso_hws_head_wait(ws[0]); left -= burst; tx_pkts += burst; @@ -2569,6 +2636,13 @@ again: lnum += 1; } + if (flags & NIX_TX_MULTI_SEG_F) { + tx_pkts[0]->next = NULL; + tx_pkts[1]->next = NULL; + tx_pkts[2]->next = NULL; + tx_pkts[3]->next = NULL; + } + tx_pkts = tx_pkts + NIX_DESCS_PER_LOOP; } @@ -2582,8 +2656,8 @@ again: if (flags & (NIX_TX_MULTI_SEG_F | NIX_TX_OFFLOAD_SECURITY_F)) wd.data[0] >>= 16; - if (flags & NIX_TX_VWQE_F) - roc_sso_hws_head_wait(ws[0]); + if ((flags & NIX_TX_VWQE_F) && !(ws[1] & BIT_ULL(35))) + ws[1] = roc_sso_hws_head_wait(ws[0]); left -= burst; @@ -2682,18 +2756,18 @@ cn10k_nix_xmit_pkts_vector(void *tx_queue, uint64_t *ws, /* [T_SEC_F] [TSP] [TSO] [NOFF] [VLAN] [OL3OL4CSUM] [L3L4CSUM] */ #define NIX_TX_FASTPATH_MODES_0_15 \ - T(no_offload, 4, NIX_TX_OFFLOAD_NONE) \ - T(l3l4csum, 4, L3L4CSUM_F) \ - T(ol3ol4csum, 4, OL3OL4CSUM_F) \ - T(ol3ol4csum_l3l4csum, 4, OL3OL4CSUM_F | L3L4CSUM_F) \ + T(no_offload, 6, NIX_TX_OFFLOAD_NONE) \ + T(l3l4csum, 6, L3L4CSUM_F) \ + T(ol3ol4csum, 6, OL3OL4CSUM_F) \ + T(ol3ol4csum_l3l4csum, 6, OL3OL4CSUM_F | L3L4CSUM_F) \ T(vlan, 6, VLAN_F) \ T(vlan_l3l4csum, 6, VLAN_F | L3L4CSUM_F) \ T(vlan_ol3ol4csum, 6, VLAN_F | OL3OL4CSUM_F) \ T(vlan_ol3ol4csum_l3l4csum, 6, VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ - T(noff, 4, NOFF_F) \ - T(noff_l3l4csum, 4, NOFF_F | L3L4CSUM_F) \ - T(noff_ol3ol4csum, 4, NOFF_F | OL3OL4CSUM_F) \ - T(noff_ol3ol4csum_l3l4csum, 4, NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(noff, 6, NOFF_F) \ + T(noff_l3l4csum, 6, NOFF_F | L3L4CSUM_F) \ + T(noff_ol3ol4csum, 6, NOFF_F | OL3OL4CSUM_F) \ + T(noff_ol3ol4csum_l3l4csum, 6, NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ T(noff_vlan, 6, NOFF_F | VLAN_F) \ T(noff_vlan_l3l4csum, 6, NOFF_F | VLAN_F | L3L4CSUM_F) \ T(noff_vlan_ol3ol4csum, 6, NOFF_F | VLAN_F | OL3OL4CSUM_F) \ @@ -2767,19 +2841,19 @@ cn10k_nix_xmit_pkts_vector(void *tx_queue, uint64_t *ws, TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) #define NIX_TX_FASTPATH_MODES_64_79 \ - T(sec, 4, T_SEC_F) \ - T(sec_l3l4csum, 4, T_SEC_F | L3L4CSUM_F) \ - T(sec_ol3ol4csum, 4, T_SEC_F | OL3OL4CSUM_F) \ - T(sec_ol3ol4csum_l3l4csum, 4, T_SEC_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec, 6, T_SEC_F) \ + T(sec_l3l4csum, 6, T_SEC_F | L3L4CSUM_F) \ + T(sec_ol3ol4csum, 6, T_SEC_F | OL3OL4CSUM_F) \ + T(sec_ol3ol4csum_l3l4csum, 6, T_SEC_F | OL3OL4CSUM_F | L3L4CSUM_F) \ T(sec_vlan, 6, T_SEC_F | VLAN_F) \ T(sec_vlan_l3l4csum, 6, T_SEC_F | VLAN_F | L3L4CSUM_F) \ T(sec_vlan_ol3ol4csum, 6, T_SEC_F | VLAN_F | OL3OL4CSUM_F) \ T(sec_vlan_ol3ol4csum_l3l4csum, 6, \ T_SEC_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ - T(sec_noff, 4, T_SEC_F | NOFF_F) \ - T(sec_noff_l3l4csum, 4, T_SEC_F | NOFF_F | L3L4CSUM_F) \ - T(sec_noff_ol3ol4csum, 4, T_SEC_F | NOFF_F | OL3OL4CSUM_F) \ - T(sec_noff_ol3ol4csum_l3l4csum, 4, \ + T(sec_noff, 6, T_SEC_F | NOFF_F) \ + T(sec_noff_l3l4csum, 6, T_SEC_F | NOFF_F | L3L4CSUM_F) \ + T(sec_noff_ol3ol4csum, 6, T_SEC_F | NOFF_F | OL3OL4CSUM_F) \ + T(sec_noff_ol3ol4csum_l3l4csum, 6, \ T_SEC_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ T(sec_noff_vlan, 6, T_SEC_F | NOFF_F | VLAN_F) \ T(sec_noff_vlan_l3l4csum, 6, T_SEC_F | NOFF_F | VLAN_F | L3L4CSUM_F) \