{
struct cn10k_sec_sess_priv sess_priv;
uint32_t pkt_len, dlen_adj, rlen;
+ uint8_t l3l4type, chksum;
uint64x2_t cmd01, cmd23;
uintptr_t dptr, nixtx;
uint64_t ucode_cmd[4];
sess_priv.u64 = *rte_security_dynfield(m);
- if (flags & NIX_TX_NEED_SEND_HDR_W1)
+ if (flags & NIX_TX_NEED_SEND_HDR_W1) {
l2_len = vgetq_lane_u8(*cmd0, 8);
- else
+ /* Extract l3l4type either from il3il4type or ol3ol4type */
+ if (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F &&
+ flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)
+ l3l4type = vgetq_lane_u8(*cmd0, 13);
+ else
+ l3l4type = vgetq_lane_u8(*cmd0, 12);
+
+ chksum = (l3l4type & 0x1) << 1 | !!(l3l4type & 0x30);
+ chksum = ~chksum;
+ sess_priv.chksum = sess_priv.chksum & chksum;
+ /* Clear SEND header flags */
+ *cmd0 = vsetq_lane_u16(0, *cmd0, 6);
+ } else {
l2_len = m->l2_len;
+ }
/* Retrieve DPTR */
dptr = vgetq_lane_u64(*cmd1, 1);
sa_base &= ~0xFFFFUL;
sa = (uintptr_t)roc_nix_inl_ot_ipsec_outb_sa(sa_base, sess_priv.sa_idx);
ucode_cmd[3] = (ROC_CPT_DFLT_ENG_GRP_SE_IE << 61 | 1UL << 60 | sa);
- ucode_cmd[0] =
- (ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC << 48 | pkt_len);
+ ucode_cmd[0] = (ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC << 48 |
+ ((uint64_t)sess_priv.chksum) << 32 | pkt_len);
/* CPT Word 0 and Word 1 */
cmd01 = vdupq_n_u64((nixtx + 16) | (cn10k_nix_tx_ext_subs(flags) + 1));
struct cn10k_sec_sess_priv sess_priv;
uint32_t pkt_len, dlen_adj, rlen;
struct nix_send_hdr_s *send_hdr;
+ uint8_t l3l4type, chksum;
uint64x2_t cmd01, cmd23;
union nix_send_sg_s *sg;
uintptr_t dptr, nixtx;
else
sg = (union nix_send_sg_s *)&cmd[2];
- if (flags & NIX_TX_NEED_SEND_HDR_W1)
+ if (flags & NIX_TX_NEED_SEND_HDR_W1) {
l2_len = cmd[1] & 0xFF;
- else
+ /* Extract l3l4type either from il3il4type or ol3ol4type */
+ if (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F &&
+ flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)
+ l3l4type = (cmd[1] >> 40) & 0xFF;
+ else
+ l3l4type = (cmd[1] >> 32) & 0xFF;
+
+ chksum = (l3l4type & 0x1) << 1 | !!(l3l4type & 0x30);
+ chksum = ~chksum;
+ sess_priv.chksum = sess_priv.chksum & chksum;
+ /* Clear SEND header flags */
+ cmd[1] &= ~(0xFFFFUL << 32);
+ } else {
l2_len = m->l2_len;
+ }
/* Retrieve DPTR */
dptr = *(uint64_t *)(sg + 1);
sa_base &= ~0xFFFFUL;
sa = (uintptr_t)roc_nix_inl_ot_ipsec_outb_sa(sa_base, sess_priv.sa_idx);
ucode_cmd[3] = (ROC_CPT_DFLT_ENG_GRP_SE_IE << 61 | 1UL << 60 | sa);
- ucode_cmd[0] =
- (ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC << 48 | pkt_len);
+ ucode_cmd[0] = (ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC << 48 |
+ ((uint64_t)sess_priv.chksum) << 32 | pkt_len);
/* CPT Word 0 and Word 1. Assume no multi-seg support */
cmd01 = vdupq_n_u64((nixtx + 16) | (cn10k_nix_tx_ext_subs(flags) + 1));
static __rte_always_inline void
cn10k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags,
- const uint64_t lso_tun_fmt, bool *sec)
+ const uint64_t lso_tun_fmt, bool *sec, uint8_t mark_flag,
+ uint64_t mark_fmt)
{
+ uint8_t mark_off = 0, mark_vlan = 0, markptr = 0;
struct nix_send_ext_s *send_hdr_ext;
struct nix_send_hdr_s *send_hdr;
uint64_t ol_flags = 0, mask;
union nix_send_hdr_w1_u w1;
union nix_send_sg_s *sg;
+ uint16_t mark_form = 0;
send_hdr = (struct nix_send_hdr_s *)cmd;
if (flags & NIX_TX_NEED_EXT_HDR) {
sg = (union nix_send_sg_s *)(cmd + 4);
/* Clear previous markings */
send_hdr_ext->w0.lso = 0;
+ send_hdr_ext->w0.mark_en = 0;
send_hdr_ext->w1.u = 0;
+ ol_flags = m->ol_flags;
} else {
sg = (union nix_send_sg_s *)(cmd + 2);
}
}
if (flags & NIX_TX_NEED_EXT_HDR && flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
+ const uint8_t ipv6 = !!(ol_flags & RTE_MBUF_F_TX_IPV6);
+ const uint8_t ip = !!(ol_flags & (RTE_MBUF_F_TX_IPV4 |
+ RTE_MBUF_F_TX_IPV6));
+
send_hdr_ext->w1.vlan1_ins_ena = !!(ol_flags & RTE_MBUF_F_TX_VLAN);
/* HW will update ptr after vlan0 update */
send_hdr_ext->w1.vlan1_ins_ptr = 12;
/* 2B before end of l2 header */
send_hdr_ext->w1.vlan0_ins_ptr = 12;
send_hdr_ext->w1.vlan0_ins_tci = m->vlan_tci_outer;
+ /* Fill for VLAN marking only when VLAN insertion enabled */
+ mark_vlan = ((mark_flag & CNXK_TM_MARK_VLAN_DEI) &
+ (send_hdr_ext->w1.vlan1_ins_ena ||
+ send_hdr_ext->w1.vlan0_ins_ena));
+
+ /* Mask requested flags with packet data information */
+ mark_off = mark_flag & ((ip << 2) | (ip << 1) | mark_vlan);
+ mark_off = ffs(mark_off & CNXK_TM_MARK_MASK);
+
+ mark_form = (mark_fmt >> ((mark_off - !!mark_off) << 4));
+ mark_form = (mark_form >> (ipv6 << 3)) & 0xFF;
+ markptr = m->l2_len + (mark_form >> 7) - (mark_vlan << 2);
+
+ send_hdr_ext->w0.mark_en = !!mark_off;
+ send_hdr_ext->w0.markform = mark_form & 0x7F;
+ send_hdr_ext->w0.markptr = markptr;
}
if (flags & NIX_TX_OFFLOAD_TSO_F && (ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
uintptr_t pa, lbase = txq->lmt_base;
uint16_t lmt_id, burst, left, i;
uintptr_t c_lbase = lbase;
+ uint64_t lso_tun_fmt = 0;
+ uint64_t mark_fmt = 0;
+ uint8_t mark_flag = 0;
rte_iova_t c_io_addr;
- uint64_t lso_tun_fmt;
uint16_t c_lmt_id;
uint64_t sa_base;
uintptr_t laddr;
if (flags & NIX_TX_OFFLOAD_TSO_F)
lso_tun_fmt = txq->lso_tun_fmt;
+ if (flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
+ mark_fmt = txq->mark_fmt;
+ mark_flag = txq->mark_flag;
+ }
+
/* Get LMT base address and LMT ID as lcore id */
ROC_LMT_BASE_ID_GET(lbase, lmt_id);
if (flags & NIX_TX_OFFLOAD_SECURITY_F) {
cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags);
cn10k_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt,
- &sec);
+ &sec, mark_flag, mark_fmt);
laddr = (uintptr_t)LMT_OFF(lbase, lnum, 0);
lnum++;
}
- if (flags & NIX_TX_VWQE_F)
- roc_sso_hws_head_wait(ws[0]);
+ if ((flags & NIX_TX_VWQE_F) && !(ws[1] & BIT_ULL(35)))
+ ws[1] = roc_sso_hws_head_wait(ws[0]);
left -= burst;
tx_pkts += burst;
uint16_t segdw, lmt_id, burst, left, i;
uint8_t lnum, c_lnum, c_loff;
uintptr_t c_lbase = lbase;
+ uint64_t lso_tun_fmt = 0;
+ uint64_t mark_fmt = 0;
+ uint8_t mark_flag = 0;
uint64_t data0, data1;
rte_iova_t c_io_addr;
- uint64_t lso_tun_fmt;
uint8_t shft, c_shft;
__uint128_t data128;
uint16_t c_lmt_id;
if (flags & NIX_TX_OFFLOAD_TSO_F)
lso_tun_fmt = txq->lso_tun_fmt;
+ if (flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
+ mark_fmt = txq->mark_fmt;
+ mark_flag = txq->mark_flag;
+ }
+
/* Get LMT base address and LMT ID as lcore id */
ROC_LMT_BASE_ID_GET(lbase, lmt_id);
if (flags & NIX_TX_OFFLOAD_SECURITY_F) {
cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags);
cn10k_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt,
- &sec);
+ &sec, mark_flag, mark_fmt);
laddr = (uintptr_t)LMT_OFF(lbase, lnum, 0);
}
}
- if (flags & NIX_TX_VWQE_F)
- roc_sso_hws_head_wait(ws[0]);
+ if ((flags & NIX_TX_VWQE_F) && !(ws[1] & BIT_ULL(35)))
+ ws[1] = roc_sso_hws_head_wait(ws[0]);
left -= burst;
tx_pkts += burst;
lnum += 1;
}
+ if (flags & NIX_TX_MULTI_SEG_F) {
+ tx_pkts[0]->next = NULL;
+ tx_pkts[1]->next = NULL;
+ tx_pkts[2]->next = NULL;
+ tx_pkts[3]->next = NULL;
+ }
+
tx_pkts = tx_pkts + NIX_DESCS_PER_LOOP;
}
if (flags & (NIX_TX_MULTI_SEG_F | NIX_TX_OFFLOAD_SECURITY_F))
wd.data[0] >>= 16;
- if (flags & NIX_TX_VWQE_F)
- roc_sso_hws_head_wait(ws[0]);
+ if ((flags & NIX_TX_VWQE_F) && !(ws[1] & BIT_ULL(35)))
+ ws[1] = roc_sso_hws_head_wait(ws[0]);
left -= burst;
/* [T_SEC_F] [TSP] [TSO] [NOFF] [VLAN] [OL3OL4CSUM] [L3L4CSUM] */
#define NIX_TX_FASTPATH_MODES_0_15 \
- T(no_offload, 4, NIX_TX_OFFLOAD_NONE) \
- T(l3l4csum, 4, L3L4CSUM_F) \
- T(ol3ol4csum, 4, OL3OL4CSUM_F) \
- T(ol3ol4csum_l3l4csum, 4, OL3OL4CSUM_F | L3L4CSUM_F) \
+ T(no_offload, 6, NIX_TX_OFFLOAD_NONE) \
+ T(l3l4csum, 6, L3L4CSUM_F) \
+ T(ol3ol4csum, 6, OL3OL4CSUM_F) \
+ T(ol3ol4csum_l3l4csum, 6, OL3OL4CSUM_F | L3L4CSUM_F) \
T(vlan, 6, VLAN_F) \
T(vlan_l3l4csum, 6, VLAN_F | L3L4CSUM_F) \
T(vlan_ol3ol4csum, 6, VLAN_F | OL3OL4CSUM_F) \
T(vlan_ol3ol4csum_l3l4csum, 6, VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
- T(noff, 4, NOFF_F) \
- T(noff_l3l4csum, 4, NOFF_F | L3L4CSUM_F) \
- T(noff_ol3ol4csum, 4, NOFF_F | OL3OL4CSUM_F) \
- T(noff_ol3ol4csum_l3l4csum, 4, NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+ T(noff, 6, NOFF_F) \
+ T(noff_l3l4csum, 6, NOFF_F | L3L4CSUM_F) \
+ T(noff_ol3ol4csum, 6, NOFF_F | OL3OL4CSUM_F) \
+ T(noff_ol3ol4csum_l3l4csum, 6, NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
T(noff_vlan, 6, NOFF_F | VLAN_F) \
T(noff_vlan_l3l4csum, 6, NOFF_F | VLAN_F | L3L4CSUM_F) \
T(noff_vlan_ol3ol4csum, 6, NOFF_F | VLAN_F | OL3OL4CSUM_F) \
TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)
#define NIX_TX_FASTPATH_MODES_64_79 \
- T(sec, 4, T_SEC_F) \
- T(sec_l3l4csum, 4, T_SEC_F | L3L4CSUM_F) \
- T(sec_ol3ol4csum, 4, T_SEC_F | OL3OL4CSUM_F) \
- T(sec_ol3ol4csum_l3l4csum, 4, T_SEC_F | OL3OL4CSUM_F | L3L4CSUM_F) \
+ T(sec, 6, T_SEC_F) \
+ T(sec_l3l4csum, 6, T_SEC_F | L3L4CSUM_F) \
+ T(sec_ol3ol4csum, 6, T_SEC_F | OL3OL4CSUM_F) \
+ T(sec_ol3ol4csum_l3l4csum, 6, T_SEC_F | OL3OL4CSUM_F | L3L4CSUM_F) \
T(sec_vlan, 6, T_SEC_F | VLAN_F) \
T(sec_vlan_l3l4csum, 6, T_SEC_F | VLAN_F | L3L4CSUM_F) \
T(sec_vlan_ol3ol4csum, 6, T_SEC_F | VLAN_F | OL3OL4CSUM_F) \
T(sec_vlan_ol3ol4csum_l3l4csum, 6, \
T_SEC_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
- T(sec_noff, 4, T_SEC_F | NOFF_F) \
- T(sec_noff_l3l4csum, 4, T_SEC_F | NOFF_F | L3L4CSUM_F) \
- T(sec_noff_ol3ol4csum, 4, T_SEC_F | NOFF_F | OL3OL4CSUM_F) \
- T(sec_noff_ol3ol4csum_l3l4csum, 4, \
+ T(sec_noff, 6, T_SEC_F | NOFF_F) \
+ T(sec_noff_l3l4csum, 6, T_SEC_F | NOFF_F | L3L4CSUM_F) \
+ T(sec_noff_ol3ol4csum, 6, T_SEC_F | NOFF_F | OL3OL4CSUM_F) \
+ T(sec_noff_ol3ol4csum_l3l4csum, 6, \
T_SEC_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
T(sec_noff_vlan, 6, T_SEC_F | NOFF_F | VLAN_F) \
T(sec_noff_vlan_l3l4csum, 6, T_SEC_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
git.droids-corp.org / dpdk.git / blobdiff