NIX_TX_OFFLOAD_TSO_F)
#define NIX_UDP_TUN_BITMASK \
- ((1ull << (PKT_TX_TUNNEL_VXLAN >> 45)) | \
- (1ull << (PKT_TX_TUNNEL_GENEVE >> 45)))
+ ((1ull << (RTE_MBUF_F_TX_TUNNEL_VXLAN >> 45)) | \
+ (1ull << (RTE_MBUF_F_TX_TUNNEL_GENEVE >> 45)))
#define NIX_LSO_FORMAT_IDX_TSOV4 (0)
#define NIX_LSO_FORMAT_IDX_TSOV6 (1)
if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
struct nix_send_mem_s *send_mem;
uint16_t off = (no_segdw - 1) << 1;
- const uint8_t is_ol_tstamp = !(ol_flags & PKT_TX_IEEE1588_TMST);
+ const uint8_t is_ol_tstamp = !(ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST);
send_mem = (struct nix_send_mem_s *)(cmd + off);
if (flags & NIX_TX_MULTI_SEG_F) {
rte_compiler_barrier();
}
- /* Packets for which PKT_TX_IEEE1588_TMST is not set, tx tstamp
+ /* Packets for which RTE_MBUF_F_TX_IEEE1588_TMST is not set, tx tstamp
* should not be recorded, hence changing the alg type to
* NIX_SENDMEMALG_SET and also changing send mem addr field to
* next 8 bytes as it corrpt the actual tx tstamp registered
uint64_t mask, ol_flags = m->ol_flags;
if (flags & NIX_TX_OFFLOAD_TSO_F &&
- (ol_flags & PKT_TX_TCP_SEG)) {
+ (ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
uintptr_t mdata = rte_pktmbuf_mtod(m, uintptr_t);
uint16_t *iplen, *oiplen, *oudplen;
uint16_t lso_sb, paylen;
- mask = -!!(ol_flags & (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IPV6));
+ mask = -!!(ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IPV6));
lso_sb = (mask & (m->outer_l2_len + m->outer_l3_len)) +
m->l2_len + m->l3_len + m->l4_len;
/* Get iplen position assuming no tunnel hdr */
iplen = (uint16_t *)(mdata + m->l2_len +
- (2 << !!(ol_flags & PKT_TX_IPV6)));
+ (2 << !!(ol_flags & RTE_MBUF_F_TX_IPV6)));
/* Handle tunnel tso */
if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
- (ol_flags & PKT_TX_TUNNEL_MASK)) {
+ (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) {
const uint8_t is_udp_tun = (NIX_UDP_TUN_BITMASK >>
- ((ol_flags & PKT_TX_TUNNEL_MASK) >> 45)) & 0x1;
+ ((ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) >> 45)) & 0x1;
oiplen = (uint16_t *)(mdata + m->outer_l2_len +
- (2 << !!(ol_flags & PKT_TX_OUTER_IPV6)));
+ (2 << !!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)));
*oiplen = rte_cpu_to_be_16(rte_be_to_cpu_16(*oiplen) -
paylen);
/* Update iplen position to inner ip hdr */
iplen = (uint16_t *)(mdata + lso_sb - m->l3_len -
- m->l4_len + (2 << !!(ol_flags & PKT_TX_IPV6)));
+ m->l4_len + (2 << !!(ol_flags & RTE_MBUF_F_TX_IPV6)));
}
*iplen = rte_cpu_to_be_16(rte_be_to_cpu_16(*iplen) - paylen);
}
static __rte_always_inline void
-otx2_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
+otx2_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags,
+ const uint64_t lso_tun_fmt)
{
struct nix_send_ext_s *send_hdr_ext;
struct nix_send_hdr_s *send_hdr;
if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
(flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) {
- const uint8_t csum = !!(ol_flags & PKT_TX_OUTER_UDP_CKSUM);
+ const uint8_t csum = !!(ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM);
const uint8_t ol3type =
- ((!!(ol_flags & PKT_TX_OUTER_IPV4)) << 1) +
- ((!!(ol_flags & PKT_TX_OUTER_IPV6)) << 2) +
- !!(ol_flags & PKT_TX_OUTER_IP_CKSUM);
+ ((!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4)) << 1) +
+ ((!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)) << 2) +
+ !!(ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM);
/* Outer L3 */
w1.ol3type = ol3type;
w1.ol4type = csum + (csum << 1);
/* Inner L3 */
- w1.il3type = ((!!(ol_flags & PKT_TX_IPV4)) << 1) +
- ((!!(ol_flags & PKT_TX_IPV6)) << 2);
+ w1.il3type = ((!!(ol_flags & RTE_MBUF_F_TX_IPV4)) << 1) +
+ ((!!(ol_flags & RTE_MBUF_F_TX_IPV6)) << 2);
w1.il3ptr = w1.ol4ptr + m->l2_len;
w1.il4ptr = w1.il3ptr + m->l3_len;
/* Increment it by 1 if it is IPV4 as 3 is with csum */
- w1.il3type = w1.il3type + !!(ol_flags & PKT_TX_IP_CKSUM);
+ w1.il3type = w1.il3type + !!(ol_flags & RTE_MBUF_F_TX_IP_CKSUM);
/* Inner L4 */
- w1.il4type = (ol_flags & PKT_TX_L4_MASK) >> 52;
+ w1.il4type = (ol_flags & RTE_MBUF_F_TX_L4_MASK) >> 52;
/* In case of no tunnel header use only
* shift IL3/IL4 fields a bit to use
((w1.u & 0X00000000FFFFFFFF) >> (mask << 4));
} else if (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) {
- const uint8_t csum = !!(ol_flags & PKT_TX_OUTER_UDP_CKSUM);
+ const uint8_t csum = !!(ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM);
const uint8_t outer_l2_len = m->outer_l2_len;
/* Outer L3 */
w1.ol3ptr = outer_l2_len;
w1.ol4ptr = outer_l2_len + m->outer_l3_len;
/* Increment it by 1 if it is IPV4 as 3 is with csum */
- w1.ol3type = ((!!(ol_flags & PKT_TX_OUTER_IPV4)) << 1) +
- ((!!(ol_flags & PKT_TX_OUTER_IPV6)) << 2) +
- !!(ol_flags & PKT_TX_OUTER_IP_CKSUM);
+ w1.ol3type = ((!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4)) << 1) +
+ ((!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)) << 2) +
+ !!(ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM);
/* Outer L4 */
w1.ol4type = csum + (csum << 1);
w1.ol3ptr = l2_len;
w1.ol4ptr = l2_len + m->l3_len;
/* Increment it by 1 if it is IPV4 as 3 is with csum */
- w1.ol3type = ((!!(ol_flags & PKT_TX_IPV4)) << 1) +
- ((!!(ol_flags & PKT_TX_IPV6)) << 2) +
- !!(ol_flags & PKT_TX_IP_CKSUM);
+ w1.ol3type = ((!!(ol_flags & RTE_MBUF_F_TX_IPV4)) << 1) +
+ ((!!(ol_flags & RTE_MBUF_F_TX_IPV6)) << 2) +
+ !!(ol_flags & RTE_MBUF_F_TX_IP_CKSUM);
/* Inner L4 */
- w1.ol4type = (ol_flags & PKT_TX_L4_MASK) >> 52;
+ w1.ol4type = (ol_flags & RTE_MBUF_F_TX_L4_MASK) >> 52;
}
if (flags & NIX_TX_NEED_EXT_HDR &&
flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
- send_hdr_ext->w1.vlan1_ins_ena = !!(ol_flags & PKT_TX_VLAN);
+ send_hdr_ext->w1.vlan1_ins_ena = !!(ol_flags & RTE_MBUF_F_TX_VLAN);
/* HW will update ptr after vlan0 update */
send_hdr_ext->w1.vlan1_ins_ptr = 12;
send_hdr_ext->w1.vlan1_ins_tci = m->vlan_tci;
- send_hdr_ext->w1.vlan0_ins_ena = !!(ol_flags & PKT_TX_QINQ);
+ send_hdr_ext->w1.vlan0_ins_ena = !!(ol_flags & RTE_MBUF_F_TX_QINQ);
/* 2B before end of l2 header */
send_hdr_ext->w1.vlan0_ins_ptr = 12;
send_hdr_ext->w1.vlan0_ins_tci = m->vlan_tci_outer;
}
if (flags & NIX_TX_OFFLOAD_TSO_F &&
- (ol_flags & PKT_TX_TCP_SEG)) {
+ (ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
uint16_t lso_sb;
uint64_t mask;
send_hdr_ext->w0.lso = 1;
send_hdr_ext->w0.lso_mps = m->tso_segsz;
send_hdr_ext->w0.lso_format =
- NIX_LSO_FORMAT_IDX_TSOV4 + !!(ol_flags & PKT_TX_IPV6);
+ NIX_LSO_FORMAT_IDX_TSOV4 + !!(ol_flags & RTE_MBUF_F_TX_IPV6);
w1.ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
/* Handle tunnel tso */
if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
- (ol_flags & PKT_TX_TUNNEL_MASK)) {
+ (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) {
const uint8_t is_udp_tun = (NIX_UDP_TUN_BITMASK >>
- ((ol_flags & PKT_TX_TUNNEL_MASK) >> 45)) & 0x1;
+ ((ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) >> 45)) & 0x1;
+ uint8_t shift = is_udp_tun ? 32 : 0;
+
+ shift += (!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6) << 4);
+ shift += (!!(ol_flags & RTE_MBUF_F_TX_IPV6) << 3);
w1.il4type = NIX_SENDL4TYPE_TCP_CKSUM;
w1.ol4type = is_udp_tun ? NIX_SENDL4TYPE_UDP_CKSUM : 0;
/* Update format for UDP tunneled packet */
- send_hdr_ext->w0.lso_format += is_udp_tun ? 2 : 6;
-
- send_hdr_ext->w0.lso_format +=
- !!(ol_flags & PKT_TX_OUTER_IPV6) << 1;
+ send_hdr_ext->w0.lso_format = (lso_tun_fmt >> shift);
}
}
* DF bit = 0 otherwise
*/
send_hdr->w0.df = otx2_nix_prefree_seg(m);
+ /* Ensuring mbuf fields which got updated in
+ * otx2_nix_prefree_seg are written before LMTST.
+ */
+ rte_io_wmb();
}
/* Mark mempool object as "put" since it is freed by NIX */
if (!send_hdr->w0.df)
- __mempool_check_cookies(m->pool, (void **)&m, 1, 0);
+ RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
}
}
return otx2_lmt_submit(io_addr);
}
+static __rte_always_inline uint64_t
+otx2_nix_xmit_submit_lmt_release(const rte_iova_t io_addr)
+{
+ return otx2_lmt_submit_release(io_addr);
+}
+
static __rte_always_inline uint16_t
otx2_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
{
sg_u = sg_u | ((uint64_t)m->data_len << (i << 4));
*slist = rte_mbuf_data_iova(m);
/* Set invert df if buffer is not to be freed by H/W */
- if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)
+ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {
sg_u |= (otx2_nix_prefree_seg(m) << (i + 55));
- /* Mark mempool object as "put" since it is freed by NIX */
- if (!(sg_u & (1ULL << (i + 55)))) {
- m->next = NULL;
- __mempool_check_cookies(m->pool, (void **)&m, 1, 0);
+ /* Commit changes to mbuf */
+ rte_io_wmb();
}
+ /* Mark mempool object as "put" since it is freed by NIX */
+#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
+ if (!(sg_u & (1ULL << (i + 55))))
+ RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
+ rte_io_wmb();
+#endif
slist++;
i++;
nb_segs--;
} while (lmt_status == 0);
}
+static __rte_always_inline void
+otx2_nix_xmit_mseg_one_release(uint64_t *cmd, void *lmt_addr,
+ rte_iova_t io_addr, uint16_t segdw)
+{
+ uint64_t lmt_status;
+
+ rte_io_wmb();
+ do {
+ otx2_lmt_mov_seg(lmt_addr, (const void *)cmd, segdw);
+ lmt_status = otx2_lmt_submit(io_addr);
+ } while (lmt_status == 0);
+}
+
#define L3L4CSUM_F NIX_TX_OFFLOAD_L3_L4_CSUM_F
#define OL3OL4CSUM_F NIX_TX_OFFLOAD_OL3_OL4_CSUM_F
#define VLAN_F NIX_TX_OFFLOAD_VLAN_QINQ_F
git.droids-corp.org / dpdk.git / blobdiff