otx2_lmt_mov(cmd, &txq->cmd[0], otx2_nix_tx_ext_subs(flags));
+ /* Perform header writes before barrier for TSO */
+ if (flags & NIX_TX_OFFLOAD_TSO_F) {
+ for (i = 0; i < pkts; i++)
+ otx2_nix_xmit_prepare_tso(tx_pkts[i], flags);
+ }
+
/* Lets commit any changes in the packet */
rte_cio_wmb();
otx2_lmt_mov(cmd, &txq->cmd[0], otx2_nix_tx_ext_subs(flags));
+ /* Perform header writes before barrier for TSO */
+ if (flags & NIX_TX_OFFLOAD_TSO_F) {
+ for (i = 0; i < pkts; i++)
+ otx2_nix_xmit_prepare_tso(tx_pkts[i], flags);
+ }
+
/* Lets commit any changes in the packet */
rte_cio_wmb();
#define NIX_DESCS_PER_LOOP 4
static __rte_always_inline uint16_t
nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
- uint16_t pkts, const uint16_t flags)
+ uint16_t pkts, uint64_t *cmd, const uint16_t flags)
{
uint64x2_t dataoff_iova0, dataoff_iova1, dataoff_iova2, dataoff_iova3;
uint64x2_t len_olflags0, len_olflags1, len_olflags2, len_olflags3;
uint64x2_t cmd20, cmd21;
uint64x2_t cmd30, cmd31;
uint64_t lmt_status, i;
-
- pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP);
+ uint16_t pkts_left;
NIX_XMIT_FC_OR_RETURN(txq, pkts);
+ pkts_left = pkts & (NIX_DESCS_PER_LOOP - 1);
+ pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP);
+
/* Reduce the cached count */
txq->fc_cache_pkts -= pkts;
mbuf = (struct rte_mbuf *)((uintptr_t)mbuf0 -
offsetof(struct rte_mbuf, buf_iova));
- if (rte_pktmbuf_prefree_seg(mbuf) == NULL)
+ if (otx2_nix_prefree_seg(mbuf))
vsetq_lane_u64(0x80000, xmask01, 0);
else
__mempool_check_cookies(mbuf->pool,
mbuf = (struct rte_mbuf *)((uintptr_t)mbuf1 -
offsetof(struct rte_mbuf, buf_iova));
- if (rte_pktmbuf_prefree_seg(mbuf) == NULL)
+ if (otx2_nix_prefree_seg(mbuf))
vsetq_lane_u64(0x80000, xmask01, 1);
else
__mempool_check_cookies(mbuf->pool,
mbuf = (struct rte_mbuf *)((uintptr_t)mbuf2 -
offsetof(struct rte_mbuf, buf_iova));
- if (rte_pktmbuf_prefree_seg(mbuf) == NULL)
+ if (otx2_nix_prefree_seg(mbuf))
vsetq_lane_u64(0x80000, xmask23, 0);
else
__mempool_check_cookies(mbuf->pool,
mbuf = (struct rte_mbuf *)((uintptr_t)mbuf3 -
offsetof(struct rte_mbuf, buf_iova));
- if (rte_pktmbuf_prefree_seg(mbuf) == NULL)
+ if (otx2_nix_prefree_seg(mbuf))
vsetq_lane_u64(0x80000, xmask23, 1);
else
__mempool_check_cookies(mbuf->pool,
} while (lmt_status == 0);
}
+ if (unlikely(pkts_left))
+ pkts += nix_xmit_pkts(tx_queue, tx_pkts, pkts_left, cmd, flags);
+
return pkts;
}
#else
static __rte_always_inline uint16_t
nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
- uint16_t pkts, const uint16_t flags)
+ uint16_t pkts, uint64_t *cmd, const uint16_t flags)
{
RTE_SET_USED(tx_queue);
RTE_SET_USED(tx_pkts);
RTE_SET_USED(pkts);
+ RTE_SET_USED(cmd);
RTE_SET_USED(flags);
return 0;
}
#endif
-#define T(name, f4, f3, f2, f1, f0, sz, flags) \
+#define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
static uint16_t __rte_noinline __hot \
otx2_nix_xmit_pkts_ ## name(void *tx_queue, \
struct rte_mbuf **tx_pkts, uint16_t pkts) \
{ \
uint64_t cmd[sz]; \
\
+ /* For TSO inner checksum is a must */ \
+ if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \
+ !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \
+ return 0; \
return nix_xmit_pkts(tx_queue, tx_pkts, pkts, cmd, flags); \
}
NIX_TX_FASTPATH_MODES
#undef T
-#define T(name, f4, f3, f2, f1, f0, sz, flags) \
+#define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
static uint16_t __rte_noinline __hot \
otx2_nix_xmit_pkts_mseg_ ## name(void *tx_queue, \
struct rte_mbuf **tx_pkts, uint16_t pkts) \
{ \
uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2]; \
\
+ /* For TSO inner checksum is a must */ \
+ if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \
+ !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \
+ return 0; \
return nix_xmit_pkts_mseg(tx_queue, tx_pkts, pkts, cmd, \
(flags) | NIX_TX_MULTI_SEG_F); \
}
NIX_TX_FASTPATH_MODES
#undef T
-#define T(name, f4, f3, f2, f1, f0, sz, flags) \
+#define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
static uint16_t __rte_noinline __hot \
otx2_nix_xmit_pkts_vec_ ## name(void *tx_queue, \
struct rte_mbuf **tx_pkts, uint16_t pkts) \
{ \
- /* VLAN and TSTMP is not supported by vec */ \
+ uint64_t cmd[sz]; \
+ \
+ /* VLAN, TSTMP, TSO is not supported by vec */ \
if ((flags) & NIX_TX_OFFLOAD_VLAN_QINQ_F || \
- (flags) & NIX_TX_OFFLOAD_TSTAMP_F) \
+ (flags) & NIX_TX_OFFLOAD_TSTAMP_F || \
+ (flags) & NIX_TX_OFFLOAD_TSO_F) \
return 0; \
- return nix_xmit_pkts_vector(tx_queue, tx_pkts, pkts, (flags)); \
+ return nix_xmit_pkts_vector(tx_queue, tx_pkts, pkts, cmd, (flags)); \
}
NIX_TX_FASTPATH_MODES
static inline void
pick_tx_func(struct rte_eth_dev *eth_dev,
- const eth_tx_burst_t tx_burst[2][2][2][2][2])
+ const eth_tx_burst_t tx_burst[2][2][2][2][2][2])
{
struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
/* [TSTMP] [NOFF] [VLAN] [OL3_OL4_CSUM] [IL3_IL4_CSUM] */
eth_dev->tx_pkt_burst = tx_burst
+ [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_TSO_F)]
[!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_TSTAMP_F)]
[!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)]
[!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_VLAN_QINQ_F)]
{
struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
- const eth_tx_burst_t nix_eth_tx_burst[2][2][2][2][2] = {
-#define T(name, f4, f3, f2, f1, f0, sz, flags) \
- [f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_ ## name,
+ const eth_tx_burst_t nix_eth_tx_burst[2][2][2][2][2][2] = {
+#define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
+ [f5][f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_ ## name,
NIX_TX_FASTPATH_MODES
#undef T
};
- const eth_tx_burst_t nix_eth_tx_burst_mseg[2][2][2][2][2] = {
-#define T(name, f4, f3, f2, f1, f0, sz, flags) \
- [f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_mseg_ ## name,
+ const eth_tx_burst_t nix_eth_tx_burst_mseg[2][2][2][2][2][2] = {
+#define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
+ [f5][f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_mseg_ ## name,
NIX_TX_FASTPATH_MODES
#undef T
};
- const eth_tx_burst_t nix_eth_tx_vec_burst[2][2][2][2][2] = {
-#define T(name, f4, f3, f2, f1, f0, sz, flags) \
- [f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_vec_ ## name,
+ const eth_tx_burst_t nix_eth_tx_vec_burst[2][2][2][2][2][2] = {
+#define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
+ [f5][f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_vec_ ## name,
NIX_TX_FASTPATH_MODES
#undef T
if (dev->scalar_ena ||
(dev->tx_offload_flags &
- (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSTAMP_F)))
+ (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSTAMP_F |
+ NIX_TX_OFFLOAD_TSO_F)))
pick_tx_func(eth_dev, nix_eth_tx_burst);
else
pick_tx_func(eth_dev, nix_eth_tx_vec_burst);