#define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx},
+/* static asserts */
+static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value");
+static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value");
+static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
+ (sizeof(uint16_t) +
+ sizeof(rte_v128u32_t)),
+ "invalid Ethernet Segment data size");
+static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
+ (sizeof(uint16_t) +
+ sizeof(struct rte_vlan_hdr) +
+ 2 * RTE_ETHER_ADDR_LEN),
+ "invalid Ethernet Segment data size");
+static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
+ (sizeof(uint16_t) +
+ sizeof(rte_v128u32_t)),
+ "invalid Ethernet Segment data size");
+static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
+ (sizeof(uint16_t) +
+ sizeof(struct rte_vlan_hdr) +
+ 2 * RTE_ETHER_ADDR_LEN),
+ "invalid Ethernet Segment data size");
+static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
+ (sizeof(uint16_t) +
+ sizeof(rte_v128u32_t)),
+ "invalid Ethernet Segment data size");
+static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
+ (sizeof(uint16_t) +
+ sizeof(struct rte_vlan_hdr) +
+ 2 * RTE_ETHER_ADDR_LEN),
+ "invalid Ethernet Segment data size");
+static_assert(MLX5_DSEG_MIN_INLINE_SIZE ==
+ (2 * RTE_ETHER_ADDR_LEN),
+ "invalid Data Segment data size");
+static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
+static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
+static_assert((sizeof(struct rte_vlan_hdr) +
+ sizeof(struct rte_ether_hdr)) ==
+ MLX5_ESEG_MIN_INLINE_SIZE,
+ "invalid min inline data size");
+static_assert(MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE <=
+ MLX5_DSEG_MAX, "invalid WQE max size");
+static_assert(MLX5_WQE_CSEG_SIZE == MLX5_WSEG_SIZE,
+ "invalid WQE Control Segment size");
+static_assert(MLX5_WQE_ESEG_SIZE == MLX5_WSEG_SIZE,
+ "invalid WQE Ethernet Segment size");
+static_assert(MLX5_WQE_DSEG_SIZE == MLX5_WSEG_SIZE,
+ "invalid WQE Data Segment size");
+static_assert(MLX5_WQE_SIZE == 4 * MLX5_WSEG_SIZE,
+ "invalid WQE size");
+
static __rte_always_inline uint32_t
-rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe);
+rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
+ volatile struct mlx5_mini_cqe8 *mcqe);
static __rte_always_inline int
mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
static __rte_always_inline void
rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
- volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res);
+ volatile struct mlx5_cqe *cqe,
+ volatile struct mlx5_mini_cqe8 *mcqe);
static int
mlx5_queue_state_modify(struct rte_eth_dev *dev,
static inline void
mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp,
volatile struct mlx5_cqe *__rte_restrict cqe,
- uint32_t phcsum);
+ uint32_t phcsum, uint8_t l4_type);
static inline void
mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
volatile struct mlx5_cqe *__rte_restrict cqe,
- uint32_t len);
+ volatile struct mlx5_mini_cqe8 *mcqe,
+ struct mlx5_rxq_data *rxq, uint32_t len);
uint32_t mlx5_ptype_table[] __rte_cache_aligned = {
[0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */
{
struct rxq_zip *zip = &rxq->zip;
volatile struct mlx5_cqe *cqe;
- unsigned int cq_ci = rxq->cq_ci;
const unsigned int cqe_n = (1 << rxq->cqe_n);
+ const unsigned int sges_n = (1 << rxq->sges_n);
+ const unsigned int elts_n = (1 << rxq->elts_n);
+ const unsigned int strd_n = (1 << rxq->strd_num_n);
const unsigned int cqe_cnt = cqe_n - 1;
- unsigned int used = 0;
+ unsigned int cq_ci, used;
+ /* if we are processing a compressed cqe */
+ if (zip->ai) {
+ used = zip->cqe_cnt - zip->ai;
+ cq_ci = zip->cq_ci;
+ } else {
+ used = 0;
+ cq_ci = rxq->cq_ci;
+ }
cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
while (check_cqe(cqe, cqe_n, cq_ci) != MLX5_CQE_STATUS_HW_OWN) {
int8_t op_own;
op_own = cqe->op_own;
if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED)
- if (unlikely(zip->ai))
- n = zip->cqe_cnt - zip->ai;
- else
- n = rte_be_to_cpu_32(cqe->byte_cnt);
+ n = rte_be_to_cpu_32(cqe->byte_cnt);
else
n = 1;
cq_ci += n;
used += n;
cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
}
- used = RTE_MIN(used, cqe_n);
+ used = RTE_MIN(used * sges_n, elts_n * strd_n);
return used;
}
if (!rxq)
return;
- qinfo->mp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ?
+ qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ?
rxq->mprq_mp : rxq->mp;
qinfo->conf.rx_thresh.pthresh = 0;
qinfo->conf.rx_thresh.hthresh = 0;
qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1;
qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
qinfo->scattered_rx = dev->data->scattered_rx;
- qinfo->nb_desc = 1 << rxq->elts_n;
+ qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ?
+ (1 << rxq->elts_n) * (1 << rxq->strd_num_n) :
+ (1 << rxq->elts_n);
}
/**
* Packet type for struct rte_mbuf.
*/
static inline uint32_t
-rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe)
+rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
+ volatile struct mlx5_mini_cqe8 *mcqe)
{
uint8_t idx;
- uint8_t pinfo = cqe->pkt_info;
- uint16_t ptype = cqe->hdr_type_etc;
+ uint8_t ptype;
+ uint8_t pinfo = (cqe->pkt_info & 0x3) << 6;
+ /* Get l3/l4 header from mini-CQE in case L3/L4 format*/
+ if (mcqe == NULL ||
+ rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
+ ptype = (cqe->hdr_type_etc & 0xfc00) >> 10;
+ else
+ ptype = mcqe->hdr_type >> 2;
/*
* The index to the array should have:
* bit[1:0] = l3_hdr_type
* bit[6] = tunneled
* bit[7] = outer_l3_type
*/
- idx = ((pinfo & 0x3) << 6) | ((ptype & 0xfc00) >> 10);
+ idx = pinfo | ptype;
return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6));
}
(volatile struct mlx5_mini_cqe8 (*)[8])
(uintptr_t)(&(*rxq->cqes)[zip->ca &
cqe_cnt].pkt_info);
-
- len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt);
+ len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt &
+ rxq->byte_mask);
*mcqe = &(*mc)[zip->ai & 7];
if ((++zip->ai & 7) == 0) {
/* Invalidate consumed CQEs */
} else {
int ret;
int8_t op_own;
+ uint32_t cq_ci;
ret = check_cqe(cqe, cqe_n, rxq->cq_ci);
if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
return 0;
}
}
- ++rxq->cq_ci;
+ /*
+ * Introduce the local variable to have queue cq_ci
+ * index in queue structure always consistent with
+ * actual CQE boundary (not pointing to the middle
+ * of compressed CQE session).
+ */
+ cq_ci = rxq->cq_ci + 1;
op_own = cqe->op_own;
if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) {
volatile struct mlx5_mini_cqe8 (*mc)[8] =
(volatile struct mlx5_mini_cqe8 (*)[8])
(uintptr_t)(&(*rxq->cqes)
- [rxq->cq_ci &
- cqe_cnt].pkt_info);
+ [cq_ci & cqe_cnt].pkt_info);
/* Fix endianness. */
zip->cqe_cnt = rte_be_to_cpu_32(cqe->byte_cnt);
* 7 CQEs after the initial CQE instead of 8
* for subsequent ones.
*/
- zip->ca = rxq->cq_ci;
+ zip->ca = cq_ci;
zip->na = zip->ca + 7;
/* Compute the next non compressed CQE. */
- --rxq->cq_ci;
zip->cq_ci = rxq->cq_ci + zip->cqe_cnt;
/* Get packet size to return. */
- len = rte_be_to_cpu_32((*mc)[0].byte_cnt);
+ len = rte_be_to_cpu_32((*mc)[0].byte_cnt &
+ rxq->byte_mask);
*mcqe = &(*mc)[0];
zip->ai = 1;
/* Prefetch all to be invalidated */
++idx;
}
} else {
+ rxq->cq_ci = cq_ci;
len = rte_be_to_cpu_32(cqe->byte_cnt);
}
}
*/
static inline void
rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
- volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res)
+ volatile struct mlx5_cqe *cqe,
+ volatile struct mlx5_mini_cqe8 *mcqe)
{
/* Update packet information. */
- pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe);
- if (rss_hash_res && rxq->rss_hash) {
- pkt->hash.rss = rss_hash_res;
- pkt->ol_flags |= PKT_RX_RSS_HASH;
+ pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe, mcqe);
+
+ if (rxq->rss_hash) {
+ uint32_t rss_hash_res = 0;
+
+ /* If compressed, take hash result from mini-CQE. */
+ if (mcqe == NULL ||
+ rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_HASH)
+ rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res);
+ else
+ rss_hash_res = rte_be_to_cpu_32(mcqe->rx_hash_result);
+ if (rss_hash_res) {
+ pkt->hash.rss = rss_hash_res;
+ pkt->ol_flags |= PKT_RX_RSS_HASH;
+ }
}
- if (rxq->mark && MLX5_FLOW_MARK_IS_VALID(cqe->sop_drop_qpn)) {
- pkt->ol_flags |= PKT_RX_FDIR;
- if (cqe->sop_drop_qpn !=
- rte_cpu_to_be_32(MLX5_FLOW_MARK_DEFAULT)) {
- uint32_t mark = cqe->sop_drop_qpn;
-
- pkt->ol_flags |= PKT_RX_FDIR_ID;
- pkt->hash.fdir.hi = mlx5_flow_mark_get(mark);
+ if (rxq->mark) {
+ uint32_t mark = 0;
+
+ /* If compressed, take flow tag from mini-CQE. */
+ if (mcqe == NULL ||
+ rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_FTAG_STRIDX)
+ mark = cqe->sop_drop_qpn;
+ else
+ mark = ((mcqe->byte_cnt_flow & 0xff) << 8) |
+ (mcqe->flow_tag_high << 16);
+ if (MLX5_FLOW_MARK_IS_VALID(mark)) {
+ pkt->ol_flags |= PKT_RX_FDIR;
+ if (mark != RTE_BE32(MLX5_FLOW_MARK_DEFAULT)) {
+ pkt->ol_flags |= PKT_RX_FDIR_ID;
+ pkt->hash.fdir.hi = mlx5_flow_mark_get(mark);
+ }
}
}
if (rxq->dynf_meta && cqe->flow_table_metadata) {
}
if (rxq->csum)
pkt->ol_flags |= rxq_cq_to_ol_flags(cqe);
- if (rxq->vlan_strip &&
- (cqe->hdr_type_etc & rte_cpu_to_be_16(MLX5_CQE_VLAN_STRIPPED))) {
- pkt->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
- pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info);
+ if (rxq->vlan_strip) {
+ bool vlan_strip;
+
+ if (mcqe == NULL ||
+ rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
+ vlan_strip = cqe->hdr_type_etc &
+ RTE_BE16(MLX5_CQE_VLAN_STRIPPED);
+ else
+ vlan_strip = mcqe->hdr_type &
+ RTE_BE16(MLX5_CQE_VLAN_STRIPPED);
+ if (vlan_strip) {
+ pkt->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
+ pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info);
+ }
}
if (rxq->hw_timestamp) {
uint64_t ts = rte_be_to_cpu_64(cqe->timestamp);
&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx];
struct rte_mbuf *rep = (*rxq->elts)[idx];
volatile struct mlx5_mini_cqe8 *mcqe = NULL;
- uint32_t rss_hash_res;
if (pkt)
NEXT(seg) = rep;
pkt = seg;
MLX5_ASSERT(len >= (rxq->crc_present << 2));
pkt->ol_flags &= EXT_ATTACHED_MBUF;
- /* If compressed, take hash result from mini-CQE. */
- rss_hash_res = rte_be_to_cpu_32(mcqe == NULL ?
- cqe->rx_hash_res :
- mcqe->rx_hash_result);
- rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res);
+ rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
if (rxq->crc_present)
len -= RTE_ETHER_CRC_LEN;
PKT_LEN(pkt) = len;
if (cqe->lro_num_seg > 1) {
mlx5_lro_update_hdr
(rte_pktmbuf_mtod(pkt, uint8_t *), cqe,
- len);
+ mcqe, rxq, len);
pkt->ol_flags |= PKT_RX_LRO;
pkt->tso_segsz = len / cqe->lro_num_seg;
}
static inline void
mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp,
volatile struct mlx5_cqe *__rte_restrict cqe,
- uint32_t phcsum)
+ uint32_t phcsum, uint8_t l4_type)
{
- uint8_t l4_type = (rte_be_to_cpu_16(cqe->hdr_type_etc) &
- MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT;
/*
* The HW calculates only the TCP payload checksum, need to complete
* the TCP header checksum and the L3 pseudo-header checksum.
static inline void
mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
volatile struct mlx5_cqe *__rte_restrict cqe,
- uint32_t len)
+ volatile struct mlx5_mini_cqe8 *mcqe,
+ struct mlx5_rxq_data *rxq, uint32_t len)
{
union {
struct rte_ether_hdr *eth;
};
uint16_t proto = h.eth->ether_type;
uint32_t phcsum;
+ uint8_t l4_type;
h.eth++;
while (proto == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
phcsum = rte_ipv6_phdr_cksum(h.ipv6, 0);
h.ipv6++;
}
- mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum);
+ if (mcqe == NULL ||
+ rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
+ l4_type = (rte_be_to_cpu_16(cqe->hdr_type_etc) &
+ MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT;
+ else
+ l4_type = (rte_be_to_cpu_16(mcqe->hdr_type) &
+ MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT;
+ mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum, l4_type);
}
void
{
struct mlx5_rxq_data *rxq = dpdk_rxq;
const uint32_t strd_n = 1 << rxq->strd_num_n;
+ const uint32_t strd_sz = 1 << rxq->strd_sz_n;
const uint32_t cq_mask = (1 << rxq->cqe_n) - 1;
const uint32_t wq_mask = (1 << rxq->elts_n) - 1;
volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
uint16_t strd_idx;
uint32_t byte_cnt;
volatile struct mlx5_mini_cqe8 *mcqe = NULL;
- uint32_t rss_hash_res = 0;
enum mlx5_rqx_code rxq_code;
if (consumed_strd == strd_n) {
if (!ret)
break;
byte_cnt = ret;
- strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >>
- MLX5_MPRQ_STRIDE_NUM_SHIFT;
+ len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT;
+ MLX5_ASSERT((int)len >= (rxq->crc_present << 2));
+ if (rxq->crc_present)
+ len -= RTE_ETHER_CRC_LEN;
+ if (mcqe &&
+ rxq->mcqe_format == MLX5_CQE_RESP_FORMAT_FTAG_STRIDX)
+ strd_cnt = (len / strd_sz) + !!(len % strd_sz);
+ else
+ strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >>
+ MLX5_MPRQ_STRIDE_NUM_SHIFT;
MLX5_ASSERT(strd_cnt);
consumed_strd += strd_cnt;
if (byte_cnt & MLX5_MPRQ_FILLER_MASK)
continue;
- if (mcqe == NULL) {
- rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res);
- strd_idx = rte_be_to_cpu_16(cqe->wqe_counter);
- } else {
- /* mini-CQE for MPRQ doesn't have hash result. */
- strd_idx = rte_be_to_cpu_16(mcqe->stride_idx);
- }
+ strd_idx = rte_be_to_cpu_16(mcqe == NULL ?
+ cqe->wqe_counter :
+ mcqe->stride_idx);
MLX5_ASSERT(strd_idx < strd_n);
MLX5_ASSERT(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci) &
wq_mask));
break;
}
}
- rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res);
+ rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
if (cqe->lro_num_seg > 1) {
mlx5_lro_update_hdr(rte_pktmbuf_mtod(pkt, uint8_t *),
- cqe, len);
+ cqe, mcqe, rxq, len);
pkt->ol_flags |= PKT_RX_LRO;
pkt->tso_segsz = len / cqe->lro_num_seg;
}
/**
* Free the mbufs from the linear array of pointers.
*
+ * @param txq
+ * Pointer to Tx queue structure.
* @param pkts
* Pointer to array of packets to be free.
* @param pkts_n
* compile time and may be used for optimization.
*/
static __rte_always_inline void
-mlx5_tx_free_mbuf(struct rte_mbuf **__rte_restrict pkts,
+mlx5_tx_free_mbuf(struct mlx5_txq_data *__rte_restrict txq,
+ struct rte_mbuf **__rte_restrict pkts,
unsigned int pkts_n,
unsigned int olx __rte_unused)
{
*/
MLX5_ASSERT(pkts);
MLX5_ASSERT(pkts_n);
+ /*
+ * Free mbufs directly to the pool in bulk
+ * if fast free offload is engaged
+ */
+ if (!MLX5_TXOFF_CONFIG(MULTI) && txq->fast_free) {
+ mbuf = *pkts;
+ pool = mbuf->pool;
+ rte_mempool_put_bulk(pool, (void *)pkts, pkts_n);
+ return;
+ }
for (;;) {
for (;;) {
/*
}
}
}
+/*
+ * No inline version to free buffers for optimal call
+ * on the tx_burst completion.
+ */
+static __rte_noinline void
+__mlx5_tx_free_mbuf(struct mlx5_txq_data *__rte_restrict txq,
+ struct rte_mbuf **__rte_restrict pkts,
+ unsigned int pkts_n,
+ unsigned int olx __rte_unused)
+{
+ mlx5_tx_free_mbuf(txq, pkts, pkts_n, olx);
+}
/**
* Free the mbuf from the elts ring buffer till new tail.
part = RTE_MIN(part, n_elts);
MLX5_ASSERT(part);
MLX5_ASSERT(part <= txq->elts_s);
- mlx5_tx_free_mbuf(&txq->elts[txq->elts_tail & txq->elts_m],
+ mlx5_tx_free_mbuf(txq,
+ &txq->elts[txq->elts_tail & txq->elts_m],
part, olx);
txq->elts_tail += part;
n_elts -= part;
bool ring_doorbell = false;
int ret;
- static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value");
- static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value");
do {
volatile struct mlx5_cqe *cqe;
}
/* Normal transmit completion. */
MLX5_ASSERT(txq->cq_ci != txq->cq_pi);
+#ifdef RTE_LIBRTE_MLX5_DEBUG
MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) ==
cqe->wqe_counter);
+#endif
ring_doorbell = true;
++txq->cq_ci;
last_cqe = cqe;
qs = RTE_PTR_ADD(wqe, MLX5_WSEG_SIZE);
qs->max_index = rte_cpu_to_be_32(wci);
- qs->qpn_cqn = rte_cpu_to_be_32(txq->sh->txpp.clock_queue.cq->id);
+ qs->qpn_cqn = rte_cpu_to_be_32(txq->sh->txpp.clock_queue.cq_obj.cq->id);
qs->reserved0 = RTE_BE32(0);
qs->reserved1 = RTE_BE32(0);
}
es->metadata = MLX5_TXOFF_CONFIG(METADATA) ?
loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ?
*RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0;
- static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
- (sizeof(uint16_t) +
- sizeof(rte_v128u32_t)),
- "invalid Ethernet Segment data size");
- static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
- (sizeof(uint16_t) +
- sizeof(struct rte_vlan_hdr) +
- 2 * RTE_ETHER_ADDR_LEN),
- "invalid Ethernet Segment data size");
psrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);
es->inline_hdr_sz = RTE_BE16(MLX5_ESEG_MIN_INLINE_SIZE);
es->inline_data = *(unaligned_uint16_t *)psrc;
es->metadata = MLX5_TXOFF_CONFIG(METADATA) ?
loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ?
*RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0;
- static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
- (sizeof(uint16_t) +
- sizeof(rte_v128u32_t)),
- "invalid Ethernet Segment data size");
- static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
- (sizeof(uint16_t) +
- sizeof(struct rte_vlan_hdr) +
- 2 * RTE_ETHER_ADDR_LEN),
- "invalid Ethernet Segment data size");
psrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);
es->inline_hdr_sz = rte_cpu_to_be_16(inlen);
es->inline_data = *(unaligned_uint16_t *)psrc;
es->metadata = MLX5_TXOFF_CONFIG(METADATA) ?
loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ?
*RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0;
- static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
- (sizeof(uint16_t) +
- sizeof(rte_v128u32_t)),
- "invalid Ethernet Segment data size");
- static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
- (sizeof(uint16_t) +
- sizeof(struct rte_vlan_hdr) +
- 2 * RTE_ETHER_ADDR_LEN),
- "invalid Ethernet Segment data size");
MLX5_ASSERT(inlen >= MLX5_ESEG_MIN_INLINE_SIZE);
pdst = (uint8_t *)&es->inline_data;
if (MLX5_TXOFF_CONFIG(VLAN) && vlan) {
uint8_t *pdst;
MLX5_ASSERT(len > MLX5_ESEG_MIN_INLINE_SIZE);
- static_assert(MLX5_DSEG_MIN_INLINE_SIZE ==
- (2 * RTE_ETHER_ADDR_LEN),
- "invalid Data Segment data size");
if (!MLX5_TXOFF_CONFIG(MPW)) {
/* Store the descriptor byte counter for eMPW sessions. */
dseg->bcount = rte_cpu_to_be_32
MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW));
MLX5_ASSERT(loc->elts_free && loc->wqe_free);
MLX5_ASSERT(pkts_n > loc->pkts_sent);
- static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
pkts += loc->pkts_sent + 1;
pkts_n -= loc->pkts_sent;
for (;;) {
MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW));
MLX5_ASSERT(loc->elts_free && loc->wqe_free);
MLX5_ASSERT(pkts_n > loc->pkts_sent);
- static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
pkts += loc->pkts_sent + 1;
pkts_n -= loc->pkts_sent;
for (;;) {
MLX5_ASSERT(room >= tlen);
room -= tlen;
/*
- * Packet data are completely inlined,
- * free the packet immediately.
+ * Packet data are completely inline,
+ * we can try to free the packet.
+ */
+ if (likely(loc->pkts_sent == loc->mbuf_free)) {
+ /*
+ * All the packets from the burst beginning
+ * are inline, we can free mbufs directly
+ * from the origin array on tx_burst exit().
+ */
+ loc->mbuf_free++;
+ goto next_mbuf;
+ }
+ /*
+ * In order no to call rte_pktmbuf_free_seg() here,
+ * in the most inner loop (that might be very
+ * expensive) we just save the mbuf in elts.
*/
- rte_pktmbuf_free_seg(loc->mbuf);
+ txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
+ loc->elts_free--;
goto next_mbuf;
pointer_empw:
/*
mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);
/* We have to store mbuf in elts.*/
txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
+ loc->elts_free--;
room -= MLX5_WQE_DSEG_SIZE;
/* Ring buffer wraparound is checked at the loop end.*/
++dseg;
slen += dlen;
#endif
loc->pkts_sent++;
- loc->elts_free--;
pkts_n--;
if (unlikely(!pkts_n || !loc->elts_free)) {
/*
loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {
vlan = sizeof(struct rte_vlan_hdr);
inlen += vlan;
- static_assert((sizeof(struct rte_vlan_hdr) +
- sizeof(struct rte_ether_hdr)) ==
- MLX5_ESEG_MIN_INLINE_SIZE,
- "invalid min inline data size");
}
/*
* If inlining is enabled at configuration time
MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));
if (unlikely(!pkts_n))
return 0;
+ if (MLX5_TXOFF_CONFIG(INLINE))
+ loc.mbuf_free = 0;
loc.pkts_sent = 0;
loc.pkts_copy = 0;
loc.wqe_last = NULL;
/* Increment sent packets counter. */
txq->stats.opackets += loc.pkts_sent;
#endif
+ if (MLX5_TXOFF_CONFIG(INLINE) && loc.mbuf_free)
+ __mlx5_tx_free_mbuf(txq, pkts, loc.mbuf_free, olx);
return loc.pkts_sent;
}
uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
unsigned int diff = 0, olx = 0, i, m;
- static_assert(MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE <=
- MLX5_DSEG_MAX, "invalid WQE max size");
- static_assert(MLX5_WQE_CSEG_SIZE == MLX5_WSEG_SIZE,
- "invalid WQE Control Segment size");
- static_assert(MLX5_WQE_ESEG_SIZE == MLX5_WSEG_SIZE,
- "invalid WQE Ethernet Segment size");
- static_assert(MLX5_WQE_DSEG_SIZE == MLX5_WSEG_SIZE,
- "invalid WQE Data Segment size");
- static_assert(MLX5_WQE_SIZE == 4 * MLX5_WSEG_SIZE,
- "invalid WQE size");
MLX5_ASSERT(priv);
if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
/* We should support Multi-Segment Packets. */
int
mlx5_tx_burst_mode_get(struct rte_eth_dev *dev,
- uint16_t tx_queue_id __rte_unused,
+ uint16_t tx_queue_id,
struct rte_eth_burst_mode *mode)
{
eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
unsigned int i, olx;
for (i = 0; i < RTE_DIM(txoff_func); i++) {
if (pkt_burst == txoff_func[i].func) {
olx = txoff_func[i].olx;
snprintf(mode->info, sizeof(mode->info),
- "%s%s%s%s%s%s%s%s%s",
+ "%s%s%s%s%s%s%s%s%s%s",
(olx & MLX5_TXOFF_CONFIG_EMPW) ?
((olx & MLX5_TXOFF_CONFIG_MPW) ?
"Legacy MPW" : "Enhanced MPW") : "No MPW",
(olx & MLX5_TXOFF_CONFIG_METADATA) ?
" + METADATA" : "",
(olx & MLX5_TXOFF_CONFIG_TXPP) ?
- " + TXPP" : "");
+ " + TXPP" : "",
+ (txq && txq->fast_free) ?
+ " + Fast Free" : "");
return 0;
}
}