- struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
- uint16_t elts_head = txq->elts_head;
- const uint16_t elts_n = 1 << txq->elts_n;
- const uint16_t elts_m = elts_n - 1;
- unsigned int i = 0;
- unsigned int j = 0;
- unsigned int k = 0;
- uint16_t max_elts;
- uint16_t max_wqe;
- unsigned int comp;
- volatile struct mlx5_wqe_ctrl *last_wqe = NULL;
- unsigned int segs_n = 0;
- const unsigned int max_inline = txq->max_inline;
- uint64_t addr_64;
-
- if (unlikely(!pkts_n))
- return 0;
- /* Prefetch first packet cacheline. */
- rte_prefetch0(*pkts);
- /* Start processing. */
- mlx5_tx_complete(txq);
- max_elts = (elts_n - (elts_head - txq->elts_tail));
- max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
- if (unlikely(!max_wqe))
- return 0;
- do {
- struct rte_mbuf *buf = *pkts; /* First_seg. */
- uint8_t *raw;
- volatile struct mlx5_wqe_v *wqe = NULL;
- volatile rte_v128u32_t *dseg = NULL;
- uint32_t length;
- unsigned int ds = 0;
- unsigned int sg = 0; /* counter of additional segs attached. */
- uintptr_t addr;
- uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE + 2;
- uint16_t tso_header_sz = 0;
- uint16_t ehdr;
- uint8_t cs_flags;
- uint8_t tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG);
- uint32_t swp_offsets = 0;
- uint8_t swp_types = 0;
- rte_be32_t metadata;
- uint16_t tso_segsz = 0;
-#ifdef MLX5_PMD_SOFT_COUNTERS
- uint32_t total_length = 0;
-#endif
- int ret;
-
- segs_n = buf->nb_segs;
- /*
- * Make sure there is enough room to store this packet and
- * that one ring entry remains unused.
- */
- assert(segs_n);
- if (max_elts < segs_n)
- break;
- max_elts -= segs_n;
- sg = --segs_n;
- if (unlikely(--max_wqe == 0))
- break;
- wqe = (volatile struct mlx5_wqe_v *)
- tx_mlx5_wqe(txq, txq->wqe_ci);
- rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
- if (pkts_n - i > 1)
- rte_prefetch0(*(pkts + 1));
- addr = rte_pktmbuf_mtod(buf, uintptr_t);
- length = DATA_LEN(buf);
- ehdr = (((uint8_t *)addr)[1] << 8) |
- ((uint8_t *)addr)[0];
-#ifdef MLX5_PMD_SOFT_COUNTERS
- total_length = length;
-#endif
- if (length < (MLX5_WQE_DWORD_SIZE + 2)) {
- txq->stats.oerrors++;
- break;
- }
- /* Update element. */
- (*txq->elts)[elts_head & elts_m] = buf;
- /* Prefetch next buffer data. */
- if (pkts_n - i > 1)
- rte_prefetch0(
- rte_pktmbuf_mtod(*(pkts + 1), volatile void *));
- cs_flags = txq_ol_cksum_to_cs(buf);
- txq_mbuf_to_swp(txq, buf, (uint8_t *)&swp_offsets, &swp_types);
- raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE;
- /* Copy metadata from mbuf if valid */
- metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
- 0;
- /* Replace the Ethernet type by the VLAN if necessary. */
- if (buf->ol_flags & PKT_TX_VLAN_PKT) {
- uint32_t vlan = rte_cpu_to_be_32(0x81000000 |
- buf->vlan_tci);
- unsigned int len = 2 * RTE_ETHER_ADDR_LEN - 2;
-
- addr += 2;
- length -= 2;
- /* Copy Destination and source mac address. */
- memcpy((uint8_t *)raw, ((uint8_t *)addr), len);
- /* Copy VLAN. */
- memcpy((uint8_t *)raw + len, &vlan, sizeof(vlan));
- /* Copy missing two bytes to end the DSeg. */
- memcpy((uint8_t *)raw + len + sizeof(vlan),
- ((uint8_t *)addr) + len, 2);
- addr += len + 2;
- length -= (len + 2);
- } else {
- memcpy((uint8_t *)raw, ((uint8_t *)addr) + 2,
- MLX5_WQE_DWORD_SIZE);
- length -= pkt_inline_sz;
- addr += pkt_inline_sz;
- }
- raw += MLX5_WQE_DWORD_SIZE;
- if (tso) {
- ret = inline_tso(txq, buf, &length,
- &addr, &pkt_inline_sz,
- &raw, &max_wqe,
- &tso_segsz, &tso_header_sz);
- if (ret == -EINVAL) {
- break;
- } else if (ret == -EAGAIN) {
- /* NOP WQE. */
- wqe->ctrl = (rte_v128u32_t){
- rte_cpu_to_be_32(txq->wqe_ci << 8),
- rte_cpu_to_be_32(txq->qp_num_8s | 1),
- 0,
- 0,
- };
- ds = 1;
-#ifdef MLX5_PMD_SOFT_COUNTERS
- total_length = 0;
-#endif
- k++;
- goto next_wqe;
- }
- }
- /* Inline if enough room. */
- if (max_inline || tso) {
- uint32_t inl = 0;
- uintptr_t end = (uintptr_t)
- (((uintptr_t)txq->wqes) +
- (1 << txq->wqe_n) * MLX5_WQE_SIZE);
- unsigned int inline_room = max_inline *
- RTE_CACHE_LINE_SIZE -
- (pkt_inline_sz - 2) -
- !!tso * sizeof(inl);
- uintptr_t addr_end;
- unsigned int copy_b;
-
-pkt_inline:
- addr_end = RTE_ALIGN_FLOOR(addr + inline_room,
- RTE_CACHE_LINE_SIZE);
- copy_b = (addr_end > addr) ?
- RTE_MIN((addr_end - addr), length) : 0;
- if (copy_b && ((end - (uintptr_t)raw) >
- (copy_b + sizeof(inl)))) {
- /*
- * One Dseg remains in the current WQE. To
- * keep the computation positive, it is
- * removed after the bytes to Dseg conversion.
- */
- uint16_t n = (MLX5_WQE_DS(copy_b) - 1 + 3) / 4;
-
- if (unlikely(max_wqe < n))
- break;
- max_wqe -= n;
- if (tso) {
- assert(inl == 0);
- inl = rte_cpu_to_be_32(copy_b |
- MLX5_INLINE_SEG);
- rte_memcpy((void *)raw,
- (void *)&inl, sizeof(inl));
- raw += sizeof(inl);
- pkt_inline_sz += sizeof(inl);
- }
- rte_memcpy((void *)raw, (void *)addr, copy_b);
- addr += copy_b;
- length -= copy_b;
- pkt_inline_sz += copy_b;
- }
- /*
- * 2 DWORDs consumed by the WQE header + ETH segment +
- * the size of the inline part of the packet.
- */
- ds = 2 + MLX5_WQE_DS(pkt_inline_sz - 2);
- if (length > 0) {
- if (ds % (MLX5_WQE_SIZE /
- MLX5_WQE_DWORD_SIZE) == 0) {
- if (unlikely(--max_wqe == 0))
- break;
- dseg = (volatile rte_v128u32_t *)
- tx_mlx5_wqe(txq, txq->wqe_ci +
- ds / 4);
- } else {
- dseg = (volatile rte_v128u32_t *)
- ((uintptr_t)wqe +
- (ds * MLX5_WQE_DWORD_SIZE));
- }
- goto use_dseg;
- } else if (!segs_n) {
- goto next_pkt;
- } else {
- /*
- * Further inline the next segment only for
- * non-TSO packets.
- */
- if (!tso) {
- raw += copy_b;
- inline_room -= copy_b;
- } else {
- inline_room = 0;
- }
- /* Move to the next segment. */
- --segs_n;
- buf = buf->next;
- assert(buf);
- addr = rte_pktmbuf_mtod(buf, uintptr_t);
- length = DATA_LEN(buf);
-#ifdef MLX5_PMD_SOFT_COUNTERS
- total_length += length;
-#endif
- (*txq->elts)[++elts_head & elts_m] = buf;
- goto pkt_inline;
- }
- } else {
- /*
- * No inline has been done in the packet, only the
- * Ethernet Header as been stored.
- */
- dseg = (volatile rte_v128u32_t *)
- ((uintptr_t)wqe + (3 * MLX5_WQE_DWORD_SIZE));
- ds = 3;
-use_dseg:
- /* Add the remaining packet as a simple ds. */
- addr_64 = rte_cpu_to_be_64(addr);
- *dseg = (rte_v128u32_t){
- rte_cpu_to_be_32(length),
- mlx5_tx_mb2mr(txq, buf),
- addr_64,
- addr_64 >> 32,
- };
- ++ds;
- if (!segs_n)
- goto next_pkt;
- }
-next_seg:
- assert(buf);
- assert(ds);
- assert(wqe);
- /*
- * Spill on next WQE when the current one does not have
- * enough room left. Size of WQE must a be a multiple
- * of data segment size.
- */
- assert(!(MLX5_WQE_SIZE % MLX5_WQE_DWORD_SIZE));
- if (!(ds % (MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE))) {
- if (unlikely(--max_wqe == 0))
- break;
- dseg = (volatile rte_v128u32_t *)
- tx_mlx5_wqe(txq, txq->wqe_ci + ds / 4);
- rte_prefetch0(tx_mlx5_wqe(txq,
- txq->wqe_ci + ds / 4 + 1));
- } else {
- ++dseg;
- }
- ++ds;
- buf = buf->next;
- assert(buf);
- length = DATA_LEN(buf);
-#ifdef MLX5_PMD_SOFT_COUNTERS
- total_length += length;
-#endif
- /* Store segment information. */
- addr_64 = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t));
- *dseg = (rte_v128u32_t){
- rte_cpu_to_be_32(length),
- mlx5_tx_mb2mr(txq, buf),
- addr_64,
- addr_64 >> 32,
- };
- (*txq->elts)[++elts_head & elts_m] = buf;
- if (--segs_n)
- goto next_seg;
-next_pkt:
- if (ds > MLX5_DSEG_MAX) {
- txq->stats.oerrors++;
- break;
- }
- ++elts_head;
- ++pkts;
- ++i;
- j += sg;
- /* Initialize known and common part of the WQE structure. */
- if (tso) {
- wqe->ctrl = (rte_v128u32_t){
- rte_cpu_to_be_32((txq->wqe_ci << 8) |
- MLX5_OPCODE_TSO),
- rte_cpu_to_be_32(txq->qp_num_8s | ds),
- 0,
- 0,
- };
- wqe->eseg = (rte_v128u32_t){
- swp_offsets,
- cs_flags | (swp_types << 8) |
- (rte_cpu_to_be_16(tso_segsz) << 16),
- metadata,
- (ehdr << 16) | rte_cpu_to_be_16(tso_header_sz),
- };
- } else {
- wqe->ctrl = (rte_v128u32_t){
- rte_cpu_to_be_32((txq->wqe_ci << 8) |
- MLX5_OPCODE_SEND),
- rte_cpu_to_be_32(txq->qp_num_8s | ds),
- 0,
- 0,
- };
- wqe->eseg = (rte_v128u32_t){
- swp_offsets,
- cs_flags | (swp_types << 8),
- metadata,
- (ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz),
- };