- * Write a regular WQE.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param wqe
- * Pointer to the WQE to fill.
- * @param addr
- * Buffer data address.
- * @param length
- * Packet length.
- * @param lkey
- * Memory region lkey.
- */
-static inline void
-mlx5_wqe_write(struct txq *txq, volatile union mlx5_wqe *wqe,
- uintptr_t addr, uint32_t length, uint32_t lkey)
-{
- wqe->wqe.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND);
- wqe->wqe.ctrl.data[1] = htonl((txq->qp_num_8s) | 4);
- wqe->wqe.ctrl.data[2] = 0;
- wqe->wqe.ctrl.data[3] = 0;
- wqe->inl.eseg.rsvd0 = 0;
- wqe->inl.eseg.rsvd1 = 0;
- wqe->inl.eseg.mss = 0;
- wqe->inl.eseg.rsvd2 = 0;
- wqe->wqe.eseg.inline_hdr_sz = htons(MLX5_ETH_INLINE_HEADER_SIZE);
- /* Copy the first 16 bytes into inline header. */
- rte_memcpy((uint8_t *)(uintptr_t)wqe->wqe.eseg.inline_hdr_start,
- (uint8_t *)(uintptr_t)addr,
- MLX5_ETH_INLINE_HEADER_SIZE);
- addr += MLX5_ETH_INLINE_HEADER_SIZE;
- length -= MLX5_ETH_INLINE_HEADER_SIZE;
- /* Store remaining data in data segment. */
- wqe->wqe.dseg.byte_count = htonl(length);
- wqe->wqe.dseg.lkey = lkey;
- wqe->wqe.dseg.addr = htonll(addr);
- /* Increment consumer index. */
- ++txq->wqe_ci;
-}
-
-/**
- * Write a regular WQE with VLAN.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param wqe
- * Pointer to the WQE to fill.
- * @param addr
- * Buffer data address.
- * @param length
- * Packet length.
- * @param lkey
- * Memory region lkey.
- * @param vlan_tci
- * VLAN field to insert in packet.
- */
-static inline void
-mlx5_wqe_write_vlan(struct txq *txq, volatile union mlx5_wqe *wqe,
- uintptr_t addr, uint32_t length, uint32_t lkey,
- uint16_t vlan_tci)
-{
- uint32_t vlan = htonl(0x81000000 | vlan_tci);
-
- wqe->wqe.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND);
- wqe->wqe.ctrl.data[1] = htonl((txq->qp_num_8s) | 4);
- wqe->wqe.ctrl.data[2] = 0;
- wqe->wqe.ctrl.data[3] = 0;
- wqe->inl.eseg.rsvd0 = 0;
- wqe->inl.eseg.rsvd1 = 0;
- wqe->inl.eseg.mss = 0;
- wqe->inl.eseg.rsvd2 = 0;
- wqe->wqe.eseg.inline_hdr_sz = htons(MLX5_ETH_VLAN_INLINE_HEADER_SIZE);
- /*
- * Copy 12 bytes of source & destination MAC address.
- * Copy 4 bytes of VLAN.
- * Copy 2 bytes of Ether type.
- */
- rte_memcpy((uint8_t *)(uintptr_t)wqe->wqe.eseg.inline_hdr_start,
- (uint8_t *)(uintptr_t)addr, 12);
- rte_memcpy((uint8_t *)((uintptr_t)wqe->wqe.eseg.inline_hdr_start + 12),
- &vlan, sizeof(vlan));
- rte_memcpy((uint8_t *)((uintptr_t)wqe->wqe.eseg.inline_hdr_start + 16),
- (uint8_t *)((uintptr_t)addr + 12), 2);
- addr += MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan);
- length -= MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan);
- /* Store remaining data in data segment. */
- wqe->wqe.dseg.byte_count = htonl(length);
- wqe->wqe.dseg.lkey = lkey;
- wqe->wqe.dseg.addr = htonll(addr);
- /* Increment consumer index. */
- ++txq->wqe_ci;
-}
-
-/**
- * Write a inline WQE.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param wqe
- * Pointer to the WQE to fill.
- * @param addr
- * Buffer data address.
- * @param length
- * Packet length.
- * @param lkey
- * Memory region lkey.
- */
-static inline void
-mlx5_wqe_write_inline(struct txq *txq, volatile union mlx5_wqe *wqe,
- uintptr_t addr, uint32_t length)
-{
- uint32_t size;
- uint16_t wqe_cnt = txq->wqe_n - 1;
- uint16_t wqe_ci = txq->wqe_ci + 1;
-
- /* Copy the first 16 bytes into inline header. */
- rte_memcpy((void *)(uintptr_t)wqe->inl.eseg.inline_hdr_start,
- (void *)(uintptr_t)addr,
- MLX5_ETH_INLINE_HEADER_SIZE);
- addr += MLX5_ETH_INLINE_HEADER_SIZE;
- length -= MLX5_ETH_INLINE_HEADER_SIZE;
- size = 3 + ((4 + length + 15) / 16);
- wqe->inl.byte_cnt = htonl(length | MLX5_INLINE_SEG);
- rte_memcpy((void *)(uintptr_t)&wqe->inl.data[0],
- (void *)addr, MLX5_WQE64_INL_DATA);
- addr += MLX5_WQE64_INL_DATA;
- length -= MLX5_WQE64_INL_DATA;
- while (length) {
- volatile union mlx5_wqe *wqe_next =
- &(*txq->wqes)[wqe_ci & wqe_cnt];
- uint32_t copy_bytes = (length > sizeof(*wqe)) ?
- sizeof(*wqe) :
- length;
-
- rte_mov64((uint8_t *)(uintptr_t)&wqe_next->data[0],
- (uint8_t *)addr);
- addr += copy_bytes;
- length -= copy_bytes;
- ++wqe_ci;
- }
- assert(size < 64);
- wqe->inl.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND);
- wqe->inl.ctrl.data[1] = htonl(txq->qp_num_8s | size);
- wqe->inl.ctrl.data[2] = 0;
- wqe->inl.ctrl.data[3] = 0;
- wqe->inl.eseg.rsvd0 = 0;
- wqe->inl.eseg.rsvd1 = 0;
- wqe->inl.eseg.mss = 0;
- wqe->inl.eseg.rsvd2 = 0;
- wqe->inl.eseg.inline_hdr_sz = htons(MLX5_ETH_INLINE_HEADER_SIZE);
- /* Increment consumer index. */
- txq->wqe_ci = wqe_ci;
-}
-
-/**
- * Write a inline WQE with VLAN.