rxq->rx_tail = 0;
rxq->nb_rx_hold = 0;
- if (rxq->pkt_first_seg != NULL)
- rte_pktmbuf_free(rxq->pkt_first_seg);
+ rte_pktmbuf_free(rxq->pkt_first_seg);
rxq->pkt_first_seg = NULL;
rxq->pkt_last_seg = NULL;
#endif
}
+static const
+iavf_rxd_to_pkt_fields_t rxd_to_pkt_fields_ops[] = {
+ [IAVF_RXDID_COMMS_AUX_VLAN] = iavf_rxd_to_pkt_fields_by_comms_aux_v1,
+ [IAVF_RXDID_COMMS_AUX_IPV4] = iavf_rxd_to_pkt_fields_by_comms_aux_v1,
+ [IAVF_RXDID_COMMS_AUX_IPV6] = iavf_rxd_to_pkt_fields_by_comms_aux_v1,
+ [IAVF_RXDID_COMMS_AUX_IPV6_FLOW] =
+ iavf_rxd_to_pkt_fields_by_comms_aux_v1,
+ [IAVF_RXDID_COMMS_AUX_TCP] = iavf_rxd_to_pkt_fields_by_comms_aux_v1,
+ [IAVF_RXDID_COMMS_AUX_IP_OFFSET] =
+ iavf_rxd_to_pkt_fields_by_comms_aux_v2,
+ [IAVF_RXDID_COMMS_IPSEC_CRYPTO] =
+ iavf_rxd_to_pkt_fields_by_comms_aux_v2,
+ [IAVF_RXDID_COMMS_OVS_1] = iavf_rxd_to_pkt_fields_by_comms_ovs,
+};
+
static void
iavf_select_rxd_to_pkt_fields_handler(struct iavf_rx_queue *rxq, uint32_t rxdid)
{
+ rxq->rxdid = rxdid;
+
switch (rxdid) {
case IAVF_RXDID_COMMS_AUX_VLAN:
rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_vlan_mask;
- rxq->rxd_to_pkt_fields =
- iavf_rxd_to_pkt_fields_by_comms_aux_v1;
break;
case IAVF_RXDID_COMMS_AUX_IPV4:
rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask;
- rxq->rxd_to_pkt_fields =
- iavf_rxd_to_pkt_fields_by_comms_aux_v1;
break;
case IAVF_RXDID_COMMS_AUX_IPV6:
rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask;
- rxq->rxd_to_pkt_fields =
- iavf_rxd_to_pkt_fields_by_comms_aux_v1;
break;
case IAVF_RXDID_COMMS_AUX_IPV6_FLOW:
rxq->xtr_ol_flag =
rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask;
- rxq->rxd_to_pkt_fields =
- iavf_rxd_to_pkt_fields_by_comms_aux_v1;
break;
case IAVF_RXDID_COMMS_AUX_TCP:
rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_tcp_mask;
- rxq->rxd_to_pkt_fields =
- iavf_rxd_to_pkt_fields_by_comms_aux_v1;
break;
case IAVF_RXDID_COMMS_AUX_IP_OFFSET:
rxq->xtr_ol_flag =
rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask;
- rxq->rxd_to_pkt_fields =
- iavf_rxd_to_pkt_fields_by_comms_aux_v2;
break;
case IAVF_RXDID_COMMS_IPSEC_CRYPTO:
rxq->xtr_ol_flag =
rte_pmd_ifd_dynflag_proto_xtr_ipsec_crypto_said_mask;
- rxq->rxd_to_pkt_fields =
- iavf_rxd_to_pkt_fields_by_comms_aux_v2;
break;
case IAVF_RXDID_COMMS_OVS_1:
- rxq->rxd_to_pkt_fields = iavf_rxd_to_pkt_fields_by_comms_ovs;
break;
default:
/* update this according to the RXDID for FLEX_DESC_NONE */
- rxq->rxd_to_pkt_fields = iavf_rxd_to_pkt_fields_by_comms_ovs;
+ rxq->rxdid = IAVF_RXDID_COMMS_OVS_1;
break;
}
return -ENOMEM;
}
- /* Allocate the maximun number of RX ring hardware descriptor with
- * a liitle more to support bulk allocate.
+ /* Allocate the maximum number of RX ring hardware descriptor with
+ * a little more to support bulk allocate.
*/
len = IAVF_MAX_RING_DESC + IAVF_RX_MAX_BURST;
ring_size = RTE_ALIGN(len * sizeof(union iavf_rx_desc),
iavf_flex_rxd_to_vlan_tci(rxm, &rxd);
iavf_flex_rxd_to_ipsec_crypto_status(rxm, &rxd,
&rxq->stats.ipsec_crypto);
- rxq->rxd_to_pkt_fields(rxq, rxm, &rxd);
+ rxd_to_pkt_fields_ops[rxq->rxdid](rxq, rxm, &rxd);
pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0);
rxm->ol_flags |= pkt_flags;
iavf_flex_rxd_to_vlan_tci(first_seg, &rxd);
iavf_flex_rxd_to_ipsec_crypto_status(first_seg, &rxd,
&rxq->stats.ipsec_crypto);
- rxq->rxd_to_pkt_fields(rxq, first_seg, &rxd);
+ rxd_to_pkt_fields_ops[rxq->rxdid](rxq, first_seg, &rxd);
pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0);
first_seg->ol_flags |= pkt_flags;
struct rte_mbuf *mb;
uint16_t stat_err0;
uint16_t pkt_len;
- int32_t s[IAVF_LOOK_AHEAD], nb_dd;
+ int32_t s[IAVF_LOOK_AHEAD], var, nb_dd;
int32_t i, j, nb_rx = 0;
uint64_t pkt_flags;
const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
rte_smp_rmb();
- /* Compute how many status bits were set */
- for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++)
- nb_dd += s[j] & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S);
+ /* Compute how many contiguous DD bits were set */
+ for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) {
+ var = s[j] & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S);
+#ifdef RTE_ARCH_ARM
+ /* For Arm platforms, count only contiguous descriptors
+ * whose DD bit is set to 1. On Arm platforms, reads of
+ * descriptors can be reordered. Since the CPU may
+ * be reading the descriptors as the NIC updates them
+ * in memory, it is possbile that the DD bit for a
+ * descriptor earlier in the queue is read as not set
+ * while the DD bit for a descriptor later in the queue
+ * is read as set.
+ */
+ if (var)
+ nb_dd += 1;
+ else
+ break;
+#else
+ nb_dd += var;
+#endif
+ }
nb_rx += nb_dd;
iavf_flex_rxd_to_vlan_tci(mb, &rxdp[j]);
iavf_flex_rxd_to_ipsec_crypto_status(mb, &rxdp[j],
&rxq->stats.ipsec_crypto);
- rxq->rxd_to_pkt_fields(rxq, mb, &rxdp[j]);
+ rxd_to_pkt_fields_ops[rxq->rxdid](rxq, mb, &rxdp[j]);
stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
pkt_flags = iavf_flex_rxd_error_to_pkt_flags(stat_err0);
uint16_t pkt_len;
uint64_t qword1;
uint32_t rx_status;
- int32_t s[IAVF_LOOK_AHEAD], nb_dd;
+ int32_t s[IAVF_LOOK_AHEAD], var, nb_dd;
int32_t i, j, nb_rx = 0;
uint64_t pkt_flags;
const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
rte_smp_rmb();
- /* Compute how many status bits were set */
- for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++)
- nb_dd += s[j] & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT);
+ /* Compute how many contiguous DD bits were set */
+ for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) {
+ var = s[j] & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT);
+#ifdef RTE_ARCH_ARM
+ /* For Arm platforms, count only contiguous descriptors
+ * whose DD bit is set to 1. On Arm platforms, reads of
+ * descriptors can be reordered. Since the CPU may
+ * be reading the descriptors as the NIC updates them
+ * in memory, it is possbile that the DD bit for a
+ * descriptor earlier in the queue is read as not set
+ * while the DD bit for a descriptor later in the queue
+ * is read as set.
+ */
+ if (var)
+ nb_dd += 1;
+ else
+ break;
+#else
+ nb_dd += var;
+#endif
+ }
nb_rx += nb_dd;
return 0;
}
-
+/* Check if the context descriptor is needed for TX offloading */
+static inline uint16_t
+iavf_calc_context_desc(uint64_t flags, uint8_t vlan_flag)
+{
+ if (flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG |
+ RTE_MBUF_F_TX_TUNNEL_MASK))
+ return 1;
+ if (flags & RTE_MBUF_F_TX_VLAN &&
+ vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2)
+ return 1;
+ return 0;
+}
static inline void
-iavf_fill_ctx_desc_cmd_field(volatile uint64_t *field, struct rte_mbuf *m)
+iavf_fill_ctx_desc_cmd_field(volatile uint64_t *field, struct rte_mbuf *m,
+ uint8_t vlan_flag)
{
uint64_t cmd = 0;
/* TSO enabled */
if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))
- cmd = IAVF_TX_CTX_DESC_TSO << IAVF_TXD_DATA_QW1_CMD_SHIFT;
-
- /* Time Sync - Currently not supported */
+ cmd = IAVF_TX_CTX_DESC_TSO << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- /* Outer L2 TAG 2 Insertion - Currently not supported */
- /* Inner L2 TAG 2 Insertion - Currently not supported */
+ if (m->ol_flags & RTE_MBUF_F_TX_VLAN &&
+ vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2) {
+ cmd |= IAVF_TX_CTX_DESC_IL2TAG2
+ << IAVF_TXD_CTX_QW1_CMD_SHIFT;
+ }
*field |= cmd;
}
static inline void
iavf_fill_context_desc(volatile struct iavf_tx_context_desc *desc,
struct rte_mbuf *m, struct iavf_ipsec_crypto_pkt_metadata *ipsec_md,
- uint16_t *tlen)
+ uint16_t *tlen, uint8_t vlan_flag)
{
volatile struct iavf_tx_context_desc_qws *desc_qws =
(volatile struct iavf_tx_context_desc_qws *)desc;
desc_qws->qw1 = IAVF_TX_DESC_DTYPE_CONTEXT;
/* fill command field */
- iavf_fill_ctx_desc_cmd_field(&desc_qws->qw1, m);
+ iavf_fill_ctx_desc_cmd_field(&desc_qws->qw1, m, vlan_flag);
/* fill segmentation field */
if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) {
desc_qws->qw0 = rte_cpu_to_le_64(desc_qws->qw0);
desc_qws->qw1 = rte_cpu_to_le_64(desc_qws->qw1);
+
+ if (vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2)
+ desc->l2tag2 = m->vlan_tci;
}
static inline void
iavf_build_data_desc_cmd_offset_fields(volatile uint64_t *qw1,
- struct rte_mbuf *m)
+ struct rte_mbuf *m, uint8_t vlan_flag)
{
uint64_t command = 0;
uint64_t offset = 0;
command = (uint64_t)IAVF_TX_DESC_CMD_ICRC;
/* Descriptor based VLAN insertion */
- if (m->ol_flags & RTE_MBUF_F_TX_VLAN) {
+ if ((vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1) &&
+ m->ol_flags & RTE_MBUF_F_TX_VLAN) {
command |= (uint64_t)IAVF_TX_DESC_CMD_IL2TAG1;
l2tag1 |= m->vlan_tci;
}
if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)
hdrlen += ipseclen;
bufsz = hdrlen + tlen;
+ } else if ((m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) &&
+ (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG |
+ RTE_MBUF_F_TX_UDP_SEG))) {
+ hdrlen += m->outer_l3_len + m->l3_len + ipseclen;
+ if (m->ol_flags & RTE_MBUF_F_TX_L4_MASK)
+ hdrlen += m->l4_len;
+ bufsz = hdrlen + tlen;
+
} else {
bufsz = m->data_len;
}
ipsec_md = iavf_ipsec_crypto_get_pkt_metadata(txq, mb);
nb_desc_data = mb->nb_segs;
- nb_desc_ctx = !!(mb->ol_flags &
- (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG |
- RTE_MBUF_F_TX_TUNNEL_MASK));
+ nb_desc_ctx =
+ iavf_calc_context_desc(mb->ol_flags, txq->vlan_flag);
nb_desc_ipsec = !!(mb->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD);
/**
}
}
- iavf_build_data_desc_cmd_offset_fields(&ddesc_template, mb);
+ iavf_build_data_desc_cmd_offset_fields(&ddesc_template, mb,
+ txq->vlan_flag);
/* Setup TX context descriptor if required */
if (nb_desc_ctx) {
txe->mbuf = NULL;
}
- iavf_fill_context_desc(ctx_desc, mb, ipsec_md, &tlen);
+ iavf_fill_context_desc(ctx_desc, mb, ipsec_md, &tlen,
+ txq->vlan_flag);
IAVF_DUMP_TX_DESC(txq, ctx_desc, desc_idx);
txe->last_id = desc_idx_last;