From: Harman Kalra Date: Tue, 28 Apr 2020 12:40:12 +0000 (+0530) Subject: event/octeontx: support Rx/Tx checksum offload X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=cf55f04a0c99fc8571afbb7a1a3133af9d33c0e1;p=dpdk.git event/octeontx: support Rx/Tx checksum offload Adding support for rx checksum offload. In case of wrong checksum received (inner/outer l3/l4) it reports the corresponding layer which has bad checksum. It also adds rx burst function pointer hook for rx checksum offload to event PMD. Signed-off-by: Harman Kalra Acked-by: Jerin Jacob --- diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c index 5d074bcbc3..1b1a5d939f 100644 --- a/drivers/event/octeontx/ssovf_evdev.c +++ b/drivers/event/octeontx/ssovf_evdev.c @@ -272,6 +272,7 @@ ssovf_port_setup(struct rte_eventdev *dev, uint8_t port_id, reg_off |= 1 << 16; /* Wait */ ws->getwork = ws->base + reg_off; ws->port = port_id; + ws->lookup_mem = octeontx_fastpath_lookup_mem_get(); for (q = 0; q < edev->nb_event_queues; q++) { ws->grps[q] = ssovf_bar(OCTEONTX_SSO_GROUP, q, 2); diff --git a/drivers/event/octeontx/ssovf_evdev.h b/drivers/event/octeontx/ssovf_evdev.h index 1f5066c9ac..aa5acf246f 100644 --- a/drivers/event/octeontx/ssovf_evdev.h +++ b/drivers/event/octeontx/ssovf_evdev.h @@ -157,6 +157,7 @@ struct ssows { uint8_t *getwork; uint8_t *grps[SSO_MAX_VHGRP]; uint8_t port; + void *lookup_mem; } __rte_cache_aligned; static inline struct ssovf_evdev * @@ -182,5 +183,6 @@ int ssovf_info(struct ssovf_info *info); void *ssovf_bar(enum ssovf_type, uint8_t id, uint8_t bar); int test_eventdev_octeontx(void); void ssovf_fastpath_fns_set(struct rte_eventdev *dev); +void *octeontx_fastpath_lookup_mem_get(void); #endif /* __SSOVF_EVDEV_H__ */ diff --git a/drivers/event/octeontx/ssovf_worker.c b/drivers/event/octeontx/ssovf_worker.c index a276269d70..d2d5eea8f1 100644 --- a/drivers/event/octeontx/ssovf_worker.c +++ b/drivers/event/octeontx/ssovf_worker.c @@ -91,7 +91,7 @@ ssows_release_event(struct ssows *ws) ssows_swtag_untag(ws); } -#define R(name, f1, f0, flags) \ +#define R(name, f2, f1, f0, flags) \ static uint16_t __rte_noinline __rte_hot \ ssows_deq_ ##name(void *port, struct rte_event *ev, uint64_t timeout_ticks) \ { \ @@ -238,7 +238,8 @@ ssows_flush_events(struct ssows *ws, uint8_t queue_id, ev.mbuf = ssovf_octeontx_wqe_to_pkt(get_work1, (ev.event >> 20) & 0x7F, OCCTX_RX_OFFLOAD_NONE | - OCCTX_RX_MULTI_SEG_F); + OCCTX_RX_MULTI_SEG_F, + ws->lookup_mem); else ev.u64 = get_work1; @@ -340,16 +341,16 @@ SSO_TX_ADPTR_ENQ_FASTPATH_FUNC dev->txa_enqueue = ssow_txa_enqueue [!!(edev->tx_offload_flags & OCCTX_TX_OFFLOAD_MBUF_NOFF_F)] - [0] - [0] + [!!(edev->tx_offload_flags & OCCTX_TX_OFFLOAD_OL3_OL4_CSUM_F)] + [!!(edev->tx_offload_flags & OCCTX_TX_OFFLOAD_L3_L4_CSUM_F)] [!!(edev->tx_offload_flags & OCCTX_TX_MULTI_SEG_F)]; dev->txa_enqueue_same_dest = dev->txa_enqueue; /* Assigning dequeue func pointers */ - const event_dequeue_t ssow_deq[2][2] = { -#define R(name, f1, f0, flags) \ - [f1][f0] = ssows_deq_ ##name, + const event_dequeue_t ssow_deq[2][2][2] = { +#define R(name, f2, f1, f0, flags) \ + [f2][f1][f0] = ssows_deq_ ##name, SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R @@ -357,11 +358,12 @@ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC dev->dequeue = ssow_deq [!!(edev->rx_offload_flags & OCCTX_RX_VLAN_FLTR_F)] + [!!(edev->rx_offload_flags & OCCTX_RX_OFFLOAD_CSUM_F)] [!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)]; - const event_dequeue_burst_t ssow_deq_burst[2][2] = { -#define R(name, f1, f0, flags) \ - [f1][f0] = ssows_deq_burst_ ##name, + const event_dequeue_burst_t ssow_deq_burst[2][2][2] = { +#define R(name, f2, f1, f0, flags) \ + [f2][f1][f0] = ssows_deq_burst_ ##name, SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R @@ -369,12 +371,13 @@ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC dev->dequeue_burst = ssow_deq_burst [!!(edev->rx_offload_flags & OCCTX_RX_VLAN_FLTR_F)] + [!!(edev->rx_offload_flags & OCCTX_RX_OFFLOAD_CSUM_F)] [!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)]; if (edev->is_timeout_deq) { - const event_dequeue_t ssow_deq_timeout[2][2] = { -#define R(name, f1, f0, flags) \ - [f1][f0] = ssows_deq_timeout_ ##name, + const event_dequeue_t ssow_deq_timeout[2][2][2] = { +#define R(name, f2, f1, f0, flags) \ + [f2][f1][f0] = ssows_deq_timeout_ ##name, SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R @@ -382,11 +385,12 @@ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC dev->dequeue = ssow_deq_timeout [!!(edev->rx_offload_flags & OCCTX_RX_VLAN_FLTR_F)] + [!!(edev->rx_offload_flags & OCCTX_RX_OFFLOAD_CSUM_F)] [!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)]; - const event_dequeue_burst_t ssow_deq_timeout_burst[2][2] = { -#define R(name, f1, f0, flags) \ - [f1][f0] = ssows_deq_timeout_burst_ ##name, + const event_dequeue_burst_t ssow_deq_timeout_burst[2][2][2] = { +#define R(name, f2, f1, f0, flags) \ + [f2][f1][f0] = ssows_deq_timeout_burst_ ##name, SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R @@ -394,6 +398,97 @@ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC dev->dequeue_burst = ssow_deq_timeout_burst [!!(edev->rx_offload_flags & OCCTX_RX_VLAN_FLTR_F)] + [!!(edev->rx_offload_flags & OCCTX_RX_OFFLOAD_CSUM_F)] [!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)]; } } + +static void +octeontx_create_rx_ol_flags_array(void *mem) +{ + uint16_t idx, errcode, errlev; + uint32_t val, *ol_flags; + + /* Skip ptype array memory */ + ol_flags = (uint32_t *)mem; + + for (idx = 0; idx < BIT(ERRCODE_ERRLEN_WIDTH); idx++) { + errcode = idx & 0xff; + errlev = (idx & 0x700) >> 8; + + val = PKT_RX_IP_CKSUM_UNKNOWN; + val |= PKT_RX_L4_CKSUM_UNKNOWN; + val |= PKT_RX_OUTER_L4_CKSUM_UNKNOWN; + + switch (errlev) { + case OCCTX_ERRLEV_RE: + if (errcode) { + val |= PKT_RX_IP_CKSUM_BAD; + val |= PKT_RX_L4_CKSUM_BAD; + } else { + val |= PKT_RX_IP_CKSUM_GOOD; + val |= PKT_RX_L4_CKSUM_GOOD; + } + break; + case OCCTX_ERRLEV_LC: + if (errcode == OCCTX_EC_IP4_CSUM) { + val |= PKT_RX_IP_CKSUM_BAD; + val |= PKT_RX_EIP_CKSUM_BAD; + } else { + val |= PKT_RX_IP_CKSUM_GOOD; + } + break; + case OCCTX_ERRLEV_LD: + /* Check if parsed packet is neither IPv4 or IPV6 */ + if (errcode == OCCTX_EC_IP4_NOT) + break; + val |= PKT_RX_IP_CKSUM_GOOD; + if (errcode == OCCTX_EC_L4_CSUM) + val |= PKT_RX_OUTER_L4_CKSUM_BAD; + else + val |= PKT_RX_L4_CKSUM_GOOD; + break; + case OCCTX_ERRLEV_LE: + if (errcode == OCCTX_EC_IP4_CSUM) + val |= PKT_RX_IP_CKSUM_BAD; + else + val |= PKT_RX_IP_CKSUM_GOOD; + break; + case OCCTX_ERRLEV_LF: + /* Check if parsed packet is neither IPv4 or IPV6 */ + if (errcode == OCCTX_EC_IP4_NOT) + break; + val |= PKT_RX_IP_CKSUM_GOOD; + if (errcode == OCCTX_EC_L4_CSUM) + val |= PKT_RX_L4_CKSUM_BAD; + else + val |= PKT_RX_L4_CKSUM_GOOD; + break; + } + + ol_flags[idx] = val; + } +} + +void * +octeontx_fastpath_lookup_mem_get(void) +{ + const char name[] = OCCTX_FASTPATH_LOOKUP_MEM; + const struct rte_memzone *mz; + void *mem; + + mz = rte_memzone_lookup(name); + if (mz != NULL) + return mz->addr; + + /* Request for the first time */ + mz = rte_memzone_reserve_aligned(name, LOOKUP_ARRAY_SZ, + SOCKET_ID_ANY, 0, OCCTX_ALIGN); + if (mz != NULL) { + mem = mz->addr; + /* Form the rx ol_flags based on errcode */ + octeontx_create_rx_ol_flags_array(mem); + return mem; + } + return NULL; +} diff --git a/drivers/event/octeontx/ssovf_worker.h b/drivers/event/octeontx/ssovf_worker.h index 04c831514d..6b2fb9b817 100644 --- a/drivers/event/octeontx/ssovf_worker.h +++ b/drivers/event/octeontx/ssovf_worker.h @@ -10,6 +10,34 @@ #include "ssovf_evdev.h" #include "octeontx_rxtx.h" +/* Alignment */ +#define OCCTX_ALIGN 128 + +/* Fastpath lookup */ +#define OCCTX_FASTPATH_LOOKUP_MEM "octeontx_fastpath_lookup_mem" + +/* WQE's ERRCODE + ERRLEV (11 bits) */ +#define ERRCODE_ERRLEN_WIDTH 11 +#define ERR_ARRAY_SZ ((BIT(ERRCODE_ERRLEN_WIDTH)) *\ + sizeof(uint32_t)) + +#define LOOKUP_ARRAY_SZ (ERR_ARRAY_SZ) + +#define OCCTX_EC_IP4_NOT 0x41 +#define OCCTX_EC_IP4_CSUM 0x42 +#define OCCTX_EC_L4_CSUM 0x62 + +enum OCCTX_ERRLEV_E { + OCCTX_ERRLEV_RE = 0, + OCCTX_ERRLEV_LA = 1, + OCCTX_ERRLEV_LB = 2, + OCCTX_ERRLEV_LC = 3, + OCCTX_ERRLEV_LD = 4, + OCCTX_ERRLEV_LE = 5, + OCCTX_ERRLEV_LF = 6, + OCCTX_ERRLEV_LG = 7, +}; + enum { SSO_SYNC_ORDERED, SSO_SYNC_ATOMIC, @@ -19,6 +47,14 @@ enum { /* SSO Operations */ +static __rte_always_inline uint32_t +ssovf_octeontx_rx_olflags_get(const void * const lookup_mem, const uint64_t in) +{ + const uint32_t * const ol_flags = (const uint32_t *)lookup_mem; + + return ol_flags[(in & 0x7ff)]; +} + static __rte_always_inline void ssovf_octeontx_wqe_xtract_mseg(octtx_wqe_t *wqe, struct rte_mbuf *mbuf) @@ -55,7 +91,7 @@ ssovf_octeontx_wqe_xtract_mseg(octtx_wqe_t *wqe, static __rte_always_inline struct rte_mbuf * ssovf_octeontx_wqe_to_pkt(uint64_t work, uint16_t port_info, - const uint16_t flag) + const uint16_t flag, const void *lookup_mem) { struct rte_mbuf *mbuf; octtx_wqe_t *wqe = (octtx_wqe_t *)(uintptr_t)work; @@ -69,6 +105,10 @@ ssovf_octeontx_wqe_to_pkt(uint64_t work, uint16_t port_info, mbuf->ol_flags = 0; mbuf->pkt_len = wqe->s.w1.len; + if (!!(flag & OCCTX_RX_OFFLOAD_CSUM_F)) + mbuf->ol_flags = ssovf_octeontx_rx_olflags_get(lookup_mem, + wqe->w[2]); + if (!!(flag & OCCTX_RX_MULTI_SEG_F)) { mbuf->nb_segs = wqe->s.w0.bufs; mbuf->data_len = wqe->s.w5.size; @@ -134,7 +174,7 @@ ssows_get_work(struct ssows *ws, struct rte_event *ev, const uint16_t flag) if (get_work1 && ev->event_type == RTE_EVENT_TYPE_ETHDEV) { ev->mbuf = ssovf_octeontx_wqe_to_pkt(get_work1, - (ev->event >> 20) & 0x7F, flag); + (ev->event >> 20) & 0x7F, flag, ws->lookup_mem); } else if (unlikely((get_work0 & 0xFFFFFFFF) == 0xFFFFFFFF)) { ssovf_octeontx_wqe_free(get_work1); return 0; diff --git a/drivers/net/octeontx/octeontx_rxtx.h b/drivers/net/octeontx/octeontx_rxtx.h index 3f2147a028..8b46105b63 100644 --- a/drivers/net/octeontx/octeontx_rxtx.h +++ b/drivers/net/octeontx/octeontx_rxtx.h @@ -486,12 +486,19 @@ T(noff_ol3ol4csum_l3l4csum_mseg, 1, 1, 1, 1, 14, \ /* RX offload macros */ #define VLAN_FLTR_F OCCTX_RX_VLAN_FLTR_F +#define CSUM_F OCCTX_RX_OFFLOAD_CSUM_F #define MULT_RX_F OCCTX_RX_MULTI_SEG_F -/* [VLAN_FLTR][MULTI_SEG] */ + +/* [VLAN_FLTR] [CSUM_F] [MULTI_SEG] */ #define OCCTX_RX_FASTPATH_MODES \ -R(no_offload, 0, 0, OCCTX_RX_OFFLOAD_NONE) \ -R(mseg, 0, 1, MULT_RX_F) \ -R(vlan, 1, 0, VLAN_FLTR_F) \ -R(vlan_mseg, 1, 1, VLAN_FLTR_F | MULT_RX_F) +R(no_offload, 0, 0, 0, OCCTX_RX_OFFLOAD_NONE) \ +R(mseg, 0, 0, 1, MULT_RX_F) \ +R(csum, 0, 1, 0, CSUM_F) \ +R(csum_mseg, 0, 1, 1, CSUM_F | MULT_RX_F) \ +R(vlan, 1, 0, 0, VLAN_FLTR_F) \ +R(vlan_mseg, 1, 0, 1, VLAN_FLTR_F | MULT_RX_F) \ +R(vlan_csum, 1, 1, 0, VLAN_FLTR_F | CSUM_F) \ +R(vlan_csum_mseg, 1, 1, 1, CSUM_F | VLAN_FLTR_F | \ + MULT_RX_F) #endif /* __OCTEONTX_RXTX_H__ */