1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
5 #ifndef __OTX2_WORKER_H__
6 #define __OTX2_WORKER_H__
8 #include <rte_common.h>
9 #include <rte_branch_prediction.h>
11 #include <otx2_common.h>
12 #include "otx2_evdev.h"
16 static __rte_always_inline uint16_t
17 otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
18 const uint32_t flags, const void * const lookup_mem)
20 union otx2_sso_event event;
24 otx2_write64(BIT_ULL(16) | /* wait for work. */
25 1, /* Use Mask set 0. */
28 if (flags & NIX_RX_OFFLOAD_PTYPE_F)
29 rte_prefetch_non_temporal(lookup_mem);
32 " ldr %[tag], [%[tag_loc]] \n"
33 " ldr %[wqp], [%[wqp_loc]] \n"
34 " tbz %[tag], 63, done%= \n"
37 " ldr %[tag], [%[tag_loc]] \n"
38 " ldr %[wqp], [%[wqp_loc]] \n"
39 " tbnz %[tag], 63, rty%= \n"
41 " prfm pldl1keep, [%[wqp], #8] \n"
42 " sub %[mbuf], %[wqp], #0x80 \n"
43 " prfm pldl1keep, [%[mbuf]] \n"
44 : [tag] "=&r" (event.get_work0),
45 [wqp] "=&r" (get_work1),
47 : [tag_loc] "r" (ws->tag_op),
48 [wqp_loc] "r" (ws->wqp_op)
51 event.get_work0 = otx2_read64(ws->tag_op);
52 while ((BIT_ULL(63)) & event.get_work0)
53 event.get_work0 = otx2_read64(ws->tag_op);
55 get_work1 = otx2_read64(ws->wqp_op);
56 rte_prefetch0((const void *)get_work1);
57 mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
58 rte_prefetch0((const void *)mbuf);
61 event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
62 (event.get_work0 & (0x3FFull << 36)) << 4 |
63 (event.get_work0 & 0xffffffff);
64 ws->cur_tt = event.sched_type;
65 ws->cur_grp = event.queue_id;
67 if (event.sched_type != SSO_TT_EMPTY &&
68 event.event_type == RTE_EVENT_TYPE_ETHDEV) {
69 otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
70 (uint32_t) event.get_work0, flags, lookup_mem);
71 /* Extracting tstamp, if PTP enabled*/
72 otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
77 ev->event = event.get_work0;
83 /* Used in cleaning up workslot. */
84 static __rte_always_inline uint16_t
85 otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev,
88 union otx2_sso_event event;
94 " ldr %[tag], [%[tag_loc]] \n"
95 " ldr %[wqp], [%[wqp_loc]] \n"
96 " tbz %[tag], 63, done%= \n"
99 " ldr %[tag], [%[tag_loc]] \n"
100 " ldr %[wqp], [%[wqp_loc]] \n"
101 " tbnz %[tag], 63, rty%= \n"
103 " prfm pldl1keep, [%[wqp], #8] \n"
104 " sub %[mbuf], %[wqp], #0x80 \n"
105 " prfm pldl1keep, [%[mbuf]] \n"
106 : [tag] "=&r" (event.get_work0),
107 [wqp] "=&r" (get_work1),
109 : [tag_loc] "r" (ws->tag_op),
110 [wqp_loc] "r" (ws->wqp_op)
113 event.get_work0 = otx2_read64(ws->tag_op);
114 while ((BIT_ULL(63)) & event.get_work0)
115 event.get_work0 = otx2_read64(ws->tag_op);
117 get_work1 = otx2_read64(ws->wqp_op);
118 rte_prefetch_non_temporal((const void *)get_work1);
119 mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
120 rte_prefetch_non_temporal((const void *)mbuf);
123 event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
124 (event.get_work0 & (0x3FFull << 36)) << 4 |
125 (event.get_work0 & 0xffffffff);
126 ws->cur_tt = event.sched_type;
127 ws->cur_grp = event.queue_id;
129 if (event.sched_type != SSO_TT_EMPTY &&
130 event.event_type == RTE_EVENT_TYPE_ETHDEV) {
131 otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
132 (uint32_t) event.get_work0, flags, NULL);
133 /* Extracting tstamp, if PTP enabled*/
134 otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
139 ev->event = event.get_work0;
145 static __rte_always_inline void
146 otx2_ssogws_add_work(struct otx2_ssogws *ws, const uint64_t event_ptr,
147 const uint32_t tag, const uint8_t new_tt,
152 add_work0 = tag | ((uint64_t)(new_tt) << 32);
153 otx2_store_pair(add_work0, event_ptr, ws->grps_base[grp]);
156 static __rte_always_inline void
157 otx2_ssogws_swtag_desched(struct otx2_ssogws *ws, uint32_t tag, uint8_t new_tt,
162 val = tag | ((uint64_t)(new_tt & 0x3) << 32) | ((uint64_t)grp << 34);
163 otx2_write64(val, ws->swtag_desched_op);
166 static __rte_always_inline void
167 otx2_ssogws_swtag_norm(struct otx2_ssogws *ws, uint32_t tag, uint8_t new_tt)
171 val = tag | ((uint64_t)(new_tt & 0x3) << 32);
172 otx2_write64(val, ws->swtag_norm_op);
175 static __rte_always_inline void
176 otx2_ssogws_swtag_untag(struct otx2_ssogws *ws)
178 otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
179 SSOW_LF_GWS_OP_SWTAG_UNTAG);
180 ws->cur_tt = SSO_SYNC_UNTAGGED;
183 static __rte_always_inline void
184 otx2_ssogws_swtag_flush(struct otx2_ssogws *ws)
186 otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
187 SSOW_LF_GWS_OP_SWTAG_FLUSH);
188 ws->cur_tt = SSO_SYNC_EMPTY;
191 static __rte_always_inline void
192 otx2_ssogws_desched(struct otx2_ssogws *ws)
194 otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
195 SSOW_LF_GWS_OP_DESCHED);
198 static __rte_always_inline void
199 otx2_ssogws_swtag_wait(struct otx2_ssogws *ws)
201 #ifdef RTE_ARCH_ARM64
205 " ldr %[swtb], [%[swtp_loc]] \n"
206 " cbz %[swtb], done%= \n"
209 " ldr %[swtb], [%[swtp_loc]] \n"
210 " cbnz %[swtb], rty%= \n"
212 : [swtb] "=&r" (swtp)
213 : [swtp_loc] "r" (ws->swtp_op)
216 /* Wait for the SWTAG/SWTAG_FULL operation */
217 while (otx2_read64(ws->swtp_op))