1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
5 #include "otx2_worker_dual.h"
6 #include "otx2_worker.h"
8 static __rte_noinline uint8_t
9 otx2_ssogws_dual_new_event(struct otx2_ssogws_dual *ws,
10 const struct rte_event *ev)
12 const uint32_t tag = (uint32_t)ev->event;
13 const uint8_t new_tt = ev->sched_type;
14 const uint64_t event_ptr = ev->u64;
15 const uint16_t grp = ev->queue_id;
17 if (ws->xaq_lmt <= *ws->fc_mem)
20 otx2_ssogws_dual_add_work(ws, event_ptr, tag, new_tt, grp);
25 static __rte_always_inline void
26 otx2_ssogws_dual_fwd_swtag(struct otx2_ssogws_state *ws,
27 const struct rte_event *ev)
29 const uint32_t tag = (uint32_t)ev->event;
30 const uint8_t new_tt = ev->sched_type;
31 const uint8_t cur_tt = ws->cur_tt;
34 * cur_tt/new_tt SSO_SYNC_ORDERED SSO_SYNC_ATOMIC SSO_SYNC_UNTAGGED
36 * SSO_SYNC_ORDERED norm norm untag
37 * SSO_SYNC_ATOMIC norm norm untag
38 * SSO_SYNC_UNTAGGED norm norm NOOP
40 if (new_tt == SSO_SYNC_UNTAGGED) {
41 if (cur_tt != SSO_SYNC_UNTAGGED)
42 otx2_ssogws_swtag_untag((struct otx2_ssogws *)ws);
44 otx2_ssogws_swtag_norm((struct otx2_ssogws *)ws, tag, new_tt);
48 static __rte_always_inline void
49 otx2_ssogws_dual_fwd_group(struct otx2_ssogws_state *ws,
50 const struct rte_event *ev, const uint16_t grp)
52 const uint32_t tag = (uint32_t)ev->event;
53 const uint8_t new_tt = ev->sched_type;
55 otx2_write64(ev->u64, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
56 SSOW_LF_GWS_OP_UPD_WQP_GRP1);
58 otx2_ssogws_swtag_desched((struct otx2_ssogws *)ws, tag, new_tt, grp);
61 static __rte_always_inline void
62 otx2_ssogws_dual_forward_event(struct otx2_ssogws_dual *ws,
63 struct otx2_ssogws_state *vws,
64 const struct rte_event *ev)
66 const uint8_t grp = ev->queue_id;
68 /* Group hasn't changed, Use SWTAG to forward the event */
69 if (vws->cur_grp == grp) {
70 otx2_ssogws_dual_fwd_swtag(vws, ev);
74 * Group has been changed for group based work pipelining,
75 * Use deschedule/add_work operation to transfer the event to
78 otx2_ssogws_dual_fwd_group(vws, ev, grp);
83 otx2_ssogws_dual_enq(void *port, const struct rte_event *ev)
85 struct otx2_ssogws_dual *ws = port;
86 struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
89 case RTE_EVENT_OP_NEW:
91 return otx2_ssogws_dual_new_event(ws, ev);
92 case RTE_EVENT_OP_FORWARD:
93 otx2_ssogws_dual_forward_event(ws, vws, ev);
95 case RTE_EVENT_OP_RELEASE:
96 otx2_ssogws_swtag_flush((struct otx2_ssogws *)vws);
106 otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[],
109 RTE_SET_USED(nb_events);
110 return otx2_ssogws_dual_enq(port, ev);
114 otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[],
117 struct otx2_ssogws_dual *ws = port;
121 if (ws->xaq_lmt <= *ws->fc_mem)
124 for (i = 0; i < nb_events && rc; i++)
125 rc = otx2_ssogws_dual_new_event(ws, &ev[i]);
131 otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[],
134 struct otx2_ssogws_dual *ws = port;
135 struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
137 RTE_SET_USED(nb_events);
138 otx2_ssogws_dual_forward_event(ws, vws, ev);
143 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
145 otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev, \
146 uint64_t timeout_ticks) \
148 struct otx2_ssogws_dual *ws = port; \
151 RTE_SET_USED(timeout_ticks); \
152 if (ws->swtag_req) { \
153 otx2_ssogws_swtag_wait((struct otx2_ssogws *) \
154 &ws->ws_state[!ws->vws]); \
159 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws], \
160 &ws->ws_state[!ws->vws], ev, \
161 flags, ws->lookup_mem, \
163 ws->vws = !ws->vws; \
169 otx2_ssogws_dual_deq_burst_ ##name(void *port, struct rte_event ev[], \
170 uint16_t nb_events, \
171 uint64_t timeout_ticks) \
173 RTE_SET_USED(nb_events); \
175 return otx2_ssogws_dual_deq_ ##name(port, ev, timeout_ticks); \
179 otx2_ssogws_dual_deq_timeout_ ##name(void *port, struct rte_event *ev, \
180 uint64_t timeout_ticks) \
182 struct otx2_ssogws_dual *ws = port; \
186 if (ws->swtag_req) { \
187 otx2_ssogws_swtag_wait((struct otx2_ssogws *) \
188 &ws->ws_state[!ws->vws]); \
193 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws], \
194 &ws->ws_state[!ws->vws], ev, \
195 flags, ws->lookup_mem, \
197 ws->vws = !ws->vws; \
198 for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) { \
199 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws], \
200 &ws->ws_state[!ws->vws], \
204 ws->vws = !ws->vws; \
211 otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port, \
212 struct rte_event ev[], \
213 uint16_t nb_events, \
214 uint64_t timeout_ticks) \
216 RTE_SET_USED(nb_events); \
218 return otx2_ssogws_dual_deq_timeout_ ##name(port, ev, \
223 otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev, \
224 uint64_t timeout_ticks) \
226 struct otx2_ssogws_dual *ws = port; \
229 RTE_SET_USED(timeout_ticks); \
230 if (ws->swtag_req) { \
231 otx2_ssogws_swtag_wait((struct otx2_ssogws *) \
232 &ws->ws_state[!ws->vws]); \
237 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws], \
238 &ws->ws_state[!ws->vws], ev, \
239 flags | NIX_RX_MULTI_SEG_F, \
242 ws->vws = !ws->vws; \
248 otx2_ssogws_dual_deq_seg_burst_ ##name(void *port, \
249 struct rte_event ev[], \
250 uint16_t nb_events, \
251 uint64_t timeout_ticks) \
253 RTE_SET_USED(nb_events); \
255 return otx2_ssogws_dual_deq_seg_ ##name(port, ev, \
260 otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port, \
261 struct rte_event *ev, \
262 uint64_t timeout_ticks) \
264 struct otx2_ssogws_dual *ws = port; \
268 if (ws->swtag_req) { \
269 otx2_ssogws_swtag_wait((struct otx2_ssogws *) \
270 &ws->ws_state[!ws->vws]); \
275 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws], \
276 &ws->ws_state[!ws->vws], ev, \
277 flags | NIX_RX_MULTI_SEG_F, \
280 ws->vws = !ws->vws; \
281 for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) { \
282 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws], \
283 &ws->ws_state[!ws->vws], \
285 NIX_RX_MULTI_SEG_F, \
288 ws->vws = !ws->vws; \
295 otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port, \
296 struct rte_event ev[], \
297 uint16_t nb_events, \
298 uint64_t timeout_ticks) \
300 RTE_SET_USED(nb_events); \
302 return otx2_ssogws_dual_deq_seg_timeout_ ##name(port, ev, \
306 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
309 #define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
311 otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port, \
312 struct rte_event ev[], \
313 uint16_t nb_events) \
315 struct otx2_ssogws_dual *ws = port; \
316 struct otx2_ssogws *vws = \
317 (struct otx2_ssogws *)&ws->ws_state[!ws->vws]; \
320 RTE_SET_USED(nb_events); \
321 return otx2_ssogws_event_tx(vws, ev, cmd, flags); \
323 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
326 #define T(name, f5, f4, f3, f2, f1, f0, sz, flags) \
328 otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port, \
329 struct rte_event ev[], \
330 uint16_t nb_events) \
332 struct otx2_ssogws_dual *ws = port; \
333 struct otx2_ssogws *vws = \
334 (struct otx2_ssogws *)&ws->ws_state[!ws->vws]; \
335 uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2]; \
337 RTE_SET_USED(nb_events); \
338 return otx2_ssogws_event_tx(vws, ev, cmd, (flags) | \
339 NIX_TX_MULTI_SEG_F); \
341 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC