1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
5 #ifndef __CN9K_WORKER_H__
6 #define __CN9K_WORKER_H__
8 #include "cnxk_ethdev.h"
9 #include "cnxk_eventdev.h"
10 #include "cnxk_worker.h"
12 #include "cn9k_ethdev.h"
17 static __rte_always_inline uint8_t
18 cn9k_sso_hws_new_event(struct cn9k_sso_hws *ws, const struct rte_event *ev)
20 const uint32_t tag = (uint32_t)ev->event;
21 const uint8_t new_tt = ev->sched_type;
22 const uint64_t event_ptr = ev->u64;
23 const uint16_t grp = ev->queue_id;
25 rte_atomic_thread_fence(__ATOMIC_ACQ_REL);
26 if (ws->xaq_lmt <= *ws->fc_mem)
29 cnxk_sso_hws_add_work(event_ptr, tag, new_tt, ws->grps_base[grp]);
33 static __rte_always_inline void
34 cn9k_sso_hws_fwd_swtag(struct cn9k_sso_hws_state *vws,
35 const struct rte_event *ev)
37 const uint32_t tag = (uint32_t)ev->event;
38 const uint8_t new_tt = ev->sched_type;
39 const uint8_t cur_tt = CNXK_TT_FROM_TAG(plt_read64(vws->tag_op));
42 * cur_tt/new_tt SSO_TT_ORDERED SSO_TT_ATOMIC SSO_TT_UNTAGGED
44 * SSO_TT_ORDERED norm norm untag
45 * SSO_TT_ATOMIC norm norm untag
46 * SSO_TT_UNTAGGED norm norm NOOP
49 if (new_tt == SSO_TT_UNTAGGED) {
50 if (cur_tt != SSO_TT_UNTAGGED)
51 cnxk_sso_hws_swtag_untag(
52 CN9K_SSOW_GET_BASE_ADDR(vws->getwrk_op) +
53 SSOW_LF_GWS_OP_SWTAG_UNTAG);
55 cnxk_sso_hws_swtag_norm(tag, new_tt, vws->swtag_norm_op);
59 static __rte_always_inline void
60 cn9k_sso_hws_fwd_group(struct cn9k_sso_hws_state *ws,
61 const struct rte_event *ev, const uint16_t grp)
63 const uint32_t tag = (uint32_t)ev->event;
64 const uint8_t new_tt = ev->sched_type;
66 plt_write64(ev->u64, CN9K_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
67 SSOW_LF_GWS_OP_UPD_WQP_GRP1);
68 cnxk_sso_hws_swtag_desched(tag, new_tt, grp, ws->swtag_desched_op);
71 static __rte_always_inline void
72 cn9k_sso_hws_forward_event(struct cn9k_sso_hws *ws, const struct rte_event *ev)
74 const uint8_t grp = ev->queue_id;
76 /* Group hasn't changed, Use SWTAG to forward the event */
77 if (CNXK_GRP_FROM_TAG(plt_read64(ws->tag_op)) == grp) {
78 cn9k_sso_hws_fwd_swtag((struct cn9k_sso_hws_state *)ws, ev);
82 * Group has been changed for group based work pipelining,
83 * Use deschedule/add_work operation to transfer the event to
86 cn9k_sso_hws_fwd_group((struct cn9k_sso_hws_state *)ws, ev,
93 static __rte_always_inline uint8_t
94 cn9k_sso_hws_dual_new_event(struct cn9k_sso_hws_dual *dws,
95 const struct rte_event *ev)
97 const uint32_t tag = (uint32_t)ev->event;
98 const uint8_t new_tt = ev->sched_type;
99 const uint64_t event_ptr = ev->u64;
100 const uint16_t grp = ev->queue_id;
102 rte_atomic_thread_fence(__ATOMIC_ACQ_REL);
103 if (dws->xaq_lmt <= *dws->fc_mem)
106 cnxk_sso_hws_add_work(event_ptr, tag, new_tt, dws->grps_base[grp]);
110 static __rte_always_inline void
111 cn9k_sso_hws_dual_forward_event(struct cn9k_sso_hws_dual *dws,
112 struct cn9k_sso_hws_state *vws,
113 const struct rte_event *ev)
115 const uint8_t grp = ev->queue_id;
117 /* Group hasn't changed, Use SWTAG to forward the event */
118 if (CNXK_GRP_FROM_TAG(plt_read64(vws->tag_op)) == grp) {
119 cn9k_sso_hws_fwd_swtag(vws, ev);
123 * Group has been changed for group based work pipelining,
124 * Use deschedule/add_work operation to transfer the event to
127 cn9k_sso_hws_fwd_group(vws, ev, grp);
131 static __rte_always_inline uint16_t
132 cn9k_sso_hws_dual_get_work(struct cn9k_sso_hws_state *ws,
133 struct cn9k_sso_hws_state *ws_pair,
134 struct rte_event *ev)
136 const uint64_t set_gw = BIT_ULL(16) | 1;
138 __uint128_t get_work;
142 #ifdef RTE_ARCH_ARM64
143 asm volatile(PLT_CPU_FEATURE_PREAMBLE
145 " ldr %[tag], [%[tag_loc]] \n"
146 " ldr %[wqp], [%[wqp_loc]] \n"
147 " tbnz %[tag], 63, rty%= \n"
148 "done%=: str %[gw], [%[pong]] \n"
150 : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1])
151 : [tag_loc] "r"(ws->tag_op), [wqp_loc] "r"(ws->wqp_op),
152 [gw] "r"(set_gw), [pong] "r"(ws_pair->getwrk_op));
154 gw.u64[0] = plt_read64(ws->tag_op);
155 while ((BIT_ULL(63)) & gw.u64[0])
156 gw.u64[0] = plt_read64(ws->tag_op);
157 gw.u64[1] = plt_read64(ws->wqp_op);
158 plt_write64(set_gw, ws_pair->getwrk_op);
161 gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 |
162 (gw.u64[0] & (0x3FFull << 36)) << 4 |
163 (gw.u64[0] & 0xffffffff);
165 ev->event = gw.u64[0];
171 static __rte_always_inline uint16_t
172 cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev)
175 __uint128_t get_work;
179 plt_write64(BIT_ULL(16) | /* wait for work. */
180 1, /* Use Mask set 0. */
182 #ifdef RTE_ARCH_ARM64
183 asm volatile(PLT_CPU_FEATURE_PREAMBLE
184 " ldr %[tag], [%[tag_loc]] \n"
185 " ldr %[wqp], [%[wqp_loc]] \n"
186 " tbz %[tag], 63, done%= \n"
189 " ldr %[tag], [%[tag_loc]] \n"
190 " ldr %[wqp], [%[wqp_loc]] \n"
191 " tbnz %[tag], 63, rty%= \n"
193 : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1])
194 : [tag_loc] "r"(ws->tag_op), [wqp_loc] "r"(ws->wqp_op));
196 gw.u64[0] = plt_read64(ws->tag_op);
197 while ((BIT_ULL(63)) & gw.u64[0])
198 gw.u64[0] = plt_read64(ws->tag_op);
200 gw.u64[1] = plt_read64(ws->wqp_op);
203 gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 |
204 (gw.u64[0] & (0x3FFull << 36)) << 4 |
205 (gw.u64[0] & 0xffffffff);
207 ev->event = gw.u64[0];
213 /* Used in cleaning up workslot. */
214 static __rte_always_inline uint16_t
215 cn9k_sso_hws_get_work_empty(struct cn9k_sso_hws_state *ws, struct rte_event *ev)
218 __uint128_t get_work;
222 #ifdef RTE_ARCH_ARM64
223 asm volatile(PLT_CPU_FEATURE_PREAMBLE
224 " ldr %[tag], [%[tag_loc]] \n"
225 " ldr %[wqp], [%[wqp_loc]] \n"
226 " tbz %[tag], 63, done%= \n"
229 " ldr %[tag], [%[tag_loc]] \n"
230 " ldr %[wqp], [%[wqp_loc]] \n"
231 " tbnz %[tag], 63, rty%= \n"
233 : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1])
234 : [tag_loc] "r"(ws->tag_op), [wqp_loc] "r"(ws->wqp_op));
236 gw.u64[0] = plt_read64(ws->tag_op);
237 while ((BIT_ULL(63)) & gw.u64[0])
238 gw.u64[0] = plt_read64(ws->tag_op);
240 gw.u64[1] = plt_read64(ws->wqp_op);
243 gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 |
244 (gw.u64[0] & (0x3FFull << 36)) << 4 |
245 (gw.u64[0] & 0xffffffff);
247 ev->event = gw.u64[0];
253 /* CN9K Fastpath functions. */
254 uint16_t __rte_hot cn9k_sso_hws_enq(void *port, const struct rte_event *ev);
255 uint16_t __rte_hot cn9k_sso_hws_enq_burst(void *port,
256 const struct rte_event ev[],
258 uint16_t __rte_hot cn9k_sso_hws_enq_new_burst(void *port,
259 const struct rte_event ev[],
261 uint16_t __rte_hot cn9k_sso_hws_enq_fwd_burst(void *port,
262 const struct rte_event ev[],
265 uint16_t __rte_hot cn9k_sso_hws_dual_enq(void *port,
266 const struct rte_event *ev);
267 uint16_t __rte_hot cn9k_sso_hws_dual_enq_burst(void *port,
268 const struct rte_event ev[],
270 uint16_t __rte_hot cn9k_sso_hws_dual_enq_new_burst(void *port,
271 const struct rte_event ev[],
273 uint16_t __rte_hot cn9k_sso_hws_dual_enq_fwd_burst(void *port,
274 const struct rte_event ev[],
277 uint16_t __rte_hot cn9k_sso_hws_deq(void *port, struct rte_event *ev,
278 uint64_t timeout_ticks);
279 uint16_t __rte_hot cn9k_sso_hws_deq_burst(void *port, struct rte_event ev[],
281 uint64_t timeout_ticks);
282 uint16_t __rte_hot cn9k_sso_hws_tmo_deq(void *port, struct rte_event *ev,
283 uint64_t timeout_ticks);
284 uint16_t __rte_hot cn9k_sso_hws_tmo_deq_burst(void *port, struct rte_event ev[],
286 uint64_t timeout_ticks);
288 uint16_t __rte_hot cn9k_sso_hws_dual_deq(void *port, struct rte_event *ev,
289 uint64_t timeout_ticks);
290 uint16_t __rte_hot cn9k_sso_hws_dual_deq_burst(void *port,
291 struct rte_event ev[],
293 uint64_t timeout_ticks);
294 uint16_t __rte_hot cn9k_sso_hws_dual_tmo_deq(void *port, struct rte_event *ev,
295 uint64_t timeout_ticks);
296 uint16_t __rte_hot cn9k_sso_hws_dual_tmo_deq_burst(void *port,
297 struct rte_event ev[],
299 uint64_t timeout_ticks);