946488eabfead612783528f56b02f0df7a58b7b1
[dpdk.git] / drivers / event / octeontx2 / otx2_worker_dual.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(C) 2019 Marvell International Ltd.
3  */
4
5 #include "otx2_worker_dual.h"
6 #include "otx2_worker.h"
7
8 static __rte_noinline uint8_t
9 otx2_ssogws_dual_new_event(struct otx2_ssogws_dual *ws,
10                            const struct rte_event *ev)
11 {
12         const uint32_t tag = (uint32_t)ev->event;
13         const uint8_t new_tt = ev->sched_type;
14         const uint64_t event_ptr = ev->u64;
15         const uint16_t grp = ev->queue_id;
16
17         if (ws->xaq_lmt <= *ws->fc_mem)
18                 return 0;
19
20         otx2_ssogws_dual_add_work(ws, event_ptr, tag, new_tt, grp);
21
22         return 1;
23 }
24
25 static __rte_always_inline void
26 otx2_ssogws_dual_fwd_swtag(struct otx2_ssogws_state *ws,
27                            const struct rte_event *ev)
28 {
29         const uint32_t tag = (uint32_t)ev->event;
30         const uint8_t new_tt = ev->sched_type;
31         const uint8_t cur_tt = ws->cur_tt;
32
33         /* 96XX model
34          * cur_tt/new_tt     SSO_SYNC_ORDERED SSO_SYNC_ATOMIC SSO_SYNC_UNTAGGED
35          *
36          * SSO_SYNC_ORDERED        norm           norm             untag
37          * SSO_SYNC_ATOMIC         norm           norm             untag
38          * SSO_SYNC_UNTAGGED       norm           norm             NOOP
39          */
40         if (new_tt == SSO_SYNC_UNTAGGED) {
41                 if (cur_tt != SSO_SYNC_UNTAGGED)
42                         otx2_ssogws_swtag_untag((struct otx2_ssogws *)ws);
43         } else {
44                 otx2_ssogws_swtag_norm((struct otx2_ssogws *)ws, tag, new_tt);
45         }
46 }
47
48 static __rte_always_inline void
49 otx2_ssogws_dual_fwd_group(struct otx2_ssogws_state *ws,
50                            const struct rte_event *ev, const uint16_t grp)
51 {
52         const uint32_t tag = (uint32_t)ev->event;
53         const uint8_t new_tt = ev->sched_type;
54
55         otx2_write64(ev->u64, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
56                      SSOW_LF_GWS_OP_UPD_WQP_GRP1);
57         rte_smp_wmb();
58         otx2_ssogws_swtag_desched((struct otx2_ssogws *)ws, tag, new_tt, grp);
59 }
60
61 static __rte_always_inline void
62 otx2_ssogws_dual_forward_event(struct otx2_ssogws_dual *ws,
63                                struct otx2_ssogws_state *vws,
64                                const struct rte_event *ev)
65 {
66         const uint8_t grp = ev->queue_id;
67
68         /* Group hasn't changed, Use SWTAG to forward the event */
69         if (vws->cur_grp == grp) {
70                 otx2_ssogws_dual_fwd_swtag(vws, ev);
71                 ws->swtag_req = 1;
72         } else {
73         /*
74          * Group has been changed for group based work pipelining,
75          * Use deschedule/add_work operation to transfer the event to
76          * new group/core
77          */
78                 otx2_ssogws_dual_fwd_group(vws, ev, grp);
79         }
80 }
81
82 uint16_t __rte_hot
83 otx2_ssogws_dual_enq(void *port, const struct rte_event *ev)
84 {
85         struct otx2_ssogws_dual *ws = port;
86         struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
87
88         switch (ev->op) {
89         case RTE_EVENT_OP_NEW:
90                 rte_smp_mb();
91                 return otx2_ssogws_dual_new_event(ws, ev);
92         case RTE_EVENT_OP_FORWARD:
93                 otx2_ssogws_dual_forward_event(ws, vws, ev);
94                 break;
95         case RTE_EVENT_OP_RELEASE:
96                 otx2_ssogws_swtag_flush((struct otx2_ssogws *)vws);
97                 break;
98         default:
99                 return 0;
100         }
101
102         return 1;
103 }
104
105 uint16_t __rte_hot
106 otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[],
107                            uint16_t nb_events)
108 {
109         RTE_SET_USED(nb_events);
110         return otx2_ssogws_dual_enq(port, ev);
111 }
112
113 uint16_t __rte_hot
114 otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[],
115                                uint16_t nb_events)
116 {
117         struct otx2_ssogws_dual *ws = port;
118         uint16_t i, rc = 1;
119
120         rte_smp_mb();
121         if (ws->xaq_lmt <= *ws->fc_mem)
122                 return 0;
123
124         for (i = 0; i < nb_events && rc; i++)
125                 rc = otx2_ssogws_dual_new_event(ws, &ev[i]);
126
127         return nb_events;
128 }
129
130 uint16_t __rte_hot
131 otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[],
132                                uint16_t nb_events)
133 {
134         struct otx2_ssogws_dual *ws = port;
135         struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
136
137         RTE_SET_USED(nb_events);
138         otx2_ssogws_dual_forward_event(ws, vws, ev);
139
140         return 1;
141 }
142
143 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                      \
144 uint16_t __rte_hot                                                              \
145 otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev,          \
146                              uint64_t timeout_ticks)                    \
147 {                                                                       \
148         struct otx2_ssogws_dual *ws = port;                             \
149         uint8_t gw;                                                     \
150                                                                         \
151         rte_prefetch_non_temporal(ws);                                  \
152         RTE_SET_USED(timeout_ticks);                                    \
153         if (ws->swtag_req) {                                            \
154                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
155                                        &ws->ws_state[!ws->vws]);        \
156                 ws->swtag_req = 0;                                      \
157                 return 1;                                               \
158         }                                                               \
159                                                                         \
160         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
161                                        &ws->ws_state[!ws->vws], ev,     \
162                                        flags, ws->lookup_mem,           \
163                                        ws->tstamp);                     \
164         ws->vws = !ws->vws;                                             \
165                                                                         \
166         return gw;                                                      \
167 }                                                                       \
168                                                                         \
169 uint16_t __rte_hot                                                              \
170 otx2_ssogws_dual_deq_burst_ ##name(void *port, struct rte_event ev[],   \
171                                    uint16_t nb_events,                  \
172                                    uint64_t timeout_ticks)              \
173 {                                                                       \
174         RTE_SET_USED(nb_events);                                        \
175                                                                         \
176         return otx2_ssogws_dual_deq_ ##name(port, ev, timeout_ticks);   \
177 }                                                                       \
178                                                                         \
179 uint16_t __rte_hot                                                              \
180 otx2_ssogws_dual_deq_timeout_ ##name(void *port, struct rte_event *ev,  \
181                                      uint64_t timeout_ticks)            \
182 {                                                                       \
183         struct otx2_ssogws_dual *ws = port;                             \
184         uint64_t iter;                                                  \
185         uint8_t gw;                                                     \
186                                                                         \
187         if (ws->swtag_req) {                                            \
188                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
189                                        &ws->ws_state[!ws->vws]);        \
190                 ws->swtag_req = 0;                                      \
191                 return 1;                                               \
192         }                                                               \
193                                                                         \
194         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
195                                        &ws->ws_state[!ws->vws], ev,     \
196                                        flags, ws->lookup_mem,           \
197                                        ws->tstamp);                     \
198         ws->vws = !ws->vws;                                             \
199         for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) {     \
200                 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],  \
201                                                &ws->ws_state[!ws->vws], \
202                                                ev, flags,               \
203                                                ws->lookup_mem,          \
204                                                ws->tstamp);             \
205                 ws->vws = !ws->vws;                                     \
206         }                                                               \
207                                                                         \
208         return gw;                                                      \
209 }                                                                       \
210                                                                         \
211 uint16_t __rte_hot                                                              \
212 otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port,                  \
213                                            struct rte_event ev[],       \
214                                            uint16_t nb_events,          \
215                                            uint64_t timeout_ticks)      \
216 {                                                                       \
217         RTE_SET_USED(nb_events);                                        \
218                                                                         \
219         return otx2_ssogws_dual_deq_timeout_ ##name(port, ev,           \
220                                                     timeout_ticks);     \
221 }                                                                       \
222                                                                         \
223 uint16_t __rte_hot                                                              \
224 otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev,      \
225                                  uint64_t timeout_ticks)                \
226 {                                                                       \
227         struct otx2_ssogws_dual *ws = port;                             \
228         uint8_t gw;                                                     \
229                                                                         \
230         RTE_SET_USED(timeout_ticks);                                    \
231         if (ws->swtag_req) {                                            \
232                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
233                                        &ws->ws_state[!ws->vws]);        \
234                 ws->swtag_req = 0;                                      \
235                 return 1;                                               \
236         }                                                               \
237                                                                         \
238         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
239                                        &ws->ws_state[!ws->vws], ev,     \
240                                        flags | NIX_RX_MULTI_SEG_F,      \
241                                        ws->lookup_mem,                  \
242                                        ws->tstamp);                     \
243         ws->vws = !ws->vws;                                             \
244                                                                         \
245         return gw;                                                      \
246 }                                                                       \
247                                                                         \
248 uint16_t __rte_hot                                                              \
249 otx2_ssogws_dual_deq_seg_burst_ ##name(void *port,                      \
250                                        struct rte_event ev[],           \
251                                        uint16_t nb_events,              \
252                                        uint64_t timeout_ticks)          \
253 {                                                                       \
254         RTE_SET_USED(nb_events);                                        \
255                                                                         \
256         return otx2_ssogws_dual_deq_seg_ ##name(port, ev,               \
257                                                 timeout_ticks);         \
258 }                                                                       \
259                                                                         \
260 uint16_t __rte_hot                                                              \
261 otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port,                    \
262                                          struct rte_event *ev,          \
263                                          uint64_t timeout_ticks)        \
264 {                                                                       \
265         struct otx2_ssogws_dual *ws = port;                             \
266         uint64_t iter;                                                  \
267         uint8_t gw;                                                     \
268                                                                         \
269         if (ws->swtag_req) {                                            \
270                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
271                                        &ws->ws_state[!ws->vws]);        \
272                 ws->swtag_req = 0;                                      \
273                 return 1;                                               \
274         }                                                               \
275                                                                         \
276         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
277                                        &ws->ws_state[!ws->vws], ev,     \
278                                        flags | NIX_RX_MULTI_SEG_F,      \
279                                        ws->lookup_mem,                  \
280                                        ws->tstamp);                     \
281         ws->vws = !ws->vws;                                             \
282         for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) {     \
283                 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],  \
284                                                &ws->ws_state[!ws->vws], \
285                                                ev, flags |              \
286                                                NIX_RX_MULTI_SEG_F,      \
287                                                ws->lookup_mem,          \
288                                                ws->tstamp);             \
289                 ws->vws = !ws->vws;                                     \
290         }                                                               \
291                                                                         \
292         return gw;                                                      \
293 }                                                                       \
294                                                                         \
295 uint16_t __rte_hot                                                              \
296 otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port,              \
297                                                struct rte_event ev[],   \
298                                                uint16_t nb_events,      \
299                                                uint64_t timeout_ticks)  \
300 {                                                                       \
301         RTE_SET_USED(nb_events);                                        \
302                                                                         \
303         return otx2_ssogws_dual_deq_seg_timeout_ ##name(port, ev,       \
304                                                         timeout_ticks); \
305 }
306
307 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
308 #undef R
309
310 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                  \
311 uint16_t __rte_hot                                                      \
312 otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port,                      \
313                                        struct rte_event ev[],           \
314                                        uint16_t nb_events)              \
315 {                                                                       \
316         struct otx2_ssogws_dual *ws = port;                             \
317         struct otx2_ssogws *vws =                                       \
318                 (struct otx2_ssogws *)&ws->ws_state[!ws->vws];          \
319         uint64_t cmd[sz];                                               \
320                                                                         \
321         RTE_SET_USED(nb_events);                                        \
322         return otx2_ssogws_event_tx(vws, ev, cmd, (const uint64_t       \
323                                     (*)[RTE_MAX_QUEUES_PER_PORT])       \
324                                     ws->tx_adptr_data,                  \
325                                     flags);                             \
326 }
327 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
328 #undef T
329
330 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                  \
331 uint16_t __rte_hot                                                      \
332 otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port,                  \
333                                            struct rte_event ev[],       \
334                                            uint16_t nb_events)          \
335 {                                                                       \
336         struct otx2_ssogws_dual *ws = port;                             \
337         struct otx2_ssogws *vws =                                       \
338                 (struct otx2_ssogws *)&ws->ws_state[!ws->vws];          \
339         uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2];                 \
340                                                                         \
341         RTE_SET_USED(nb_events);                                        \
342         return otx2_ssogws_event_tx(vws, ev, cmd, (const uint64_t       \
343                                     (*)[RTE_MAX_QUEUES_PER_PORT])       \
344                                     ws->tx_adptr_data,                  \
345                                     (flags) | NIX_TX_MULTI_SEG_F);      \
346 }
347 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
348 #undef T