event/octeontx2: fix unconditional Tx flush
[dpdk.git] / drivers / event / octeontx2 / otx2_evdev.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(C) 2019 Marvell International Ltd.
3  */
4
5 #ifndef __OTX2_EVDEV_H__
6 #define __OTX2_EVDEV_H__
7
8 #include <rte_eventdev.h>
9 #include <rte_eventdev_pmd.h>
10 #include <rte_event_eth_rx_adapter.h>
11 #include <rte_event_eth_tx_adapter.h>
12
13 #include "otx2_common.h"
14 #include "otx2_dev.h"
15 #include "otx2_ethdev.h"
16 #include "otx2_mempool.h"
17 #include "otx2_tim_evdev.h"
18
19 #define EVENTDEV_NAME_OCTEONTX2_PMD event_octeontx2
20
21 #define sso_func_trace otx2_sso_dbg
22
23 #define OTX2_SSO_MAX_VHGRP                  RTE_EVENT_MAX_QUEUES_PER_DEV
24 #define OTX2_SSO_MAX_VHWS                   (UINT8_MAX)
25 #define OTX2_SSO_FC_NAME                    "otx2_evdev_xaq_fc"
26 #define OTX2_SSO_SQB_LIMIT                  (0x180)
27 #define OTX2_SSO_XAQ_SLACK                  (8)
28 #define OTX2_SSO_XAQ_CACHE_CNT              (0x7)
29 #define OTX2_SSO_WQE_SG_PTR                 (9)
30
31 /* SSO LF register offsets (BAR2) */
32 #define SSO_LF_GGRP_OP_ADD_WORK0            (0x0ull)
33 #define SSO_LF_GGRP_OP_ADD_WORK1            (0x8ull)
34
35 #define SSO_LF_GGRP_QCTL                    (0x20ull)
36 #define SSO_LF_GGRP_EXE_DIS                 (0x80ull)
37 #define SSO_LF_GGRP_INT                     (0x100ull)
38 #define SSO_LF_GGRP_INT_W1S                 (0x108ull)
39 #define SSO_LF_GGRP_INT_ENA_W1S             (0x110ull)
40 #define SSO_LF_GGRP_INT_ENA_W1C             (0x118ull)
41 #define SSO_LF_GGRP_INT_THR                 (0x140ull)
42 #define SSO_LF_GGRP_INT_CNT                 (0x180ull)
43 #define SSO_LF_GGRP_XAQ_CNT                 (0x1b0ull)
44 #define SSO_LF_GGRP_AQ_CNT                  (0x1c0ull)
45 #define SSO_LF_GGRP_AQ_THR                  (0x1e0ull)
46 #define SSO_LF_GGRP_MISC_CNT                (0x200ull)
47
48 /* SSOW LF register offsets (BAR2) */
49 #define SSOW_LF_GWS_LINKS                   (0x10ull)
50 #define SSOW_LF_GWS_PENDWQP                 (0x40ull)
51 #define SSOW_LF_GWS_PENDSTATE               (0x50ull)
52 #define SSOW_LF_GWS_NW_TIM                  (0x70ull)
53 #define SSOW_LF_GWS_GRPMSK_CHG              (0x80ull)
54 #define SSOW_LF_GWS_INT                     (0x100ull)
55 #define SSOW_LF_GWS_INT_W1S                 (0x108ull)
56 #define SSOW_LF_GWS_INT_ENA_W1S             (0x110ull)
57 #define SSOW_LF_GWS_INT_ENA_W1C             (0x118ull)
58 #define SSOW_LF_GWS_TAG                     (0x200ull)
59 #define SSOW_LF_GWS_WQP                     (0x210ull)
60 #define SSOW_LF_GWS_SWTP                    (0x220ull)
61 #define SSOW_LF_GWS_PENDTAG                 (0x230ull)
62 #define SSOW_LF_GWS_OP_ALLOC_WE             (0x400ull)
63 #define SSOW_LF_GWS_OP_GET_WORK             (0x600ull)
64 #define SSOW_LF_GWS_OP_SWTAG_FLUSH          (0x800ull)
65 #define SSOW_LF_GWS_OP_SWTAG_UNTAG          (0x810ull)
66 #define SSOW_LF_GWS_OP_SWTP_CLR             (0x820ull)
67 #define SSOW_LF_GWS_OP_UPD_WQP_GRP0         (0x830ull)
68 #define SSOW_LF_GWS_OP_UPD_WQP_GRP1         (0x838ull)
69 #define SSOW_LF_GWS_OP_DESCHED              (0x880ull)
70 #define SSOW_LF_GWS_OP_DESCHED_NOSCH        (0x8c0ull)
71 #define SSOW_LF_GWS_OP_SWTAG_DESCHED        (0x980ull)
72 #define SSOW_LF_GWS_OP_SWTAG_NOSCHED        (0x9c0ull)
73 #define SSOW_LF_GWS_OP_CLR_NSCHED0          (0xa00ull)
74 #define SSOW_LF_GWS_OP_CLR_NSCHED1          (0xa08ull)
75 #define SSOW_LF_GWS_OP_SWTP_SET             (0xc00ull)
76 #define SSOW_LF_GWS_OP_SWTAG_NORM           (0xc10ull)
77 #define SSOW_LF_GWS_OP_SWTAG_FULL0          (0xc20ull)
78 #define SSOW_LF_GWS_OP_SWTAG_FULL1          (0xc28ull)
79 #define SSOW_LF_GWS_OP_GWC_INVAL            (0xe00ull)
80
81 #define OTX2_SSOW_GET_BASE_ADDR(_GW)        ((_GW) - SSOW_LF_GWS_OP_GET_WORK)
82 #define OTX2_SSOW_TT_FROM_TAG(x)            (((x) >> 32) & SSO_TT_EMPTY)
83
84 #define NSEC2USEC(__ns)                 ((__ns) / 1E3)
85 #define USEC2NSEC(__us)                 ((__us) * 1E3)
86 #define NSEC2TICK(__ns, __freq)         (((__ns) * (__freq)) / 1E9)
87 #define TICK2NSEC(__tck, __freq)        (((__tck) * 1E9) / (__freq))
88
89 enum otx2_sso_lf_type {
90         SSO_LF_GGRP,
91         SSO_LF_GWS
92 };
93
94 union otx2_sso_event {
95         uint64_t get_work0;
96         struct {
97                 uint32_t flow_id:20;
98                 uint32_t sub_event_type:8;
99                 uint32_t event_type:4;
100                 uint8_t op:2;
101                 uint8_t rsvd:4;
102                 uint8_t sched_type:2;
103                 uint8_t queue_id;
104                 uint8_t priority;
105                 uint8_t impl_opaque;
106         };
107 } __rte_aligned(64);
108
109 enum {
110         SSO_SYNC_ORDERED,
111         SSO_SYNC_ATOMIC,
112         SSO_SYNC_UNTAGGED,
113         SSO_SYNC_EMPTY
114 };
115
116 struct otx2_sso_qos {
117         uint8_t queue;
118         uint8_t xaq_prcnt;
119         uint8_t taq_prcnt;
120         uint8_t iaq_prcnt;
121 };
122
123 struct otx2_sso_evdev {
124         OTX2_DEV; /* Base class */
125         uint8_t max_event_queues;
126         uint8_t max_event_ports;
127         uint8_t is_timeout_deq;
128         uint8_t nb_event_queues;
129         uint8_t nb_event_ports;
130         uint8_t configured;
131         uint32_t deq_tmo_ns;
132         uint32_t min_dequeue_timeout_ns;
133         uint32_t max_dequeue_timeout_ns;
134         int32_t max_num_events;
135         uint64_t *fc_mem;
136         uint64_t xaq_lmt;
137         uint64_t nb_xaq_cfg;
138         rte_iova_t fc_iova;
139         struct rte_mempool *xaq_pool;
140         uint64_t rx_offloads;
141         uint64_t tx_offloads;
142         uint64_t adptr_xae_cnt;
143         uint16_t rx_adptr_pool_cnt;
144         uint64_t *rx_adptr_pools;
145         uint16_t max_port_id;
146         uint16_t tim_adptr_ring_cnt;
147         uint16_t *timer_adptr_rings;
148         uint64_t *timer_adptr_sz;
149         /* Dev args */
150         uint8_t dual_ws;
151         uint32_t xae_cnt;
152         uint8_t qos_queue_cnt;
153         struct otx2_sso_qos *qos_parse_data;
154         /* HW const */
155         uint32_t xae_waes;
156         uint32_t xaq_buf_size;
157         uint32_t iue;
158         /* MSIX offsets */
159         uint16_t sso_msixoff[OTX2_SSO_MAX_VHGRP];
160         uint16_t ssow_msixoff[OTX2_SSO_MAX_VHWS];
161         /* PTP timestamp */
162         struct otx2_timesync_info *tstamp;
163 } __rte_cache_aligned;
164
165 #define OTX2_SSOGWS_OPS                                                        \
166         /* WS ops */                                                           \
167         uintptr_t getwrk_op;                                                   \
168         uintptr_t tag_op;                                                      \
169         uintptr_t wqp_op;                                                      \
170         uintptr_t swtag_flush_op;                                              \
171         uintptr_t swtag_norm_op;                                               \
172         uintptr_t swtag_desched_op;                                            \
173         uint8_t cur_tt;                                                        \
174         uint8_t cur_grp
175
176 /* Event port aka GWS */
177 struct otx2_ssogws {
178         /* Get Work Fastpath data */
179         OTX2_SSOGWS_OPS;
180         uint8_t swtag_req;
181         void *lookup_mem;
182         uint8_t port;
183         /* Add Work Fastpath data */
184         uint64_t xaq_lmt __rte_cache_aligned;
185         uint64_t *fc_mem;
186         uintptr_t grps_base[OTX2_SSO_MAX_VHGRP];
187         /* PTP timestamp */
188         struct otx2_timesync_info *tstamp;
189         /* Tx Fastpath data */
190         uint8_t tx_adptr_data[] __rte_cache_aligned;
191 } __rte_cache_aligned;
192
193 struct otx2_ssogws_state {
194         OTX2_SSOGWS_OPS;
195 };
196
197 struct otx2_ssogws_dual {
198         /* Get Work Fastpath data */
199         struct otx2_ssogws_state ws_state[2]; /* Ping and Pong */
200         uint8_t swtag_req;
201         uint8_t vws; /* Ping pong bit */
202         void *lookup_mem;
203         uint8_t port;
204         /* Add Work Fastpath data */
205         uint64_t xaq_lmt __rte_cache_aligned;
206         uint64_t *fc_mem;
207         uintptr_t grps_base[OTX2_SSO_MAX_VHGRP];
208         /* PTP timestamp */
209         struct otx2_timesync_info *tstamp;
210         /* Tx Fastpath data */
211         uint8_t tx_adptr_data[] __rte_cache_aligned;
212 } __rte_cache_aligned;
213
214 static inline struct otx2_sso_evdev *
215 sso_pmd_priv(const struct rte_eventdev *event_dev)
216 {
217         return event_dev->data->dev_private;
218 }
219
220 static const union mbuf_initializer mbuf_init = {
221         .fields = {
222                 .data_off = RTE_PKTMBUF_HEADROOM,
223                 .refcnt = 1,
224                 .nb_segs = 1,
225                 .port = 0
226         }
227 };
228
229 static __rte_always_inline void
230 otx2_wqe_to_mbuf(uint64_t get_work1, const uint64_t mbuf, uint8_t port_id,
231                  const uint32_t tag, const uint32_t flags,
232                  const void * const lookup_mem)
233 {
234         struct nix_wqe_hdr_s *wqe = (struct nix_wqe_hdr_s *)get_work1;
235         uint64_t val = mbuf_init.value | (uint64_t)port_id << 48;
236
237         if (flags & NIX_RX_OFFLOAD_TSTAMP_F)
238                 val |= NIX_TIMESYNC_RX_OFFSET;
239
240         otx2_nix_cqe_to_mbuf((struct nix_cqe_hdr_s *)wqe, tag,
241                              (struct rte_mbuf *)mbuf, lookup_mem,
242                               val, flags);
243
244 }
245
246 static inline int
247 parse_kvargs_flag(const char *key, const char *value, void *opaque)
248 {
249         RTE_SET_USED(key);
250
251         *(uint8_t *)opaque = !!atoi(value);
252         return 0;
253 }
254
255 static inline int
256 parse_kvargs_value(const char *key, const char *value, void *opaque)
257 {
258         RTE_SET_USED(key);
259
260         *(uint32_t *)opaque = (uint32_t)atoi(value);
261         return 0;
262 }
263
264 #define SSO_RX_ADPTR_ENQ_FASTPATH_FUNC  NIX_RX_FASTPATH_MODES
265 #define SSO_TX_ADPTR_ENQ_FASTPATH_FUNC  NIX_TX_FASTPATH_MODES
266
267 /* Single WS API's */
268 uint16_t otx2_ssogws_enq(void *port, const struct rte_event *ev);
269 uint16_t otx2_ssogws_enq_burst(void *port, const struct rte_event ev[],
270                                uint16_t nb_events);
271 uint16_t otx2_ssogws_enq_new_burst(void *port, const struct rte_event ev[],
272                                    uint16_t nb_events);
273 uint16_t otx2_ssogws_enq_fwd_burst(void *port, const struct rte_event ev[],
274                                    uint16_t nb_events);
275
276 /* Dual WS API's */
277 uint16_t otx2_ssogws_dual_enq(void *port, const struct rte_event *ev);
278 uint16_t otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[],
279                                     uint16_t nb_events);
280 uint16_t otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[],
281                                         uint16_t nb_events);
282 uint16_t otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[],
283                                         uint16_t nb_events);
284
285 /* Auto generated API's */
286 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
287 uint16_t otx2_ssogws_deq_ ##name(void *port, struct rte_event *ev,             \
288                                  uint64_t timeout_ticks);                      \
289 uint16_t otx2_ssogws_deq_burst_ ##name(void *port, struct rte_event ev[],      \
290                                        uint16_t nb_events,                     \
291                                        uint64_t timeout_ticks);                \
292 uint16_t otx2_ssogws_deq_timeout_ ##name(void *port,                           \
293                                          struct rte_event *ev,                 \
294                                          uint64_t timeout_ticks);              \
295 uint16_t otx2_ssogws_deq_timeout_burst_ ##name(void *port,                     \
296                                                struct rte_event ev[],          \
297                                                uint16_t nb_events,             \
298                                                uint64_t timeout_ticks);        \
299 uint16_t otx2_ssogws_deq_seg_ ##name(void *port, struct rte_event *ev,         \
300                                      uint64_t timeout_ticks);                  \
301 uint16_t otx2_ssogws_deq_seg_burst_ ##name(void *port,                         \
302                                            struct rte_event ev[],              \
303                                            uint16_t nb_events,                 \
304                                            uint64_t timeout_ticks);            \
305 uint16_t otx2_ssogws_deq_seg_timeout_ ##name(void *port,                       \
306                                              struct rte_event *ev,             \
307                                              uint64_t timeout_ticks);          \
308 uint16_t otx2_ssogws_deq_seg_timeout_burst_ ##name(void *port,                 \
309                                                    struct rte_event ev[],      \
310                                                    uint16_t nb_events,         \
311                                                    uint64_t timeout_ticks);    \
312                                                                                \
313 uint16_t otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev,        \
314                                       uint64_t timeout_ticks);                 \
315 uint16_t otx2_ssogws_dual_deq_burst_ ##name(void *port,                        \
316                                             struct rte_event ev[],             \
317                                             uint16_t nb_events,                \
318                                             uint64_t timeout_ticks);           \
319 uint16_t otx2_ssogws_dual_deq_timeout_ ##name(void *port,                      \
320                                               struct rte_event *ev,            \
321                                               uint64_t timeout_ticks);         \
322 uint16_t otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port,                \
323                                                     struct rte_event ev[],     \
324                                                     uint16_t nb_events,        \
325                                                     uint64_t timeout_ticks);   \
326 uint16_t otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev,    \
327                                           uint64_t timeout_ticks);             \
328 uint16_t otx2_ssogws_dual_deq_seg_burst_ ##name(void *port,                    \
329                                                 struct rte_event ev[],         \
330                                                 uint16_t nb_events,            \
331                                                 uint64_t timeout_ticks);       \
332 uint16_t otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port,                  \
333                                                   struct rte_event *ev,        \
334                                                   uint64_t timeout_ticks);     \
335 uint16_t otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port,            \
336                                                         struct rte_event ev[], \
337                                                         uint16_t nb_events,    \
338                                                        uint64_t timeout_ticks);\
339
340 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
341 #undef R
342
343 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                       \
344 uint16_t otx2_ssogws_tx_adptr_enq_ ## name(void *port, struct rte_event ev[],\
345                                            uint16_t nb_events);              \
346 uint16_t otx2_ssogws_tx_adptr_enq_seg_ ## name(void *port,                   \
347                                                struct rte_event ev[],        \
348                                                uint16_t nb_events);          \
349 uint16_t otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port,                  \
350                                                 struct rte_event ev[],       \
351                                                 uint16_t nb_events);         \
352 uint16_t otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port,              \
353                                                     struct rte_event ev[],   \
354                                                     uint16_t nb_events);     \
355
356 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
357 #undef T
358
359 void sso_updt_xae_cnt(struct otx2_sso_evdev *dev, void *data,
360                       uint32_t event_type);
361 int sso_xae_reconfigure(struct rte_eventdev *event_dev);
362 void sso_fastpath_fns_set(struct rte_eventdev *event_dev);
363
364 int otx2_sso_rx_adapter_caps_get(const struct rte_eventdev *event_dev,
365                                  const struct rte_eth_dev *eth_dev,
366                                  uint32_t *caps);
367 int otx2_sso_rx_adapter_queue_add(const struct rte_eventdev *event_dev,
368                                   const struct rte_eth_dev *eth_dev,
369                                   int32_t rx_queue_id,
370                 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf);
371 int otx2_sso_rx_adapter_queue_del(const struct rte_eventdev *event_dev,
372                                   const struct rte_eth_dev *eth_dev,
373                                   int32_t rx_queue_id);
374 int otx2_sso_rx_adapter_start(const struct rte_eventdev *event_dev,
375                               const struct rte_eth_dev *eth_dev);
376 int otx2_sso_rx_adapter_stop(const struct rte_eventdev *event_dev,
377                              const struct rte_eth_dev *eth_dev);
378 int otx2_sso_tx_adapter_caps_get(const struct rte_eventdev *dev,
379                                  const struct rte_eth_dev *eth_dev,
380                                  uint32_t *caps);
381 int otx2_sso_tx_adapter_queue_add(uint8_t id,
382                                   const struct rte_eventdev *event_dev,
383                                   const struct rte_eth_dev *eth_dev,
384                                   int32_t tx_queue_id);
385
386 int otx2_sso_tx_adapter_queue_del(uint8_t id,
387                                   const struct rte_eventdev *event_dev,
388                                   const struct rte_eth_dev *eth_dev,
389                                   int32_t tx_queue_id);
390
391 /* Event crypto adapter API's */
392 int otx2_ca_caps_get(const struct rte_eventdev *dev,
393                      const struct rte_cryptodev *cdev, uint32_t *caps);
394
395 int otx2_ca_qp_add(const struct rte_eventdev *dev,
396                    const struct rte_cryptodev *cdev, int32_t queue_pair_id,
397                    const struct rte_event *event);
398
399 int otx2_ca_qp_del(const struct rte_eventdev *dev,
400                    const struct rte_cryptodev *cdev, int32_t queue_pair_id);
401
402 /* Clean up API's */
403 typedef void (*otx2_handle_event_t)(void *arg, struct rte_event ev);
404 void ssogws_flush_events(struct otx2_ssogws *ws, uint8_t queue_id,
405                          uintptr_t base, otx2_handle_event_t fn, void *arg);
406 void ssogws_reset(struct otx2_ssogws *ws);
407 /* Selftest */
408 int otx2_sso_selftest(void);
409 /* Init and Fini API's */
410 int otx2_sso_init(struct rte_eventdev *event_dev);
411 int otx2_sso_fini(struct rte_eventdev *event_dev);
412 /* IRQ handlers */
413 int sso_register_irqs(const struct rte_eventdev *event_dev);
414 void sso_unregister_irqs(const struct rte_eventdev *event_dev);
415
416 #endif /* __OTX2_EVDEV_H__ */