1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
7 #include <rte_bus_pci.h>
8 #include <rte_common.h>
10 #include <rte_eventdev_pmd_pci.h>
11 #include <rte_kvargs.h>
12 #include <rte_mbuf_pool_ops.h>
15 #include "otx2_evdev_stats.h"
16 #include "otx2_evdev.h"
18 #include "otx2_tim_evdev.h"
21 sso_get_msix_offsets(const struct rte_eventdev *event_dev)
23 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
24 uint8_t nb_ports = dev->nb_event_ports * (dev->dual_ws ? 2 : 1);
25 struct otx2_mbox *mbox = dev->mbox;
26 struct msix_offset_rsp *msix_rsp;
29 /* Get SSO and SSOW MSIX vector offsets */
30 otx2_mbox_alloc_msg_msix_offset(mbox);
31 rc = otx2_mbox_process_msg(mbox, (void *)&msix_rsp);
33 for (i = 0; i < nb_ports; i++)
34 dev->ssow_msixoff[i] = msix_rsp->ssow_msixoff[i];
36 for (i = 0; i < dev->nb_event_queues; i++)
37 dev->sso_msixoff[i] = msix_rsp->sso_msixoff[i];
43 sso_fastpath_fns_set(struct rte_eventdev *event_dev)
45 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
47 const event_dequeue_t ssogws_deq[2][2][2][2][2][2] = {
48 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
49 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_ ##name,
50 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
54 const event_dequeue_burst_t ssogws_deq_burst[2][2][2][2][2][2] = {
55 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
56 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_burst_ ##name,
57 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
61 const event_dequeue_t ssogws_deq_timeout[2][2][2][2][2][2] = {
62 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
63 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_timeout_ ##name,
64 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
68 const event_dequeue_burst_t
69 ssogws_deq_timeout_burst[2][2][2][2][2][2] = {
70 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
71 [f5][f4][f3][f2][f1][f0] = \
72 otx2_ssogws_deq_timeout_burst_ ##name,
73 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
77 const event_dequeue_t ssogws_deq_seg[2][2][2][2][2][2] = {
78 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
79 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_seg_ ##name,
80 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
84 const event_dequeue_burst_t ssogws_deq_seg_burst[2][2][2][2][2][2] = {
85 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
86 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_seg_burst_ ##name,
87 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
91 const event_dequeue_t ssogws_deq_seg_timeout[2][2][2][2][2][2] = {
92 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
93 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_seg_timeout_ ##name,
94 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
98 const event_dequeue_burst_t
99 ssogws_deq_seg_timeout_burst[2][2][2][2][2][2] = {
100 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
101 [f5][f4][f3][f2][f1][f0] = \
102 otx2_ssogws_deq_seg_timeout_burst_ ##name,
103 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
109 const event_dequeue_t ssogws_dual_deq[2][2][2][2][2][2] = {
110 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
111 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_ ##name,
112 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
116 const event_dequeue_burst_t ssogws_dual_deq_burst[2][2][2][2][2][2] = {
117 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
118 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_burst_ ##name,
119 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
123 const event_dequeue_t ssogws_dual_deq_timeout[2][2][2][2][2][2] = {
124 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
125 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_timeout_ ##name,
126 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
130 const event_dequeue_burst_t
131 ssogws_dual_deq_timeout_burst[2][2][2][2][2][2] = {
132 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
133 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_timeout_burst_ ##name,
134 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
138 const event_dequeue_t ssogws_dual_deq_seg[2][2][2][2][2][2] = {
139 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
140 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_seg_ ##name,
141 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
145 const event_dequeue_burst_t
146 ssogws_dual_deq_seg_burst[2][2][2][2][2][2] = {
147 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
148 [f5][f4][f3][f2][f1][f0] = \
149 otx2_ssogws_dual_deq_seg_burst_ ##name,
150 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
154 const event_dequeue_t ssogws_dual_deq_seg_timeout[2][2][2][2][2][2] = {
155 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
156 [f5][f4][f3][f2][f1][f0] = \
157 otx2_ssogws_dual_deq_seg_timeout_ ##name,
158 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
162 const event_dequeue_burst_t
163 ssogws_dual_deq_seg_timeout_burst[2][2][2][2][2][2] = {
164 #define R(name, f5, f4, f3, f2, f1, f0, flags) \
165 [f5][f4][f3][f2][f1][f0] = \
166 otx2_ssogws_dual_deq_seg_timeout_burst_ ##name,
167 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
171 event_dev->enqueue = otx2_ssogws_enq;
172 event_dev->enqueue_burst = otx2_ssogws_enq_burst;
173 event_dev->enqueue_new_burst = otx2_ssogws_enq_new_burst;
174 event_dev->enqueue_forward_burst = otx2_ssogws_enq_fwd_burst;
175 if (dev->rx_offloads & NIX_RX_MULTI_SEG_F) {
176 event_dev->dequeue = ssogws_deq_seg
177 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
178 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
179 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
180 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
181 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
182 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
183 event_dev->dequeue_burst = ssogws_deq_seg_burst
184 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
185 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
186 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
187 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
188 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
189 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
190 if (dev->is_timeout_deq) {
191 event_dev->dequeue = ssogws_deq_seg_timeout
192 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
193 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
194 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
195 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
196 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
197 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
198 event_dev->dequeue_burst =
199 ssogws_deq_seg_timeout_burst
200 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
201 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
202 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
203 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
204 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
205 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
208 event_dev->dequeue = ssogws_deq
209 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
210 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
211 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
212 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
213 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
214 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
215 event_dev->dequeue_burst = ssogws_deq_burst
216 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
217 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
218 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
219 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
220 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
221 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
222 if (dev->is_timeout_deq) {
223 event_dev->dequeue = ssogws_deq_timeout
224 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
225 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
226 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
227 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
228 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
229 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
230 event_dev->dequeue_burst =
231 ssogws_deq_timeout_burst
232 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
233 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
234 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
235 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
236 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
237 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
242 event_dev->enqueue = otx2_ssogws_dual_enq;
243 event_dev->enqueue_burst = otx2_ssogws_dual_enq_burst;
244 event_dev->enqueue_new_burst =
245 otx2_ssogws_dual_enq_new_burst;
246 event_dev->enqueue_forward_burst =
247 otx2_ssogws_dual_enq_fwd_burst;
249 if (dev->rx_offloads & NIX_RX_MULTI_SEG_F) {
250 event_dev->dequeue = ssogws_dual_deq_seg
251 [!!(dev->rx_offloads &
252 NIX_RX_OFFLOAD_TSTAMP_F)]
253 [!!(dev->rx_offloads &
254 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
255 [!!(dev->rx_offloads &
256 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
257 [!!(dev->rx_offloads &
258 NIX_RX_OFFLOAD_CHECKSUM_F)]
259 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
260 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
261 event_dev->dequeue_burst = ssogws_dual_deq_seg_burst
262 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
263 [!!(dev->rx_offloads &
264 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
265 [!!(dev->rx_offloads &
266 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
267 [!!(dev->rx_offloads &
268 NIX_RX_OFFLOAD_CHECKSUM_F)]
269 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
270 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
271 if (dev->is_timeout_deq) {
273 ssogws_dual_deq_seg_timeout
274 [!!(dev->rx_offloads &
275 NIX_RX_OFFLOAD_TSTAMP_F)]
276 [!!(dev->rx_offloads &
277 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
278 [!!(dev->rx_offloads &
279 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
280 [!!(dev->rx_offloads &
281 NIX_RX_OFFLOAD_CHECKSUM_F)]
282 [!!(dev->rx_offloads &
283 NIX_RX_OFFLOAD_PTYPE_F)]
284 [!!(dev->rx_offloads &
285 NIX_RX_OFFLOAD_RSS_F)];
286 event_dev->dequeue_burst =
287 ssogws_dual_deq_seg_timeout_burst
288 [!!(dev->rx_offloads &
289 NIX_RX_OFFLOAD_TSTAMP_F)]
290 [!!(dev->rx_offloads &
291 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
292 [!!(dev->rx_offloads &
293 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
294 [!!(dev->rx_offloads &
295 NIX_RX_OFFLOAD_CHECKSUM_F)]
296 [!!(dev->rx_offloads &
297 NIX_RX_OFFLOAD_PTYPE_F)]
298 [!!(dev->rx_offloads &
299 NIX_RX_OFFLOAD_RSS_F)];
302 event_dev->dequeue = ssogws_dual_deq
303 [!!(dev->rx_offloads &
304 NIX_RX_OFFLOAD_TSTAMP_F)]
305 [!!(dev->rx_offloads &
306 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
307 [!!(dev->rx_offloads &
308 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
309 [!!(dev->rx_offloads &
310 NIX_RX_OFFLOAD_CHECKSUM_F)]
311 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
312 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
313 event_dev->dequeue_burst = ssogws_dual_deq_burst
314 [!!(dev->rx_offloads &
315 NIX_RX_OFFLOAD_TSTAMP_F)]
316 [!!(dev->rx_offloads &
317 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
318 [!!(dev->rx_offloads &
319 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
320 [!!(dev->rx_offloads &
321 NIX_RX_OFFLOAD_CHECKSUM_F)]
322 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
323 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
324 if (dev->is_timeout_deq) {
326 ssogws_dual_deq_timeout
327 [!!(dev->rx_offloads &
328 NIX_RX_OFFLOAD_TSTAMP_F)]
329 [!!(dev->rx_offloads &
330 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
331 [!!(dev->rx_offloads &
332 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
333 [!!(dev->rx_offloads &
334 NIX_RX_OFFLOAD_CHECKSUM_F)]
335 [!!(dev->rx_offloads &
336 NIX_RX_OFFLOAD_PTYPE_F)]
337 [!!(dev->rx_offloads &
338 NIX_RX_OFFLOAD_RSS_F)];
339 event_dev->dequeue_burst =
340 ssogws_dual_deq_timeout_burst
341 [!!(dev->rx_offloads &
342 NIX_RX_OFFLOAD_TSTAMP_F)]
343 [!!(dev->rx_offloads &
344 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
345 [!!(dev->rx_offloads &
346 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
347 [!!(dev->rx_offloads &
348 NIX_RX_OFFLOAD_CHECKSUM_F)]
349 [!!(dev->rx_offloads &
350 NIX_RX_OFFLOAD_PTYPE_F)]
351 [!!(dev->rx_offloads &
352 NIX_RX_OFFLOAD_RSS_F)];
360 otx2_sso_info_get(struct rte_eventdev *event_dev,
361 struct rte_event_dev_info *dev_info)
363 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
365 dev_info->driver_name = RTE_STR(EVENTDEV_NAME_OCTEONTX2_PMD);
366 dev_info->min_dequeue_timeout_ns = dev->min_dequeue_timeout_ns;
367 dev_info->max_dequeue_timeout_ns = dev->max_dequeue_timeout_ns;
368 dev_info->max_event_queues = dev->max_event_queues;
369 dev_info->max_event_queue_flows = (1ULL << 20);
370 dev_info->max_event_queue_priority_levels = 8;
371 dev_info->max_event_priority_levels = 1;
372 dev_info->max_event_ports = dev->max_event_ports;
373 dev_info->max_event_port_dequeue_depth = 1;
374 dev_info->max_event_port_enqueue_depth = 1;
375 dev_info->max_num_events = dev->max_num_events;
376 dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_QUEUE_QOS |
377 RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
378 RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES |
379 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK |
380 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT |
381 RTE_EVENT_DEV_CAP_NONSEQ_MODE;
385 sso_port_link_modify(struct otx2_ssogws *ws, uint8_t queue, uint8_t enable)
387 uintptr_t base = OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op);
391 val |= 0ULL << 12; /* SET 0 */
392 val |= 0x8000800080000000; /* Dont modify rest of the masks */
393 val |= (uint64_t)enable << 14; /* Enable/Disable Membership. */
395 otx2_write64(val, base + SSOW_LF_GWS_GRPMSK_CHG);
399 otx2_sso_port_link(struct rte_eventdev *event_dev, void *port,
400 const uint8_t queues[], const uint8_t priorities[],
403 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
407 RTE_SET_USED(priorities);
408 for (link = 0; link < nb_links; link++) {
410 struct otx2_ssogws_dual *ws = port;
413 sso_port_link_modify((struct otx2_ssogws *)
414 &ws->ws_state[0], queues[link], true);
415 sso_port_link_modify((struct otx2_ssogws *)
416 &ws->ws_state[1], queues[link], true);
418 struct otx2_ssogws *ws = port;
421 sso_port_link_modify(ws, queues[link], true);
424 sso_func_trace("Port=%d nb_links=%d", port_id, nb_links);
426 return (int)nb_links;
430 otx2_sso_port_unlink(struct rte_eventdev *event_dev, void *port,
431 uint8_t queues[], uint16_t nb_unlinks)
433 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
437 for (unlink = 0; unlink < nb_unlinks; unlink++) {
439 struct otx2_ssogws_dual *ws = port;
442 sso_port_link_modify((struct otx2_ssogws *)
443 &ws->ws_state[0], queues[unlink],
445 sso_port_link_modify((struct otx2_ssogws *)
446 &ws->ws_state[1], queues[unlink],
449 struct otx2_ssogws *ws = port;
452 sso_port_link_modify(ws, queues[unlink], false);
455 sso_func_trace("Port=%d nb_unlinks=%d", port_id, nb_unlinks);
457 return (int)nb_unlinks;
461 sso_hw_lf_cfg(struct otx2_mbox *mbox, enum otx2_sso_lf_type type,
462 uint16_t nb_lf, uint8_t attach)
465 struct rsrc_attach_req *req;
467 req = otx2_mbox_alloc_msg_attach_resources(mbox);
479 if (otx2_mbox_process(mbox) < 0)
482 struct rsrc_detach_req *req;
484 req = otx2_mbox_alloc_msg_detach_resources(mbox);
496 if (otx2_mbox_process(mbox) < 0)
504 sso_lf_cfg(struct otx2_sso_evdev *dev, struct otx2_mbox *mbox,
505 enum otx2_sso_lf_type type, uint16_t nb_lf, uint8_t alloc)
514 struct sso_lf_alloc_req *req_ggrp;
515 req_ggrp = otx2_mbox_alloc_msg_sso_lf_alloc(mbox);
516 req_ggrp->hwgrps = nb_lf;
521 struct ssow_lf_alloc_req *req_hws;
522 req_hws = otx2_mbox_alloc_msg_ssow_lf_alloc(mbox);
523 req_hws->hws = nb_lf;
533 struct sso_lf_free_req *req_ggrp;
534 req_ggrp = otx2_mbox_alloc_msg_sso_lf_free(mbox);
535 req_ggrp->hwgrps = nb_lf;
540 struct ssow_lf_free_req *req_hws;
541 req_hws = otx2_mbox_alloc_msg_ssow_lf_free(mbox);
542 req_hws->hws = nb_lf;
550 rc = otx2_mbox_process_msg_tmo(mbox, (void **)&rsp, ~0);
554 if (alloc && type == SSO_LF_GGRP) {
555 struct sso_lf_alloc_rsp *rsp_ggrp = rsp;
557 dev->xaq_buf_size = rsp_ggrp->xaq_buf_size;
558 dev->xae_waes = rsp_ggrp->xaq_wq_entries;
559 dev->iue = rsp_ggrp->in_unit_entries;
566 otx2_sso_port_release(void *port)
572 otx2_sso_queue_release(struct rte_eventdev *event_dev, uint8_t queue_id)
574 RTE_SET_USED(event_dev);
575 RTE_SET_USED(queue_id);
579 sso_clr_links(const struct rte_eventdev *event_dev)
581 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
584 for (i = 0; i < dev->nb_event_ports; i++) {
586 struct otx2_ssogws_dual *ws;
588 ws = event_dev->data->ports[i];
589 for (j = 0; j < dev->nb_event_queues; j++) {
590 sso_port_link_modify((struct otx2_ssogws *)
591 &ws->ws_state[0], j, false);
592 sso_port_link_modify((struct otx2_ssogws *)
593 &ws->ws_state[1], j, false);
596 struct otx2_ssogws *ws;
598 ws = event_dev->data->ports[i];
599 for (j = 0; j < dev->nb_event_queues; j++)
600 sso_port_link_modify(ws, j, false);
606 sso_set_port_ops(struct otx2_ssogws *ws, uintptr_t base)
608 ws->tag_op = base + SSOW_LF_GWS_TAG;
609 ws->wqp_op = base + SSOW_LF_GWS_WQP;
610 ws->getwrk_op = base + SSOW_LF_GWS_OP_GET_WORK;
611 ws->swtp_op = base + SSOW_LF_GWS_SWTP;
612 ws->swtag_norm_op = base + SSOW_LF_GWS_OP_SWTAG_NORM;
613 ws->swtag_desched_op = base + SSOW_LF_GWS_OP_SWTAG_DESCHED;
617 sso_configure_dual_ports(const struct rte_eventdev *event_dev)
619 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
620 struct otx2_mbox *mbox = dev->mbox;
625 otx2_sso_dbg("Configuring event ports %d", dev->nb_event_ports);
627 nb_lf = dev->nb_event_ports * 2;
628 /* Ask AF to attach required LFs. */
629 rc = sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, true);
631 otx2_err("Failed to attach SSO GWS LF");
635 if (sso_lf_cfg(dev, mbox, SSO_LF_GWS, nb_lf, true) < 0) {
636 sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, false);
637 otx2_err("Failed to init SSO GWS LF");
641 for (i = 0; i < dev->nb_event_ports; i++) {
642 struct otx2_ssogws_dual *ws;
645 /* Free memory prior to re-allocation if needed */
646 if (event_dev->data->ports[i] != NULL) {
647 ws = event_dev->data->ports[i];
652 /* Allocate event port memory */
653 ws = rte_zmalloc_socket("otx2_sso_ws",
654 sizeof(struct otx2_ssogws_dual),
656 event_dev->data->socket_id);
658 otx2_err("Failed to alloc memory for port=%d", i);
664 base = dev->bar2 + (RVU_BLOCK_ADDR_SSOW << 20 | vws << 12);
665 sso_set_port_ops((struct otx2_ssogws *)&ws->ws_state[0], base);
668 base = dev->bar2 + (RVU_BLOCK_ADDR_SSOW << 20 | vws << 12);
669 sso_set_port_ops((struct otx2_ssogws *)&ws->ws_state[1], base);
672 event_dev->data->ports[i] = ws;
676 sso_lf_cfg(dev, mbox, SSO_LF_GWS, nb_lf, false);
677 sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, false);
684 sso_configure_ports(const struct rte_eventdev *event_dev)
686 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
687 struct otx2_mbox *mbox = dev->mbox;
691 otx2_sso_dbg("Configuring event ports %d", dev->nb_event_ports);
693 nb_lf = dev->nb_event_ports;
694 /* Ask AF to attach required LFs. */
695 rc = sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, true);
697 otx2_err("Failed to attach SSO GWS LF");
701 if (sso_lf_cfg(dev, mbox, SSO_LF_GWS, nb_lf, true) < 0) {
702 sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, false);
703 otx2_err("Failed to init SSO GWS LF");
707 for (i = 0; i < nb_lf; i++) {
708 struct otx2_ssogws *ws;
711 /* Free memory prior to re-allocation if needed */
712 if (event_dev->data->ports[i] != NULL) {
713 ws = event_dev->data->ports[i];
718 /* Allocate event port memory */
719 ws = rte_zmalloc_socket("otx2_sso_ws",
720 sizeof(struct otx2_ssogws),
722 event_dev->data->socket_id);
724 otx2_err("Failed to alloc memory for port=%d", i);
730 base = dev->bar2 + (RVU_BLOCK_ADDR_SSOW << 20 | i << 12);
731 sso_set_port_ops(ws, base);
733 event_dev->data->ports[i] = ws;
737 sso_lf_cfg(dev, mbox, SSO_LF_GWS, nb_lf, false);
738 sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, false);
745 sso_configure_queues(const struct rte_eventdev *event_dev)
747 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
748 struct otx2_mbox *mbox = dev->mbox;
752 otx2_sso_dbg("Configuring event queues %d", dev->nb_event_queues);
754 nb_lf = dev->nb_event_queues;
755 /* Ask AF to attach required LFs. */
756 rc = sso_hw_lf_cfg(mbox, SSO_LF_GGRP, nb_lf, true);
758 otx2_err("Failed to attach SSO GGRP LF");
762 if (sso_lf_cfg(dev, mbox, SSO_LF_GGRP, nb_lf, true) < 0) {
763 sso_hw_lf_cfg(mbox, SSO_LF_GGRP, nb_lf, false);
764 otx2_err("Failed to init SSO GGRP LF");
772 sso_xaq_allocate(struct otx2_sso_evdev *dev)
774 const struct rte_memzone *mz;
775 struct npa_aura_s *aura;
776 static int reconfig_cnt;
777 char pool_name[RTE_MEMZONE_NAMESIZE];
782 rte_mempool_free(dev->xaq_pool);
785 * Allocate memory for Add work backpressure.
787 mz = rte_memzone_lookup(OTX2_SSO_FC_NAME);
789 mz = rte_memzone_reserve_aligned(OTX2_SSO_FC_NAME,
791 sizeof(struct npa_aura_s),
793 RTE_MEMZONE_IOVA_CONTIG,
796 otx2_err("Failed to allocate mem for fcmem");
800 dev->fc_iova = mz->iova;
801 dev->fc_mem = mz->addr;
803 aura = (struct npa_aura_s *)((uintptr_t)dev->fc_mem + OTX2_ALIGN);
804 memset(aura, 0, sizeof(struct npa_aura_s));
807 aura->fc_addr = dev->fc_iova;
808 aura->fc_hyst_bits = 0; /* Store count on all updates */
810 /* Taken from HRM 14.3.3(4) */
811 xaq_cnt = dev->nb_event_queues * OTX2_SSO_XAQ_CACHE_CNT;
813 xaq_cnt += dev->xae_cnt / dev->xae_waes;
814 else if (dev->adptr_xae_cnt)
815 xaq_cnt += (dev->adptr_xae_cnt / dev->xae_waes) +
816 (OTX2_SSO_XAQ_SLACK * dev->nb_event_queues);
818 xaq_cnt += (dev->iue / dev->xae_waes) +
819 (OTX2_SSO_XAQ_SLACK * dev->nb_event_queues);
821 otx2_sso_dbg("Configuring %d xaq buffers", xaq_cnt);
822 /* Setup XAQ based on number of nb queues. */
823 snprintf(pool_name, 30, "otx2_xaq_buf_pool_%d", reconfig_cnt);
824 dev->xaq_pool = (void *)rte_mempool_create_empty(pool_name,
825 xaq_cnt, dev->xaq_buf_size, 0, 0,
828 if (dev->xaq_pool == NULL) {
829 otx2_err("Unable to create empty mempool.");
830 rte_memzone_free(mz);
834 rc = rte_mempool_set_ops_byname(dev->xaq_pool,
835 rte_mbuf_platform_mempool_ops(), aura);
837 otx2_err("Unable to set xaqpool ops.");
841 rc = rte_mempool_populate_default(dev->xaq_pool);
843 otx2_err("Unable to set populate xaqpool.");
847 /* When SW does addwork (enqueue) check if there is space in XAQ by
848 * comparing fc_addr above against the xaq_lmt calculated below.
849 * There should be a minimum headroom (OTX2_SSO_XAQ_SLACK / 2) for SSO
850 * to request XAQ to cache them even before enqueue is called.
852 dev->xaq_lmt = xaq_cnt - (OTX2_SSO_XAQ_SLACK / 2 *
853 dev->nb_event_queues);
854 dev->nb_xaq_cfg = xaq_cnt;
858 rte_mempool_free(dev->xaq_pool);
859 rte_memzone_free(mz);
864 sso_ggrp_alloc_xaq(struct otx2_sso_evdev *dev)
866 struct otx2_mbox *mbox = dev->mbox;
867 struct sso_hw_setconfig *req;
869 otx2_sso_dbg("Configuring XAQ for GGRPs");
870 req = otx2_mbox_alloc_msg_sso_hw_setconfig(mbox);
871 req->npa_pf_func = otx2_npa_pf_func_get();
872 req->npa_aura_id = npa_lf_aura_handle_to_aura(dev->xaq_pool->pool_id);
873 req->hwgrps = dev->nb_event_queues;
875 return otx2_mbox_process(mbox);
879 sso_lf_teardown(struct otx2_sso_evdev *dev,
880 enum otx2_sso_lf_type lf_type)
886 nb_lf = dev->nb_event_queues;
889 nb_lf = dev->nb_event_ports;
890 nb_lf *= dev->dual_ws ? 2 : 1;
896 sso_lf_cfg(dev, dev->mbox, lf_type, nb_lf, false);
897 sso_hw_lf_cfg(dev->mbox, lf_type, nb_lf, false);
901 otx2_sso_configure(const struct rte_eventdev *event_dev)
903 struct rte_event_dev_config *conf = &event_dev->data->dev_conf;
904 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
909 deq_tmo_ns = conf->dequeue_timeout_ns;
912 deq_tmo_ns = dev->min_dequeue_timeout_ns;
914 if (deq_tmo_ns < dev->min_dequeue_timeout_ns ||
915 deq_tmo_ns > dev->max_dequeue_timeout_ns) {
916 otx2_err("Unsupported dequeue timeout requested");
920 if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
921 dev->is_timeout_deq = 1;
923 dev->deq_tmo_ns = deq_tmo_ns;
925 if (conf->nb_event_ports > dev->max_event_ports ||
926 conf->nb_event_queues > dev->max_event_queues) {
927 otx2_err("Unsupported event queues/ports requested");
931 if (conf->nb_event_port_dequeue_depth > 1) {
932 otx2_err("Unsupported event port deq depth requested");
936 if (conf->nb_event_port_enqueue_depth > 1) {
937 otx2_err("Unsupported event port enq depth requested");
942 sso_unregister_irqs(event_dev);
944 if (dev->nb_event_queues) {
945 /* Finit any previous queues. */
946 sso_lf_teardown(dev, SSO_LF_GGRP);
948 if (dev->nb_event_ports) {
949 /* Finit any previous ports. */
950 sso_lf_teardown(dev, SSO_LF_GWS);
953 dev->nb_event_queues = conf->nb_event_queues;
954 dev->nb_event_ports = conf->nb_event_ports;
957 rc = sso_configure_dual_ports(event_dev);
959 rc = sso_configure_ports(event_dev);
962 otx2_err("Failed to configure event ports");
966 if (sso_configure_queues(event_dev) < 0) {
967 otx2_err("Failed to configure event queues");
972 if (sso_xaq_allocate(dev) < 0) {
974 goto teardown_hwggrp;
977 /* Clear any prior port-queue mapping. */
978 sso_clr_links(event_dev);
979 rc = sso_ggrp_alloc_xaq(dev);
981 otx2_err("Failed to alloc xaq to ggrp %d", rc);
982 goto teardown_hwggrp;
985 rc = sso_get_msix_offsets(event_dev);
987 otx2_err("Failed to get msix offsets %d", rc);
988 goto teardown_hwggrp;
991 rc = sso_register_irqs(event_dev);
993 otx2_err("Failed to register irq %d", rc);
994 goto teardown_hwggrp;
1002 sso_lf_teardown(dev, SSO_LF_GGRP);
1004 sso_lf_teardown(dev, SSO_LF_GWS);
1005 dev->nb_event_queues = 0;
1006 dev->nb_event_ports = 0;
1007 dev->configured = 0;
1012 otx2_sso_queue_def_conf(struct rte_eventdev *event_dev, uint8_t queue_id,
1013 struct rte_event_queue_conf *queue_conf)
1015 RTE_SET_USED(event_dev);
1016 RTE_SET_USED(queue_id);
1018 queue_conf->nb_atomic_flows = (1ULL << 20);
1019 queue_conf->nb_atomic_order_sequences = (1ULL << 20);
1020 queue_conf->event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES;
1021 queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
1025 otx2_sso_queue_setup(struct rte_eventdev *event_dev, uint8_t queue_id,
1026 const struct rte_event_queue_conf *queue_conf)
1028 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1029 struct otx2_mbox *mbox = dev->mbox;
1030 struct sso_grp_priority *req;
1033 sso_func_trace("Queue=%d prio=%d", queue_id, queue_conf->priority);
1035 req = otx2_mbox_alloc_msg_sso_grp_set_priority(dev->mbox);
1036 req->grp = queue_id;
1038 req->affinity = 0xFF;
1039 /* Normalize <0-255> to <0-7> */
1040 req->priority = queue_conf->priority / 32;
1042 rc = otx2_mbox_process(mbox);
1044 otx2_err("Failed to set priority queue=%d", queue_id);
1052 otx2_sso_port_def_conf(struct rte_eventdev *event_dev, uint8_t port_id,
1053 struct rte_event_port_conf *port_conf)
1055 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1057 RTE_SET_USED(port_id);
1058 port_conf->new_event_threshold = dev->max_num_events;
1059 port_conf->dequeue_depth = 1;
1060 port_conf->enqueue_depth = 1;
1064 otx2_sso_port_setup(struct rte_eventdev *event_dev, uint8_t port_id,
1065 const struct rte_event_port_conf *port_conf)
1067 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1068 uintptr_t grps_base[OTX2_SSO_MAX_VHGRP] = {0};
1072 sso_func_trace("Port=%d", port_id);
1073 RTE_SET_USED(port_conf);
1075 if (event_dev->data->ports[port_id] == NULL) {
1076 otx2_err("Invalid port Id %d", port_id);
1080 for (q = 0; q < dev->nb_event_queues; q++) {
1081 grps_base[q] = dev->bar2 + (RVU_BLOCK_ADDR_SSO << 20 | q << 12);
1082 if (grps_base[q] == 0) {
1083 otx2_err("Failed to get grp[%d] base addr", q);
1088 /* Set get_work timeout for HWS */
1089 val = NSEC2USEC(dev->deq_tmo_ns) - 1;
1092 struct otx2_ssogws_dual *ws = event_dev->data->ports[port_id];
1094 rte_memcpy(ws->grps_base, grps_base,
1095 sizeof(uintptr_t) * OTX2_SSO_MAX_VHGRP);
1096 ws->fc_mem = dev->fc_mem;
1097 ws->xaq_lmt = dev->xaq_lmt;
1098 ws->tstamp = dev->tstamp;
1099 otx2_write64(val, OTX2_SSOW_GET_BASE_ADDR(
1100 ws->ws_state[0].getwrk_op) + SSOW_LF_GWS_NW_TIM);
1101 otx2_write64(val, OTX2_SSOW_GET_BASE_ADDR(
1102 ws->ws_state[1].getwrk_op) + SSOW_LF_GWS_NW_TIM);
1104 struct otx2_ssogws *ws = event_dev->data->ports[port_id];
1105 uintptr_t base = OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op);
1107 rte_memcpy(ws->grps_base, grps_base,
1108 sizeof(uintptr_t) * OTX2_SSO_MAX_VHGRP);
1109 ws->fc_mem = dev->fc_mem;
1110 ws->xaq_lmt = dev->xaq_lmt;
1111 ws->tstamp = dev->tstamp;
1112 otx2_write64(val, base + SSOW_LF_GWS_NW_TIM);
1115 otx2_sso_dbg("Port=%d ws=%p", port_id, event_dev->data->ports[port_id]);
1121 otx2_sso_timeout_ticks(struct rte_eventdev *event_dev, uint64_t ns,
1122 uint64_t *tmo_ticks)
1124 RTE_SET_USED(event_dev);
1125 *tmo_ticks = NSEC2TICK(ns, rte_get_timer_hz());
1131 ssogws_dump(struct otx2_ssogws *ws, FILE *f)
1133 uintptr_t base = OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op);
1135 fprintf(f, "SSOW_LF_GWS Base addr 0x%" PRIx64 "\n", (uint64_t)base);
1136 fprintf(f, "SSOW_LF_GWS_LINKS 0x%" PRIx64 "\n",
1137 otx2_read64(base + SSOW_LF_GWS_LINKS));
1138 fprintf(f, "SSOW_LF_GWS_PENDWQP 0x%" PRIx64 "\n",
1139 otx2_read64(base + SSOW_LF_GWS_PENDWQP));
1140 fprintf(f, "SSOW_LF_GWS_PENDSTATE 0x%" PRIx64 "\n",
1141 otx2_read64(base + SSOW_LF_GWS_PENDSTATE));
1142 fprintf(f, "SSOW_LF_GWS_NW_TIM 0x%" PRIx64 "\n",
1143 otx2_read64(base + SSOW_LF_GWS_NW_TIM));
1144 fprintf(f, "SSOW_LF_GWS_TAG 0x%" PRIx64 "\n",
1145 otx2_read64(base + SSOW_LF_GWS_TAG));
1146 fprintf(f, "SSOW_LF_GWS_WQP 0x%" PRIx64 "\n",
1147 otx2_read64(base + SSOW_LF_GWS_TAG));
1148 fprintf(f, "SSOW_LF_GWS_SWTP 0x%" PRIx64 "\n",
1149 otx2_read64(base + SSOW_LF_GWS_SWTP));
1150 fprintf(f, "SSOW_LF_GWS_PENDTAG 0x%" PRIx64 "\n",
1151 otx2_read64(base + SSOW_LF_GWS_PENDTAG));
1155 ssoggrp_dump(uintptr_t base, FILE *f)
1157 fprintf(f, "SSO_LF_GGRP Base addr 0x%" PRIx64 "\n", (uint64_t)base);
1158 fprintf(f, "SSO_LF_GGRP_QCTL 0x%" PRIx64 "\n",
1159 otx2_read64(base + SSO_LF_GGRP_QCTL));
1160 fprintf(f, "SSO_LF_GGRP_XAQ_CNT 0x%" PRIx64 "\n",
1161 otx2_read64(base + SSO_LF_GGRP_XAQ_CNT));
1162 fprintf(f, "SSO_LF_GGRP_INT_THR 0x%" PRIx64 "\n",
1163 otx2_read64(base + SSO_LF_GGRP_INT_THR));
1164 fprintf(f, "SSO_LF_GGRP_INT_CNT 0x%" PRIX64 "\n",
1165 otx2_read64(base + SSO_LF_GGRP_INT_CNT));
1166 fprintf(f, "SSO_LF_GGRP_AQ_CNT 0x%" PRIX64 "\n",
1167 otx2_read64(base + SSO_LF_GGRP_AQ_CNT));
1168 fprintf(f, "SSO_LF_GGRP_AQ_THR 0x%" PRIX64 "\n",
1169 otx2_read64(base + SSO_LF_GGRP_AQ_THR));
1170 fprintf(f, "SSO_LF_GGRP_MISC_CNT 0x%" PRIx64 "\n",
1171 otx2_read64(base + SSO_LF_GGRP_MISC_CNT));
1175 otx2_sso_dump(struct rte_eventdev *event_dev, FILE *f)
1177 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1181 fprintf(f, "[%s] SSO running in [%s] mode\n", __func__, dev->dual_ws ?
1182 "dual_ws" : "single_ws");
1183 /* Dump SSOW registers */
1184 for (port = 0; port < dev->nb_event_ports; port++) {
1186 struct otx2_ssogws_dual *ws =
1187 event_dev->data->ports[port];
1189 fprintf(f, "[%s] SSO dual workslot[%d] vws[%d] dump\n",
1191 ssogws_dump((struct otx2_ssogws *)&ws->ws_state[0], f);
1192 fprintf(f, "[%s]SSO dual workslot[%d] vws[%d] dump\n",
1194 ssogws_dump((struct otx2_ssogws *)&ws->ws_state[1], f);
1196 fprintf(f, "[%s]SSO single workslot[%d] dump\n",
1198 ssogws_dump(event_dev->data->ports[port], f);
1202 /* Dump SSO registers */
1203 for (queue = 0; queue < dev->nb_event_queues; queue++) {
1204 fprintf(f, "[%s]SSO group[%d] dump\n", __func__, queue);
1206 struct otx2_ssogws_dual *ws = event_dev->data->ports[0];
1207 ssoggrp_dump(ws->grps_base[queue], f);
1209 struct otx2_ssogws *ws = event_dev->data->ports[0];
1210 ssoggrp_dump(ws->grps_base[queue], f);
1216 otx2_handle_event(void *arg, struct rte_event event)
1218 struct rte_eventdev *event_dev = arg;
1220 if (event_dev->dev_ops->dev_stop_flush != NULL)
1221 event_dev->dev_ops->dev_stop_flush(event_dev->data->dev_id,
1222 event, event_dev->data->dev_stop_flush_arg);
1226 sso_qos_cfg(struct rte_eventdev *event_dev)
1228 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1229 struct sso_grp_qos_cfg *req;
1232 for (i = 0; i < dev->qos_queue_cnt; i++) {
1233 uint8_t xaq_prcnt = dev->qos_parse_data[i].xaq_prcnt;
1234 uint8_t iaq_prcnt = dev->qos_parse_data[i].iaq_prcnt;
1235 uint8_t taq_prcnt = dev->qos_parse_data[i].taq_prcnt;
1237 if (dev->qos_parse_data[i].queue >= dev->nb_event_queues)
1240 req = otx2_mbox_alloc_msg_sso_grp_qos_config(dev->mbox);
1241 req->xaq_limit = (dev->nb_xaq_cfg *
1242 (xaq_prcnt ? xaq_prcnt : 100)) / 100;
1243 req->taq_thr = (SSO_HWGRP_IAQ_MAX_THR_MASK *
1244 (iaq_prcnt ? iaq_prcnt : 100)) / 100;
1245 req->iaq_thr = (SSO_HWGRP_TAQ_MAX_THR_MASK *
1246 (taq_prcnt ? taq_prcnt : 100)) / 100;
1249 if (dev->qos_queue_cnt)
1250 otx2_mbox_process(dev->mbox);
1254 sso_cleanup(struct rte_eventdev *event_dev, uint8_t enable)
1256 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1259 for (i = 0; i < dev->nb_event_ports; i++) {
1261 struct otx2_ssogws_dual *ws;
1263 ws = event_dev->data->ports[i];
1264 ssogws_reset((struct otx2_ssogws *)&ws->ws_state[0]);
1265 ssogws_reset((struct otx2_ssogws *)&ws->ws_state[1]);
1268 ws->ws_state[0].cur_grp = 0;
1269 ws->ws_state[0].cur_tt = SSO_SYNC_EMPTY;
1270 ws->ws_state[1].cur_grp = 0;
1271 ws->ws_state[1].cur_tt = SSO_SYNC_EMPTY;
1273 struct otx2_ssogws *ws;
1275 ws = event_dev->data->ports[i];
1279 ws->cur_tt = SSO_SYNC_EMPTY;
1285 struct otx2_ssogws_dual *ws = event_dev->data->ports[0];
1286 struct otx2_ssogws temp_ws;
1288 memcpy(&temp_ws, &ws->ws_state[0],
1289 sizeof(struct otx2_ssogws_state));
1290 for (i = 0; i < dev->nb_event_queues; i++) {
1291 /* Consume all the events through HWS0 */
1292 ssogws_flush_events(&temp_ws, i, ws->grps_base[i],
1293 otx2_handle_event, event_dev);
1294 /* Enable/Disable SSO GGRP */
1295 otx2_write64(enable, ws->grps_base[i] +
1298 ws->ws_state[0].cur_grp = 0;
1299 ws->ws_state[0].cur_tt = SSO_SYNC_EMPTY;
1301 struct otx2_ssogws *ws = event_dev->data->ports[0];
1303 for (i = 0; i < dev->nb_event_queues; i++) {
1304 /* Consume all the events through HWS0 */
1305 ssogws_flush_events(ws, i, ws->grps_base[i],
1306 otx2_handle_event, event_dev);
1307 /* Enable/Disable SSO GGRP */
1308 otx2_write64(enable, ws->grps_base[i] +
1312 ws->cur_tt = SSO_SYNC_EMPTY;
1315 /* reset SSO GWS cache */
1316 otx2_mbox_alloc_msg_sso_ws_cache_inv(dev->mbox);
1317 otx2_mbox_process(dev->mbox);
1321 sso_xae_reconfigure(struct rte_eventdev *event_dev)
1323 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1324 struct rte_mempool *prev_xaq_pool;
1327 if (event_dev->data->dev_started)
1328 sso_cleanup(event_dev, 0);
1330 prev_xaq_pool = dev->xaq_pool;
1331 dev->xaq_pool = NULL;
1332 sso_xaq_allocate(dev);
1333 rc = sso_ggrp_alloc_xaq(dev);
1335 otx2_err("Failed to alloc xaq to ggrp %d", rc);
1336 rte_mempool_free(prev_xaq_pool);
1340 rte_mempool_free(prev_xaq_pool);
1342 if (event_dev->data->dev_started)
1343 sso_cleanup(event_dev, 1);
1349 otx2_sso_start(struct rte_eventdev *event_dev)
1352 sso_qos_cfg(event_dev);
1353 sso_cleanup(event_dev, 1);
1354 sso_fastpath_fns_set(event_dev);
1360 otx2_sso_stop(struct rte_eventdev *event_dev)
1363 sso_cleanup(event_dev, 0);
1368 otx2_sso_close(struct rte_eventdev *event_dev)
1370 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1371 uint8_t all_queues[RTE_EVENT_MAX_QUEUES_PER_DEV];
1374 if (!dev->configured)
1377 sso_unregister_irqs(event_dev);
1379 for (i = 0; i < dev->nb_event_queues; i++)
1382 for (i = 0; i < dev->nb_event_ports; i++)
1383 otx2_sso_port_unlink(event_dev, event_dev->data->ports[i],
1384 all_queues, dev->nb_event_queues);
1386 sso_lf_teardown(dev, SSO_LF_GGRP);
1387 sso_lf_teardown(dev, SSO_LF_GWS);
1388 dev->nb_event_ports = 0;
1389 dev->nb_event_queues = 0;
1390 rte_mempool_free(dev->xaq_pool);
1391 rte_memzone_free(rte_memzone_lookup(OTX2_SSO_FC_NAME));
1396 /* Initialize and register event driver with DPDK Application */
1397 static struct rte_eventdev_ops otx2_sso_ops = {
1398 .dev_infos_get = otx2_sso_info_get,
1399 .dev_configure = otx2_sso_configure,
1400 .queue_def_conf = otx2_sso_queue_def_conf,
1401 .queue_setup = otx2_sso_queue_setup,
1402 .queue_release = otx2_sso_queue_release,
1403 .port_def_conf = otx2_sso_port_def_conf,
1404 .port_setup = otx2_sso_port_setup,
1405 .port_release = otx2_sso_port_release,
1406 .port_link = otx2_sso_port_link,
1407 .port_unlink = otx2_sso_port_unlink,
1408 .timeout_ticks = otx2_sso_timeout_ticks,
1410 .eth_rx_adapter_caps_get = otx2_sso_rx_adapter_caps_get,
1411 .eth_rx_adapter_queue_add = otx2_sso_rx_adapter_queue_add,
1412 .eth_rx_adapter_queue_del = otx2_sso_rx_adapter_queue_del,
1413 .eth_rx_adapter_start = otx2_sso_rx_adapter_start,
1414 .eth_rx_adapter_stop = otx2_sso_rx_adapter_stop,
1416 .timer_adapter_caps_get = otx2_tim_caps_get,
1418 .xstats_get = otx2_sso_xstats_get,
1419 .xstats_reset = otx2_sso_xstats_reset,
1420 .xstats_get_names = otx2_sso_xstats_get_names,
1422 .dump = otx2_sso_dump,
1423 .dev_start = otx2_sso_start,
1424 .dev_stop = otx2_sso_stop,
1425 .dev_close = otx2_sso_close,
1426 .dev_selftest = otx2_sso_selftest,
1429 #define OTX2_SSO_XAE_CNT "xae_cnt"
1430 #define OTX2_SSO_SINGLE_WS "single_ws"
1431 #define OTX2_SSO_GGRP_QOS "qos"
1432 #define OTX2_SSO_SELFTEST "selftest"
1435 parse_queue_param(char *value, void *opaque)
1437 struct otx2_sso_qos queue_qos = {0};
1438 uint8_t *val = (uint8_t *)&queue_qos;
1439 struct otx2_sso_evdev *dev = opaque;
1440 char *tok = strtok(value, "-");
1445 while (tok != NULL) {
1447 tok = strtok(NULL, "-");
1451 if (val != (&queue_qos.iaq_prcnt + 1)) {
1452 otx2_err("Invalid QoS parameter expected [Qx-XAQ-TAQ-IAQ]");
1456 dev->qos_queue_cnt++;
1457 dev->qos_parse_data = rte_realloc(dev->qos_parse_data,
1458 sizeof(struct otx2_sso_qos) *
1459 dev->qos_queue_cnt, 0);
1460 dev->qos_parse_data[dev->qos_queue_cnt - 1] = queue_qos;
1464 parse_qos_list(const char *value, void *opaque)
1466 char *s = strdup(value);
1477 if (start < end && *start) {
1479 parse_queue_param(start + 1, opaque);
1490 parse_sso_kvargs_dict(const char *key, const char *value, void *opaque)
1494 /* Dict format [Qx-XAQ-TAQ-IAQ][Qz-XAQ-TAQ-IAQ] use '-' cause ','
1495 * isn't allowed. Everything is expressed in percentages, 0 represents
1498 parse_qos_list(value, opaque);
1504 sso_parse_devargs(struct otx2_sso_evdev *dev, struct rte_devargs *devargs)
1506 struct rte_kvargs *kvlist;
1507 uint8_t single_ws = 0;
1509 if (devargs == NULL)
1511 kvlist = rte_kvargs_parse(devargs->args, NULL);
1515 rte_kvargs_process(kvlist, OTX2_SSO_SELFTEST, &parse_kvargs_flag,
1517 rte_kvargs_process(kvlist, OTX2_SSO_XAE_CNT, &parse_kvargs_value,
1519 rte_kvargs_process(kvlist, OTX2_SSO_SINGLE_WS, &parse_kvargs_flag,
1521 rte_kvargs_process(kvlist, OTX2_SSO_GGRP_QOS, &parse_sso_kvargs_dict,
1524 dev->dual_ws = !single_ws;
1525 rte_kvargs_free(kvlist);
1529 otx2_sso_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
1531 return rte_event_pmd_pci_probe(pci_drv, pci_dev,
1532 sizeof(struct otx2_sso_evdev),
1537 otx2_sso_remove(struct rte_pci_device *pci_dev)
1539 return rte_event_pmd_pci_remove(pci_dev, otx2_sso_fini);
1542 static const struct rte_pci_id pci_sso_map[] = {
1544 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
1545 PCI_DEVID_OCTEONTX2_RVU_SSO_TIM_PF)
1552 static struct rte_pci_driver pci_sso = {
1553 .id_table = pci_sso_map,
1554 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA,
1555 .probe = otx2_sso_probe,
1556 .remove = otx2_sso_remove,
1560 otx2_sso_init(struct rte_eventdev *event_dev)
1562 struct free_rsrcs_rsp *rsrc_cnt;
1563 struct rte_pci_device *pci_dev;
1564 struct otx2_sso_evdev *dev;
1567 event_dev->dev_ops = &otx2_sso_ops;
1568 /* For secondary processes, the primary has done all the work */
1569 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1570 sso_fastpath_fns_set(event_dev);
1574 dev = sso_pmd_priv(event_dev);
1576 pci_dev = container_of(event_dev->dev, struct rte_pci_device, device);
1578 /* Initialize the base otx2_dev object */
1579 rc = otx2_dev_init(pci_dev, dev);
1581 otx2_err("Failed to initialize otx2_dev rc=%d", rc);
1585 /* Get SSO and SSOW MSIX rsrc cnt */
1586 otx2_mbox_alloc_msg_free_rsrc_cnt(dev->mbox);
1587 rc = otx2_mbox_process_msg(dev->mbox, (void *)&rsrc_cnt);
1589 otx2_err("Unable to get free rsrc count");
1590 goto otx2_dev_uninit;
1592 otx2_sso_dbg("SSO %d SSOW %d NPA %d provisioned", rsrc_cnt->sso,
1593 rsrc_cnt->ssow, rsrc_cnt->npa);
1595 dev->max_event_ports = RTE_MIN(rsrc_cnt->ssow, OTX2_SSO_MAX_VHWS);
1596 dev->max_event_queues = RTE_MIN(rsrc_cnt->sso, OTX2_SSO_MAX_VHGRP);
1597 /* Grab the NPA LF if required */
1598 rc = otx2_npa_lf_init(pci_dev, dev);
1600 otx2_err("Unable to init NPA lf. It might not be provisioned");
1601 goto otx2_dev_uninit;
1604 dev->drv_inited = true;
1605 dev->is_timeout_deq = 0;
1606 dev->min_dequeue_timeout_ns = USEC2NSEC(1);
1607 dev->max_dequeue_timeout_ns = USEC2NSEC(0x3FF);
1608 dev->max_num_events = -1;
1609 dev->nb_event_queues = 0;
1610 dev->nb_event_ports = 0;
1612 if (!dev->max_event_ports || !dev->max_event_queues) {
1613 otx2_err("Not enough eventdev resource queues=%d ports=%d",
1614 dev->max_event_queues, dev->max_event_ports);
1616 goto otx2_npa_lf_uninit;
1620 sso_parse_devargs(dev, pci_dev->device.devargs);
1622 otx2_sso_dbg("Using dual workslot mode");
1623 dev->max_event_ports = dev->max_event_ports / 2;
1625 otx2_sso_dbg("Using single workslot mode");
1628 otx2_sso_pf_func_set(dev->pf_func);
1629 otx2_sso_dbg("Initializing %s max_queues=%d max_ports=%d",
1630 event_dev->data->name, dev->max_event_queues,
1631 dev->max_event_ports);
1632 if (dev->selftest) {
1633 event_dev->dev->driver = &pci_sso.driver;
1634 event_dev->dev_ops->dev_selftest();
1637 otx2_tim_init(pci_dev, (struct otx2_dev *)dev);
1644 otx2_dev_fini(pci_dev, dev);
1650 otx2_sso_fini(struct rte_eventdev *event_dev)
1652 struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1653 struct rte_pci_device *pci_dev;
1655 /* For secondary processes, nothing to be done */
1656 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1659 pci_dev = container_of(event_dev->dev, struct rte_pci_device, device);
1661 if (!dev->drv_inited)
1664 dev->drv_inited = false;
1668 if (otx2_npa_lf_active(dev)) {
1669 otx2_info("Common resource in use by other devices");
1674 otx2_dev_fini(pci_dev, dev);
1679 RTE_PMD_REGISTER_PCI(event_octeontx2, pci_sso);
1680 RTE_PMD_REGISTER_PCI_TABLE(event_octeontx2, pci_sso_map);
1681 RTE_PMD_REGISTER_KMOD_DEP(event_octeontx2, "vfio-pci");
1682 RTE_PMD_REGISTER_PARAM_STRING(event_octeontx2, OTX2_SSO_XAE_CNT "=<int>"
1683 OTX2_SSO_SINGLE_WS "=1"
1684 OTX2_SSO_GGRP_QOS "=<string>"
1685 OTX2_SSO_SELFTEST "=1");