otx2_ssogws_swtag_flush(ws);
}
+#define R(name, f5, f4, f3, f2, f1, f0, flags) \
+uint16_t __hot \
+otx2_ssogws_deq_ ##name(void *port, struct rte_event *ev, \
+ uint64_t timeout_ticks) \
+{ \
+ struct otx2_ssogws *ws = port; \
+ \
+ RTE_SET_USED(timeout_ticks); \
+ \
+ if (ws->swtag_req) { \
+ ws->swtag_req = 0; \
+ otx2_ssogws_swtag_wait(ws); \
+ return 1; \
+ } \
+ \
+ return otx2_ssogws_get_work(ws, ev, flags, ws->lookup_mem); \
+} \
+ \
+uint16_t __hot \
+otx2_ssogws_deq_burst_ ##name(void *port, struct rte_event ev[], \
+ uint16_t nb_events, \
+ uint64_t timeout_ticks) \
+{ \
+ RTE_SET_USED(nb_events); \
+ \
+ return otx2_ssogws_deq_ ##name(port, ev, timeout_ticks); \
+} \
+ \
+uint16_t __hot \
+otx2_ssogws_deq_timeout_ ##name(void *port, struct rte_event *ev, \
+ uint64_t timeout_ticks) \
+{ \
+ struct otx2_ssogws *ws = port; \
+ uint16_t ret = 1; \
+ uint64_t iter; \
+ \
+ if (ws->swtag_req) { \
+ ws->swtag_req = 0; \
+ otx2_ssogws_swtag_wait(ws); \
+ return ret; \
+ } \
+ \
+ ret = otx2_ssogws_get_work(ws, ev, flags, ws->lookup_mem); \
+ for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \
+ ret = otx2_ssogws_get_work(ws, ev, flags, \
+ ws->lookup_mem); \
+ \
+ return ret; \
+} \
+ \
+uint16_t __hot \
+otx2_ssogws_deq_timeout_burst_ ##name(void *port, struct rte_event ev[],\
+ uint16_t nb_events, \
+ uint64_t timeout_ticks) \
+{ \
+ RTE_SET_USED(nb_events); \
+ \
+ return otx2_ssogws_deq_timeout_ ##name(port, ev, timeout_ticks);\
+} \
+ \
+uint16_t __hot \
+otx2_ssogws_deq_seg_ ##name(void *port, struct rte_event *ev, \
+ uint64_t timeout_ticks) \
+{ \
+ struct otx2_ssogws *ws = port; \
+ \
+ RTE_SET_USED(timeout_ticks); \
+ \
+ if (ws->swtag_req) { \
+ ws->swtag_req = 0; \
+ otx2_ssogws_swtag_wait(ws); \
+ return 1; \
+ } \
+ \
+ return otx2_ssogws_get_work(ws, ev, flags | NIX_RX_MULTI_SEG_F, \
+ ws->lookup_mem); \
+} \
+ \
+uint16_t __hot \
+otx2_ssogws_deq_seg_burst_ ##name(void *port, struct rte_event ev[], \
+ uint16_t nb_events, \
+ uint64_t timeout_ticks) \
+{ \
+ RTE_SET_USED(nb_events); \
+ \
+ return otx2_ssogws_deq_seg_ ##name(port, ev, timeout_ticks); \
+} \
+ \
+uint16_t __hot \
+otx2_ssogws_deq_seg_timeout_ ##name(void *port, struct rte_event *ev, \
+ uint64_t timeout_ticks) \
+{ \
+ struct otx2_ssogws *ws = port; \
+ uint16_t ret = 1; \
+ uint64_t iter; \
+ \
+ if (ws->swtag_req) { \
+ ws->swtag_req = 0; \
+ otx2_ssogws_swtag_wait(ws); \
+ return ret; \
+ } \
+ \
+ ret = otx2_ssogws_get_work(ws, ev, flags | NIX_RX_MULTI_SEG_F, \
+ ws->lookup_mem); \
+ for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \
+ ret = otx2_ssogws_get_work(ws, ev, \
+ flags | NIX_RX_MULTI_SEG_F, \
+ ws->lookup_mem); \
+ \
+ return ret; \
+} \
+ \
+uint16_t __hot \
+otx2_ssogws_deq_seg_timeout_burst_ ##name(void *port, \
+ struct rte_event ev[], \
+ uint16_t nb_events, \
+ uint64_t timeout_ticks) \
+{ \
+ RTE_SET_USED(nb_events); \
+ \
+ return otx2_ssogws_deq_seg_timeout_ ##name(port, ev, \
+ timeout_ticks); \
+}
+
+SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
+#undef R
+
uint16_t __hot
otx2_ssogws_enq(void *port, const struct rte_event *ev)
{
return 1;
}
+
+#define T(name, f4, f3, f2, f1, f0, sz, flags) \
+uint16_t __hot \
+otx2_ssogws_tx_adptr_enq_ ## name(void *port, struct rte_event ev[], \
+ uint16_t nb_events) \
+{ \
+ struct otx2_ssogws *ws = port; \
+ uint64_t cmd[sz]; \
+ \
+ RTE_SET_USED(nb_events); \
+ return otx2_ssogws_event_tx(ws, ev, cmd, flags); \
+}
+SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
+#undef T
+
+#define T(name, f4, f3, f2, f1, f0, sz, flags) \
+uint16_t __hot \
+otx2_ssogws_tx_adptr_enq_seg_ ## name(void *port, struct rte_event ev[],\
+ uint16_t nb_events) \
+{ \
+ struct otx2_ssogws *ws = port; \
+ uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2]; \
+ \
+ RTE_SET_USED(nb_events); \
+ return otx2_ssogws_event_tx(ws, ev, cmd, (flags) | \
+ NIX_TX_MULTI_SEG_F); \
+}
+SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
+#undef T
+
+void
+ssogws_flush_events(struct otx2_ssogws *ws, uint8_t queue_id, uintptr_t base,
+ otx2_handle_event_t fn, void *arg)
+{
+ uint64_t cq_ds_cnt = 1;
+ uint64_t aq_cnt = 1;
+ uint64_t ds_cnt = 1;
+ struct rte_event ev;
+ uint64_t enable;
+ uint64_t val;
+
+ enable = otx2_read64(base + SSO_LF_GGRP_QCTL);
+ if (!enable)
+ return;
+
+ val = queue_id; /* GGRP ID */
+ val |= BIT_ULL(18); /* Grouped */
+ val |= BIT_ULL(16); /* WAIT */
+
+ aq_cnt = otx2_read64(base + SSO_LF_GGRP_AQ_CNT);
+ ds_cnt = otx2_read64(base + SSO_LF_GGRP_MISC_CNT);
+ cq_ds_cnt = otx2_read64(base + SSO_LF_GGRP_INT_CNT);
+ cq_ds_cnt &= 0x3FFF3FFF0000;
+
+ while (aq_cnt || cq_ds_cnt || ds_cnt) {
+ otx2_write64(val, ws->getwrk_op);
+ otx2_ssogws_get_work_empty(ws, &ev, 0);
+ if (fn != NULL && ev.u64 != 0)
+ fn(arg, ev);
+ if (ev.sched_type != SSO_TT_EMPTY)
+ otx2_ssogws_swtag_flush(ws);
+ rte_mb();
+ aq_cnt = otx2_read64(base + SSO_LF_GGRP_AQ_CNT);
+ ds_cnt = otx2_read64(base + SSO_LF_GGRP_MISC_CNT);
+ cq_ds_cnt = otx2_read64(base + SSO_LF_GGRP_INT_CNT);
+ /* Extract cq and ds count */
+ cq_ds_cnt &= 0x3FFF3FFF0000;
+ }
+
+ otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
+ SSOW_LF_GWS_OP_GWC_INVAL);
+ rte_mb();
+}
+
+void
+ssogws_reset(struct otx2_ssogws *ws)
+{
+ uintptr_t base = OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op);
+ uint64_t pend_state;
+ uint8_t pend_tt;
+ uint64_t tag;
+
+ /* Wait till getwork/swtp/waitw/desched completes. */
+ do {
+ pend_state = otx2_read64(base + SSOW_LF_GWS_PENDSTATE);
+ rte_mb();
+ } while (pend_state & (BIT_ULL(63) | BIT_ULL(62) | BIT_ULL(58)));
+
+ tag = otx2_read64(base + SSOW_LF_GWS_TAG);
+ pend_tt = (tag >> 32) & 0x3;
+ if (pend_tt != SSO_TT_EMPTY) { /* Work was pending */
+ if (pend_tt == SSO_SYNC_ATOMIC || pend_tt == SSO_SYNC_ORDERED)
+ otx2_ssogws_swtag_untag(ws);
+ otx2_ssogws_desched(ws);
+ }
+ rte_mb();
+
+ /* Wait for desched to complete. */
+ do {
+ pend_state = otx2_read64(base + SSOW_LF_GWS_PENDSTATE);
+ rte_mb();
+ } while (pend_state & BIT_ULL(58));
+}