+static int
+cn9k_sso_updt_tx_adptr_data(const struct rte_eventdev *event_dev)
+{
+ struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
+ int i;
+
+ if (dev->tx_adptr_data == NULL)
+ return 0;
+
+ for (i = 0; i < dev->nb_event_ports; i++) {
+ if (dev->dual_ws) {
+ struct cn9k_sso_hws_dual *dws =
+ event_dev->data->ports[i];
+ void *ws_cookie;
+
+ ws_cookie = cnxk_sso_hws_get_cookie(dws);
+ ws_cookie = rte_realloc_socket(
+ ws_cookie,
+ sizeof(struct cnxk_sso_hws_cookie) +
+ sizeof(struct cn9k_sso_hws_dual) +
+ dev->tx_adptr_data_sz,
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+ if (ws_cookie == NULL)
+ return -ENOMEM;
+ dws = RTE_PTR_ADD(ws_cookie,
+ sizeof(struct cnxk_sso_hws_cookie));
+ memcpy(&dws->tx_adptr_data, dev->tx_adptr_data,
+ dev->tx_adptr_data_sz);
+ event_dev->data->ports[i] = dws;
+ } else {
+ struct cn9k_sso_hws *ws = event_dev->data->ports[i];
+ void *ws_cookie;
+
+ ws_cookie = cnxk_sso_hws_get_cookie(ws);
+ ws_cookie = rte_realloc_socket(
+ ws_cookie,
+ sizeof(struct cnxk_sso_hws_cookie) +
+ sizeof(struct cn9k_sso_hws_dual) +
+ dev->tx_adptr_data_sz,
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+ if (ws_cookie == NULL)
+ return -ENOMEM;
+ ws = RTE_PTR_ADD(ws_cookie,
+ sizeof(struct cnxk_sso_hws_cookie));
+ memcpy(&ws->tx_adptr_data, dev->tx_adptr_data,
+ dev->tx_adptr_data_sz);
+ event_dev->data->ports[i] = ws;
+ }
+ }
+ rte_mb();
+
+ return 0;
+}
+
+static void
+cn9k_sso_fp_fns_set(struct rte_eventdev *event_dev)
+{
+ struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
+ /* Single WS modes */
+ const event_dequeue_t sso_hws_deq[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_deq_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_tmo[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_tmo_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_deq_tmo_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_tmo_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_ca_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_deq_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_ca_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_tmo_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_tmo_ca_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_deq_tmo_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_tmo_ca_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_deq_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_tmo_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_tmo_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_deq_tmo_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_tmo_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_ca_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_deq_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_ca_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_tmo_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_tmo_ca_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_deq_tmo_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_deq_tmo_ca_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ /* Dual WS modes */
+ const event_dequeue_t sso_hws_dual_deq[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_dual_deq_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_tmo[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_tmo_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_dual_deq_tmo_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_tmo_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_ca_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_dual_deq_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_ca_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_tmo_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_tmo_ca_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_dual_deq_tmo_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_tmo_ca_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_dual_deq_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_tmo_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_tmo_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_dual_deq_tmo_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_tmo_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_ca_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_dual_deq_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_ca_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_tmo_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_tmo_ca_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t sso_hws_dual_deq_tmo_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn9k_sso_hws_dual_deq_tmo_ca_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ /* Tx modes */
+ const event_tx_adapter_enqueue_t sso_hws_tx_adptr_enq[NIX_TX_OFFLOAD_MAX] = {
+#define T(name, sz, flags)[flags] = cn9k_sso_hws_tx_adptr_enq_##name,
+ NIX_TX_FASTPATH_MODES
+#undef T
+ };
+
+ const event_tx_adapter_enqueue_t sso_hws_tx_adptr_enq_seg[NIX_TX_OFFLOAD_MAX] = {
+#define T(name, sz, flags)[flags] = cn9k_sso_hws_tx_adptr_enq_seg_##name,
+ NIX_TX_FASTPATH_MODES
+#undef T
+ };
+
+ const event_tx_adapter_enqueue_t sso_hws_dual_tx_adptr_enq[NIX_TX_OFFLOAD_MAX] = {
+#define T(name, sz, flags)[flags] = cn9k_sso_hws_dual_tx_adptr_enq_##name,
+ NIX_TX_FASTPATH_MODES
+#undef T
+ };
+
+ const event_tx_adapter_enqueue_t sso_hws_dual_tx_adptr_enq_seg[NIX_TX_OFFLOAD_MAX] = {
+#define T(name, sz, flags)[flags] = cn9k_sso_hws_dual_tx_adptr_enq_seg_##name,
+ NIX_TX_FASTPATH_MODES
+#undef T
+ };
+
+ event_dev->enqueue = cn9k_sso_hws_enq;
+ event_dev->enqueue_burst = cn9k_sso_hws_enq_burst;
+ event_dev->enqueue_new_burst = cn9k_sso_hws_enq_new_burst;
+ event_dev->enqueue_forward_burst = cn9k_sso_hws_enq_fwd_burst;
+ if (dev->rx_offloads & NIX_RX_MULTI_SEG_F) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue, sso_hws_deq_seg);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_seg_burst);
+ if (dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_deq_tmo_seg);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_tmo_seg_burst);
+ }
+ if (dev->is_ca_internal_port) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_deq_ca_seg);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_ca_seg_burst);
+ }
+
+ if (dev->is_ca_internal_port && dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_deq_tmo_ca_seg);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_tmo_ca_seg_burst);
+ }
+ } else {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue, sso_hws_deq);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_burst);
+ if (dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_deq_tmo);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_tmo_burst);
+ }
+ if (dev->is_ca_internal_port) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_deq_ca);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_ca_burst);
+ }
+
+ if (dev->is_ca_internal_port && dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_deq_tmo_ca);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_tmo_ca_burst);
+ }
+ }
+ event_dev->ca_enqueue = cn9k_sso_hws_ca_enq;
+
+ if (dev->tx_offloads & NIX_TX_MULTI_SEG_F)
+ CN9K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,
+ sso_hws_tx_adptr_enq_seg);
+ else
+ CN9K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,
+ sso_hws_tx_adptr_enq);
+
+ if (dev->dual_ws) {
+ event_dev->enqueue = cn9k_sso_hws_dual_enq;
+ event_dev->enqueue_burst = cn9k_sso_hws_dual_enq_burst;
+ event_dev->enqueue_new_burst = cn9k_sso_hws_dual_enq_new_burst;
+ event_dev->enqueue_forward_burst =
+ cn9k_sso_hws_dual_enq_fwd_burst;
+ event_dev->ca_enqueue = cn9k_sso_hws_dual_ca_enq;
+
+ if (dev->rx_offloads & NIX_RX_MULTI_SEG_F) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_dual_deq_seg);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_seg_burst);
+ if (dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_dual_deq_tmo_seg);
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_tmo_seg_burst);
+ }
+ if (dev->is_ca_internal_port) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_dual_deq_ca_seg);
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_ca_seg_burst);
+ }
+ if (dev->is_ca_internal_port && dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue,
+ sso_hws_dual_deq_tmo_ca_seg);
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_tmo_ca_seg_burst);
+ }
+ } else {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_dual_deq);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_burst);
+ if (dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_dual_deq_tmo);
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_tmo_burst);
+ }
+ if (dev->is_ca_internal_port) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_dual_deq_ca);
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_ca_burst);
+ }
+ if (dev->is_ca_internal_port && dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_dual_deq_tmo_ca);
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_tmo_ca_burst);
+ }
+ }
+
+ if (dev->tx_offloads & NIX_TX_MULTI_SEG_F)
+ CN9K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,
+ sso_hws_dual_tx_adptr_enq_seg);
+ else
+ CN9K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,
+ sso_hws_dual_tx_adptr_enq);
+ }
+
+ event_dev->txa_enqueue_same_dest = event_dev->txa_enqueue;
+ rte_mb();
+}
+
+static void *
+cn9k_sso_init_hws_mem(void *arg, uint8_t port_id)
+{
+ struct cnxk_sso_evdev *dev = arg;
+ struct cn9k_sso_hws_dual *dws;
+ struct cn9k_sso_hws *ws;
+ void *data;
+
+ if (dev->dual_ws) {
+ dws = rte_zmalloc("cn9k_dual_ws",
+ sizeof(struct cn9k_sso_hws_dual) +
+ RTE_CACHE_LINE_SIZE,
+ RTE_CACHE_LINE_SIZE);
+ if (dws == NULL) {
+ plt_err("Failed to alloc memory for port=%d", port_id);
+ return NULL;
+ }
+
+ dws = RTE_PTR_ADD(dws, sizeof(struct cnxk_sso_hws_cookie));
+ dws->base[0] = roc_sso_hws_base_get(
+ &dev->sso, CN9K_DUAL_WS_PAIR_ID(port_id, 0));
+ dws->base[1] = roc_sso_hws_base_get(
+ &dev->sso, CN9K_DUAL_WS_PAIR_ID(port_id, 1));
+ dws->hws_id = port_id;
+ dws->swtag_req = 0;
+ dws->vws = 0;
+ if (dev->deq_tmo_ns)
+ dws->gw_wdata = BIT_ULL(16);
+ dws->gw_wdata |= 1;
+
+ data = dws;
+ } else {
+ /* Allocate event port memory */
+ ws = rte_zmalloc("cn9k_ws",
+ sizeof(struct cn9k_sso_hws) +
+ RTE_CACHE_LINE_SIZE,
+ RTE_CACHE_LINE_SIZE);
+ if (ws == NULL) {
+ plt_err("Failed to alloc memory for port=%d", port_id);
+ return NULL;
+ }
+
+ /* First cache line is reserved for cookie */
+ ws = RTE_PTR_ADD(ws, sizeof(struct cnxk_sso_hws_cookie));
+ ws->base = roc_sso_hws_base_get(&dev->sso, port_id);
+ ws->hws_id = port_id;
+ ws->swtag_req = 0;
+ if (dev->deq_tmo_ns)
+ ws->gw_wdata = BIT_ULL(16);
+ ws->gw_wdata |= 1;
+
+ data = ws;
+ }
+
+ return data;
+}
+