drivers/event/octeontx2/otx2_evdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <inttypes.h>
   6
   7 #include <rte_bus_pci.h>
   8 #include <rte_common.h>
   9 #include <rte_eal.h>
  10 #include <rte_eventdev_pmd_pci.h>
  11 #include <rte_kvargs.h>
  12 #include <rte_mbuf_pool_ops.h>
  13 #include <rte_pci.h>
  14
  15 #include "otx2_evdev_stats.h"
  16 #include "otx2_evdev.h"
  17 #include "otx2_irq.h"
  18 #include "otx2_tim_evdev.h"
  19
  20 static inline int
  21 sso_get_msix_offsets(const struct rte_eventdev *event_dev)
  22 {
  23         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
  24         uint8_t nb_ports = dev->nb_event_ports * (dev->dual_ws ? 2 : 1);
  25         struct otx2_mbox *mbox = dev->mbox;
  26         struct msix_offset_rsp *msix_rsp;
  27         int i, rc;
  28
  29         /* Get SSO and SSOW MSIX vector offsets */
  30         otx2_mbox_alloc_msg_msix_offset(mbox);
  31         rc = otx2_mbox_process_msg(mbox, (void *)&msix_rsp);
  32
  33         for (i = 0; i < nb_ports; i++)
  34                 dev->ssow_msixoff[i] = msix_rsp->ssow_msixoff[i];
  35
  36         for (i = 0; i < dev->nb_event_queues; i++)
  37                 dev->sso_msixoff[i] = msix_rsp->sso_msixoff[i];
  38
  39         return rc;
  40 }
  41
  42 void
  43 sso_fastpath_fns_set(struct rte_eventdev *event_dev)
  44 {
  45         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
  46         /* Single WS modes */
  47         const event_dequeue_t ssogws_deq[2][2][2][2][2][2] = {
  48 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
  49                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_ ##name,
  50 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
  51 #undef R
  52         };
  53
  54         const event_dequeue_burst_t ssogws_deq_burst[2][2][2][2][2][2] = {
  55 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
  56                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_burst_ ##name,
  57 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
  58 #undef R
  59         };
  60
  61         const event_dequeue_t ssogws_deq_timeout[2][2][2][2][2][2] = {
  62 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
  63                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_timeout_ ##name,
  64 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
  65 #undef R
  66         };
  67
  68         const event_dequeue_burst_t
  69                 ssogws_deq_timeout_burst[2][2][2][2][2][2] = {
  70 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
  71                 [f5][f4][f3][f2][f1][f0] =                              \
  72                         otx2_ssogws_deq_timeout_burst_ ##name,
  73 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
  74 #undef R
  75         };
  76
  77         const event_dequeue_t ssogws_deq_seg[2][2][2][2][2][2] = {
  78 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
  79                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_seg_ ##name,
  80 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
  81 #undef R
  82         };
  83
  84         const event_dequeue_burst_t ssogws_deq_seg_burst[2][2][2][2][2][2] = {
  85 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
  86                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_seg_burst_ ##name,
  87 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
  88 #undef R
  89         };
  90
  91         const event_dequeue_t ssogws_deq_seg_timeout[2][2][2][2][2][2] = {
  92 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
  93                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_deq_seg_timeout_ ##name,
  94 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
  95 #undef R
  96         };
  97
  98         const event_dequeue_burst_t
  99                 ssogws_deq_seg_timeout_burst[2][2][2][2][2][2] = {
 100 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 101                 [f5][f4][f3][f2][f1][f0] =                              \
 102                                 otx2_ssogws_deq_seg_timeout_burst_ ##name,
 103 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 104 #undef R
 105         };
 106
 107
 108         /* Dual WS modes */
 109         const event_dequeue_t ssogws_dual_deq[2][2][2][2][2][2] = {
 110 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 111                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_ ##name,
 112 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 113 #undef R
 114         };
 115
 116         const event_dequeue_burst_t ssogws_dual_deq_burst[2][2][2][2][2][2] = {
 117 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 118                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_burst_ ##name,
 119 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 120 #undef R
 121         };
 122
 123         const event_dequeue_t ssogws_dual_deq_timeout[2][2][2][2][2][2] = {
 124 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 125                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_timeout_ ##name,
 126 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 127 #undef R
 128         };
 129
 130         const event_dequeue_burst_t
 131                 ssogws_dual_deq_timeout_burst[2][2][2][2][2][2] = {
 132 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 133         [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_timeout_burst_ ##name,
 134 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 135 #undef R
 136         };
 137
 138         const event_dequeue_t ssogws_dual_deq_seg[2][2][2][2][2][2] = {
 139 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 140                 [f5][f4][f3][f2][f1][f0] = otx2_ssogws_dual_deq_seg_ ##name,
 141 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 142 #undef R
 143         };
 144
 145         const event_dequeue_burst_t
 146                 ssogws_dual_deq_seg_burst[2][2][2][2][2][2] = {
 147 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 148                 [f5][f4][f3][f2][f1][f0] =                              \
 149                                 otx2_ssogws_dual_deq_seg_burst_ ##name,
 150 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 151 #undef R
 152         };
 153
 154         const event_dequeue_t ssogws_dual_deq_seg_timeout[2][2][2][2][2][2] = {
 155 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 156                 [f5][f4][f3][f2][f1][f0] =                              \
 157                                 otx2_ssogws_dual_deq_seg_timeout_ ##name,
 158 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 159 #undef R
 160         };
 161
 162         const event_dequeue_burst_t
 163                 ssogws_dual_deq_seg_timeout_burst[2][2][2][2][2][2] = {
 164 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 165         [f5][f4][f3][f2][f1][f0] =                                      \
 166                 otx2_ssogws_dual_deq_seg_timeout_burst_ ##name,
 167 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 168 #undef R
 169         };
 170
 171         event_dev->enqueue                      = otx2_ssogws_enq;
 172         event_dev->enqueue_burst                = otx2_ssogws_enq_burst;
 173         event_dev->enqueue_new_burst            = otx2_ssogws_enq_new_burst;
 174         event_dev->enqueue_forward_burst        = otx2_ssogws_enq_fwd_burst;
 175         if (dev->rx_offloads & NIX_RX_MULTI_SEG_F) {
 176                 event_dev->dequeue              = ssogws_deq_seg
 177                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 178                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 179                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 180                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 181                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 182                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 183                 event_dev->dequeue_burst        = ssogws_deq_seg_burst
 184                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 185                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 186                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 187                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 188                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 189                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 190                 if (dev->is_timeout_deq) {
 191                         event_dev->dequeue      = ssogws_deq_seg_timeout
 192                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 193                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 194                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 195                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 196                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 197                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 198                         event_dev->dequeue_burst        =
 199                                 ssogws_deq_seg_timeout_burst
 200                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 201                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 202                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 203                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 204                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 205                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 206                 }
 207         } else {
 208                 event_dev->dequeue                      = ssogws_deq
 209                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 210                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 211                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 212                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 213                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 214                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 215                 event_dev->dequeue_burst                = ssogws_deq_burst
 216                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 217                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 218                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 219                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 220                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 221                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 222                 if (dev->is_timeout_deq) {
 223                         event_dev->dequeue              = ssogws_deq_timeout
 224                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 225                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 226                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 227                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 228                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 229                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 230                         event_dev->dequeue_burst        =
 231                                 ssogws_deq_timeout_burst
 232                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 233                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 234                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 235                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 236                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 237                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 238                 }
 239         }
 240
 241         if (dev->dual_ws) {
 242                 event_dev->enqueue              = otx2_ssogws_dual_enq;
 243                 event_dev->enqueue_burst        = otx2_ssogws_dual_enq_burst;
 244                 event_dev->enqueue_new_burst    =
 245                                         otx2_ssogws_dual_enq_new_burst;
 246                 event_dev->enqueue_forward_burst =
 247                                         otx2_ssogws_dual_enq_fwd_burst;
 248
 249                 if (dev->rx_offloads & NIX_RX_MULTI_SEG_F) {
 250                         event_dev->dequeue      = ssogws_dual_deq_seg
 251                                 [!!(dev->rx_offloads &
 252                                                 NIX_RX_OFFLOAD_TSTAMP_F)]
 253                                 [!!(dev->rx_offloads &
 254                                                 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 255                                 [!!(dev->rx_offloads &
 256                                                 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 257                                 [!!(dev->rx_offloads &
 258                                                 NIX_RX_OFFLOAD_CHECKSUM_F)]
 259                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 260                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 261                         event_dev->dequeue_burst = ssogws_dual_deq_seg_burst
 262                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 263                                 [!!(dev->rx_offloads &
 264                                                 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 265                                 [!!(dev->rx_offloads &
 266                                                 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 267                                 [!!(dev->rx_offloads &
 268                                                 NIX_RX_OFFLOAD_CHECKSUM_F)]
 269                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 270                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 271                         if (dev->is_timeout_deq) {
 272                                 event_dev->dequeue      =
 273                                         ssogws_dual_deq_seg_timeout
 274                                         [!!(dev->rx_offloads &
 275                                                 NIX_RX_OFFLOAD_TSTAMP_F)]
 276                                         [!!(dev->rx_offloads &
 277                                                 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 278                                         [!!(dev->rx_offloads &
 279                                                 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 280                                         [!!(dev->rx_offloads &
 281                                                 NIX_RX_OFFLOAD_CHECKSUM_F)]
 282                                         [!!(dev->rx_offloads &
 283                                                         NIX_RX_OFFLOAD_PTYPE_F)]
 284                                         [!!(dev->rx_offloads &
 285                                                         NIX_RX_OFFLOAD_RSS_F)];
 286                                 event_dev->dequeue_burst =
 287                                         ssogws_dual_deq_seg_timeout_burst
 288                                         [!!(dev->rx_offloads &
 289                                                 NIX_RX_OFFLOAD_TSTAMP_F)]
 290                                         [!!(dev->rx_offloads &
 291                                                 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 292                                         [!!(dev->rx_offloads &
 293                                                 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 294                                         [!!(dev->rx_offloads &
 295                                                 NIX_RX_OFFLOAD_CHECKSUM_F)]
 296                                         [!!(dev->rx_offloads &
 297                                                         NIX_RX_OFFLOAD_PTYPE_F)]
 298                                         [!!(dev->rx_offloads &
 299                                                         NIX_RX_OFFLOAD_RSS_F)];
 300                         }
 301                 } else {
 302                         event_dev->dequeue              = ssogws_dual_deq
 303                                 [!!(dev->rx_offloads &
 304                                                 NIX_RX_OFFLOAD_TSTAMP_F)]
 305                                 [!!(dev->rx_offloads &
 306                                                 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 307                                 [!!(dev->rx_offloads &
 308                                                 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 309                                 [!!(dev->rx_offloads &
 310                                                 NIX_RX_OFFLOAD_CHECKSUM_F)]
 311                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 312                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 313                         event_dev->dequeue_burst        = ssogws_dual_deq_burst
 314                                 [!!(dev->rx_offloads &
 315                                                 NIX_RX_OFFLOAD_TSTAMP_F)]
 316                                 [!!(dev->rx_offloads &
 317                                                 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 318                                 [!!(dev->rx_offloads &
 319                                                 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 320                                 [!!(dev->rx_offloads &
 321                                                 NIX_RX_OFFLOAD_CHECKSUM_F)]
 322                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 323                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 324                         if (dev->is_timeout_deq) {
 325                                 event_dev->dequeue      =
 326                                         ssogws_dual_deq_timeout
 327                                         [!!(dev->rx_offloads &
 328                                                 NIX_RX_OFFLOAD_TSTAMP_F)]
 329                                         [!!(dev->rx_offloads &
 330                                                 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 331                                         [!!(dev->rx_offloads &
 332                                                 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 333                                         [!!(dev->rx_offloads &
 334                                                 NIX_RX_OFFLOAD_CHECKSUM_F)]
 335                                         [!!(dev->rx_offloads &
 336                                                         NIX_RX_OFFLOAD_PTYPE_F)]
 337                                         [!!(dev->rx_offloads &
 338                                                         NIX_RX_OFFLOAD_RSS_F)];
 339                                 event_dev->dequeue_burst =
 340                                         ssogws_dual_deq_timeout_burst
 341                                         [!!(dev->rx_offloads &
 342                                                 NIX_RX_OFFLOAD_TSTAMP_F)]
 343                                         [!!(dev->rx_offloads &
 344                                                 NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 345                                         [!!(dev->rx_offloads &
 346                                                 NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 347                                         [!!(dev->rx_offloads &
 348                                                 NIX_RX_OFFLOAD_CHECKSUM_F)]
 349                                         [!!(dev->rx_offloads &
 350                                                         NIX_RX_OFFLOAD_PTYPE_F)]
 351                                         [!!(dev->rx_offloads &
 352                                                         NIX_RX_OFFLOAD_RSS_F)];
 353                         }
 354                 }
 355         }
 356         rte_mb();
 357 }
 358
 359 static void
 360 otx2_sso_info_get(struct rte_eventdev *event_dev,
 361                   struct rte_event_dev_info *dev_info)
 362 {
 363         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
 364
 365         dev_info->driver_name = RTE_STR(EVENTDEV_NAME_OCTEONTX2_PMD);
 366         dev_info->min_dequeue_timeout_ns = dev->min_dequeue_timeout_ns;
 367         dev_info->max_dequeue_timeout_ns = dev->max_dequeue_timeout_ns;
 368         dev_info->max_event_queues = dev->max_event_queues;
 369         dev_info->max_event_queue_flows = (1ULL << 20);
 370         dev_info->max_event_queue_priority_levels = 8;
 371         dev_info->max_event_priority_levels = 1;
 372         dev_info->max_event_ports = dev->max_event_ports;
 373         dev_info->max_event_port_dequeue_depth = 1;
 374         dev_info->max_event_port_enqueue_depth = 1;
 375         dev_info->max_num_events =  dev->max_num_events;
 376         dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_QUEUE_QOS |
 377                                         RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
 378                                         RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES |
 379                                         RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK |
 380                                         RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT |
 381                                         RTE_EVENT_DEV_CAP_NONSEQ_MODE;
 382 }
 383
 384 static void
 385 sso_port_link_modify(struct otx2_ssogws *ws, uint8_t queue, uint8_t enable)
 386 {
 387         uintptr_t base = OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op);
 388         uint64_t val;
 389
 390         val = queue;
 391         val |= 0ULL << 12; /* SET 0 */
 392         val |= 0x8000800080000000; /* Dont modify rest of the masks */
 393         val |= (uint64_t)enable << 14;   /* Enable/Disable Membership. */
 394
 395         otx2_write64(val, base + SSOW_LF_GWS_GRPMSK_CHG);
 396 }
 397
 398 static int
 399 otx2_sso_port_link(struct rte_eventdev *event_dev, void *port,
 400                    const uint8_t queues[], const uint8_t priorities[],
 401                    uint16_t nb_links)
 402 {
 403         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
 404         uint8_t port_id = 0;
 405         uint16_t link;
 406
 407         RTE_SET_USED(priorities);
 408         for (link = 0; link < nb_links; link++) {
 409                 if (dev->dual_ws) {
 410                         struct otx2_ssogws_dual *ws = port;
 411
 412                         port_id = ws->port;
 413                         sso_port_link_modify((struct otx2_ssogws *)
 414                                         &ws->ws_state[0], queues[link], true);
 415                         sso_port_link_modify((struct otx2_ssogws *)
 416                                         &ws->ws_state[1], queues[link], true);
 417                 } else {
 418                         struct otx2_ssogws *ws = port;
 419
 420                         port_id = ws->port;
 421                         sso_port_link_modify(ws, queues[link], true);
 422                 }
 423         }
 424         sso_func_trace("Port=%d nb_links=%d", port_id, nb_links);
 425
 426         return (int)nb_links;
 427 }
 428
 429 static int
 430 otx2_sso_port_unlink(struct rte_eventdev *event_dev, void *port,
 431                      uint8_t queues[], uint16_t nb_unlinks)
 432 {
 433         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
 434         uint8_t port_id = 0;
 435         uint16_t unlink;
 436
 437         for (unlink = 0; unlink < nb_unlinks; unlink++) {
 438                 if (dev->dual_ws) {
 439                         struct otx2_ssogws_dual *ws = port;
 440
 441                         port_id = ws->port;
 442                         sso_port_link_modify((struct otx2_ssogws *)
 443                                         &ws->ws_state[0], queues[unlink],
 444                                         false);
 445                         sso_port_link_modify((struct otx2_ssogws *)
 446                                         &ws->ws_state[1], queues[unlink],
 447                                         false);
 448                 } else {
 449                         struct otx2_ssogws *ws = port;
 450
 451                         port_id = ws->port;
 452                         sso_port_link_modify(ws, queues[unlink], false);
 453                 }
 454         }
 455         sso_func_trace("Port=%d nb_unlinks=%d", port_id, nb_unlinks);
 456
 457         return (int)nb_unlinks;
 458 }
 459
 460 static int
 461 sso_hw_lf_cfg(struct otx2_mbox *mbox, enum otx2_sso_lf_type type,
 462               uint16_t nb_lf, uint8_t attach)
 463 {
 464         if (attach) {
 465                 struct rsrc_attach_req *req;
 466
 467                 req = otx2_mbox_alloc_msg_attach_resources(mbox);
 468                 switch (type) {
 469                 case SSO_LF_GGRP:
 470                         req->sso = nb_lf;
 471                         break;
 472                 case SSO_LF_GWS:
 473                         req->ssow = nb_lf;
 474                         break;
 475                 default:
 476                         return -EINVAL;
 477                 }
 478                 req->modify = true;
 479                 if (otx2_mbox_process(mbox) < 0)
 480                         return -EIO;
 481         } else {
 482                 struct rsrc_detach_req *req;
 483
 484                 req = otx2_mbox_alloc_msg_detach_resources(mbox);
 485                 switch (type) {
 486                 case SSO_LF_GGRP:
 487                         req->sso = true;
 488                         break;
 489                 case SSO_LF_GWS:
 490                         req->ssow = true;
 491                         break;
 492                 default:
 493                         return -EINVAL;
 494                 }
 495                 req->partial = true;
 496                 if (otx2_mbox_process(mbox) < 0)
 497                         return -EIO;
 498         }
 499
 500         return 0;
 501 }
 502
 503 static int
 504 sso_lf_cfg(struct otx2_sso_evdev *dev, struct otx2_mbox *mbox,
 505            enum otx2_sso_lf_type type, uint16_t nb_lf, uint8_t alloc)
 506 {
 507         void *rsp;
 508         int rc;
 509
 510         if (alloc) {
 511                 switch (type) {
 512                 case SSO_LF_GGRP:
 513                         {
 514                         struct sso_lf_alloc_req *req_ggrp;
 515                         req_ggrp = otx2_mbox_alloc_msg_sso_lf_alloc(mbox);
 516                         req_ggrp->hwgrps = nb_lf;
 517                         }
 518                         break;
 519                 case SSO_LF_GWS:
 520                         {
 521                         struct ssow_lf_alloc_req *req_hws;
 522                         req_hws = otx2_mbox_alloc_msg_ssow_lf_alloc(mbox);
 523                         req_hws->hws = nb_lf;
 524                         }
 525                         break;
 526                 default:
 527                         return -EINVAL;
 528                 }
 529         } else {
 530                 switch (type) {
 531                 case SSO_LF_GGRP:
 532                         {
 533                         struct sso_lf_free_req *req_ggrp;
 534                         req_ggrp = otx2_mbox_alloc_msg_sso_lf_free(mbox);
 535                         req_ggrp->hwgrps = nb_lf;
 536                         }
 537                         break;
 538                 case SSO_LF_GWS:
 539                         {
 540                         struct ssow_lf_free_req *req_hws;
 541                         req_hws = otx2_mbox_alloc_msg_ssow_lf_free(mbox);
 542                         req_hws->hws = nb_lf;
 543                         }
 544                         break;
 545                 default:
 546                         return -EINVAL;
 547                 }
 548         }
 549
 550         rc = otx2_mbox_process_msg_tmo(mbox, (void **)&rsp, ~0);
 551         if (rc < 0)
 552                 return rc;
 553
 554         if (alloc && type == SSO_LF_GGRP) {
 555                 struct sso_lf_alloc_rsp *rsp_ggrp = rsp;
 556
 557                 dev->xaq_buf_size = rsp_ggrp->xaq_buf_size;
 558                 dev->xae_waes = rsp_ggrp->xaq_wq_entries;
 559                 dev->iue = rsp_ggrp->in_unit_entries;
 560         }
 561
 562         return 0;
 563 }
 564
 565 static void
 566 otx2_sso_port_release(void *port)
 567 {
 568         rte_free(port);
 569 }
 570
 571 static void
 572 otx2_sso_queue_release(struct rte_eventdev *event_dev, uint8_t queue_id)
 573 {
 574         RTE_SET_USED(event_dev);
 575         RTE_SET_USED(queue_id);
 576 }
 577
 578 static void
 579 sso_clr_links(const struct rte_eventdev *event_dev)
 580 {
 581         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
 582         int i, j;
 583
 584         for (i = 0; i < dev->nb_event_ports; i++) {
 585                 if (dev->dual_ws) {
 586                         struct otx2_ssogws_dual *ws;
 587
 588                         ws = event_dev->data->ports[i];
 589                         for (j = 0; j < dev->nb_event_queues; j++) {
 590                                 sso_port_link_modify((struct otx2_ssogws *)
 591                                                 &ws->ws_state[0], j, false);
 592                                 sso_port_link_modify((struct otx2_ssogws *)
 593                                                 &ws->ws_state[1], j, false);
 594                         }
 595                 } else {
 596                         struct otx2_ssogws *ws;
 597
 598                         ws = event_dev->data->ports[i];
 599                         for (j = 0; j < dev->nb_event_queues; j++)
 600                                 sso_port_link_modify(ws, j, false);
 601                 }
 602         }
 603 }
 604
 605 static void
 606 sso_set_port_ops(struct otx2_ssogws *ws, uintptr_t base)
 607 {
 608         ws->tag_op              = base + SSOW_LF_GWS_TAG;
 609         ws->wqp_op              = base + SSOW_LF_GWS_WQP;
 610         ws->getwrk_op           = base + SSOW_LF_GWS_OP_GET_WORK;
 611         ws->swtp_op             = base + SSOW_LF_GWS_SWTP;
 612         ws->swtag_norm_op       = base + SSOW_LF_GWS_OP_SWTAG_NORM;
 613         ws->swtag_desched_op    = base + SSOW_LF_GWS_OP_SWTAG_DESCHED;
 614 }
 615
 616 static int
 617 sso_configure_dual_ports(const struct rte_eventdev *event_dev)
 618 {
 619         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
 620         struct otx2_mbox *mbox = dev->mbox;
 621         uint8_t vws = 0;
 622         uint8_t nb_lf;
 623         int i, rc;
 624
 625         otx2_sso_dbg("Configuring event ports %d", dev->nb_event_ports);
 626
 627         nb_lf = dev->nb_event_ports * 2;
 628         /* Ask AF to attach required LFs. */
 629         rc = sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, true);
 630         if (rc < 0) {
 631                 otx2_err("Failed to attach SSO GWS LF");
 632                 return -ENODEV;
 633         }
 634
 635         if (sso_lf_cfg(dev, mbox, SSO_LF_GWS, nb_lf, true) < 0) {
 636                 sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, false);
 637                 otx2_err("Failed to init SSO GWS LF");
 638                 return -ENODEV;
 639         }
 640
 641         for (i = 0; i < dev->nb_event_ports; i++) {
 642                 struct otx2_ssogws_dual *ws;
 643                 uintptr_t base;
 644
 645                 /* Free memory prior to re-allocation if needed */
 646                 if (event_dev->data->ports[i] != NULL) {
 647                         ws = event_dev->data->ports[i];
 648                         rte_free(ws);
 649                         ws = NULL;
 650                 }
 651
 652                 /* Allocate event port memory */
 653                 ws = rte_zmalloc_socket("otx2_sso_ws",
 654                                         sizeof(struct otx2_ssogws_dual),
 655                                         RTE_CACHE_LINE_SIZE,
 656                                         event_dev->data->socket_id);
 657                 if (ws == NULL) {
 658                         otx2_err("Failed to alloc memory for port=%d", i);
 659                         rc = -ENOMEM;
 660                         break;
 661                 }
 662
 663                 ws->port = i;
 664                 base = dev->bar2 + (RVU_BLOCK_ADDR_SSOW << 20 | vws << 12);
 665                 sso_set_port_ops((struct otx2_ssogws *)&ws->ws_state[0], base);
 666                 vws++;
 667
 668                 base = dev->bar2 + (RVU_BLOCK_ADDR_SSOW << 20 | vws << 12);
 669                 sso_set_port_ops((struct otx2_ssogws *)&ws->ws_state[1], base);
 670                 vws++;
 671
 672                 event_dev->data->ports[i] = ws;
 673         }
 674
 675         if (rc < 0) {
 676                 sso_lf_cfg(dev, mbox, SSO_LF_GWS, nb_lf, false);
 677                 sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, false);
 678         }
 679
 680         return rc;
 681 }
 682
 683 static int
 684 sso_configure_ports(const struct rte_eventdev *event_dev)
 685 {
 686         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
 687         struct otx2_mbox *mbox = dev->mbox;
 688         uint8_t nb_lf;
 689         int i, rc;
 690
 691         otx2_sso_dbg("Configuring event ports %d", dev->nb_event_ports);
 692
 693         nb_lf = dev->nb_event_ports;
 694         /* Ask AF to attach required LFs. */
 695         rc = sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, true);
 696         if (rc < 0) {
 697                 otx2_err("Failed to attach SSO GWS LF");
 698                 return -ENODEV;
 699         }
 700
 701         if (sso_lf_cfg(dev, mbox, SSO_LF_GWS, nb_lf, true) < 0) {
 702                 sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, false);
 703                 otx2_err("Failed to init SSO GWS LF");
 704                 return -ENODEV;
 705         }
 706
 707         for (i = 0; i < nb_lf; i++) {
 708                 struct otx2_ssogws *ws;
 709                 uintptr_t base;
 710
 711                 /* Free memory prior to re-allocation if needed */
 712                 if (event_dev->data->ports[i] != NULL) {
 713                         ws = event_dev->data->ports[i];
 714                         rte_free(ws);
 715                         ws = NULL;
 716                 }
 717
 718                 /* Allocate event port memory */
 719                 ws = rte_zmalloc_socket("otx2_sso_ws",
 720                                         sizeof(struct otx2_ssogws),
 721                                         RTE_CACHE_LINE_SIZE,
 722                                         event_dev->data->socket_id);
 723                 if (ws == NULL) {
 724                         otx2_err("Failed to alloc memory for port=%d", i);
 725                         rc = -ENOMEM;
 726                         break;
 727                 }
 728
 729                 ws->port = i;
 730                 base = dev->bar2 + (RVU_BLOCK_ADDR_SSOW << 20 | i << 12);
 731                 sso_set_port_ops(ws, base);
 732
 733                 event_dev->data->ports[i] = ws;
 734         }
 735
 736         if (rc < 0) {
 737                 sso_lf_cfg(dev, mbox, SSO_LF_GWS, nb_lf, false);
 738                 sso_hw_lf_cfg(mbox, SSO_LF_GWS, nb_lf, false);
 739         }
 740
 741         return rc;
 742 }
 743
 744 static int
 745 sso_configure_queues(const struct rte_eventdev *event_dev)
 746 {
 747         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
 748         struct otx2_mbox *mbox = dev->mbox;
 749         uint8_t nb_lf;
 750         int rc;
 751
 752         otx2_sso_dbg("Configuring event queues %d", dev->nb_event_queues);
 753
 754         nb_lf = dev->nb_event_queues;
 755         /* Ask AF to attach required LFs. */
 756         rc = sso_hw_lf_cfg(mbox, SSO_LF_GGRP, nb_lf, true);
 757         if (rc < 0) {
 758                 otx2_err("Failed to attach SSO GGRP LF");
 759                 return -ENODEV;
 760         }
 761
 762         if (sso_lf_cfg(dev, mbox, SSO_LF_GGRP, nb_lf, true) < 0) {
 763                 sso_hw_lf_cfg(mbox, SSO_LF_GGRP, nb_lf, false);
 764                 otx2_err("Failed to init SSO GGRP LF");
 765                 return -ENODEV;
 766         }
 767
 768         return rc;
 769 }
 770
 771 static int
 772 sso_xaq_allocate(struct otx2_sso_evdev *dev)
 773 {
 774         const struct rte_memzone *mz;
 775         struct npa_aura_s *aura;
 776         static int reconfig_cnt;
 777         char pool_name[RTE_MEMZONE_NAMESIZE];
 778         uint32_t xaq_cnt;
 779         int rc;
 780
 781         if (dev->xaq_pool)
 782                 rte_mempool_free(dev->xaq_pool);
 783
 784         /*
 785          * Allocate memory for Add work backpressure.
 786          */
 787         mz = rte_memzone_lookup(OTX2_SSO_FC_NAME);
 788         if (mz == NULL)
 789                 mz = rte_memzone_reserve_aligned(OTX2_SSO_FC_NAME,
 790                                                  OTX2_ALIGN +
 791                                                  sizeof(struct npa_aura_s),
 792                                                  rte_socket_id(),
 793                                                  RTE_MEMZONE_IOVA_CONTIG,
 794                                                  OTX2_ALIGN);
 795         if (mz == NULL) {
 796                 otx2_err("Failed to allocate mem for fcmem");
 797                 return -ENOMEM;
 798         }
 799
 800         dev->fc_iova = mz->iova;
 801         dev->fc_mem = mz->addr;
 802
 803         aura = (struct npa_aura_s *)((uintptr_t)dev->fc_mem + OTX2_ALIGN);
 804         memset(aura, 0, sizeof(struct npa_aura_s));
 805
 806         aura->fc_ena = 1;
 807         aura->fc_addr = dev->fc_iova;
 808         aura->fc_hyst_bits = 0; /* Store count on all updates */
 809
 810         /* Taken from HRM 14.3.3(4) */
 811         xaq_cnt = dev->nb_event_queues * OTX2_SSO_XAQ_CACHE_CNT;
 812         if (dev->xae_cnt)
 813                 xaq_cnt += dev->xae_cnt / dev->xae_waes;
 814         else if (dev->adptr_xae_cnt)
 815                 xaq_cnt += (dev->adptr_xae_cnt / dev->xae_waes) +
 816                         (OTX2_SSO_XAQ_SLACK * dev->nb_event_queues);
 817         else
 818                 xaq_cnt += (dev->iue / dev->xae_waes) +
 819                         (OTX2_SSO_XAQ_SLACK * dev->nb_event_queues);
 820
 821         otx2_sso_dbg("Configuring %d xaq buffers", xaq_cnt);
 822         /* Setup XAQ based on number of nb queues. */
 823         snprintf(pool_name, 30, "otx2_xaq_buf_pool_%d", reconfig_cnt);
 824         dev->xaq_pool = (void *)rte_mempool_create_empty(pool_name,
 825                         xaq_cnt, dev->xaq_buf_size, 0, 0,
 826                         rte_socket_id(), 0);
 827
 828         if (dev->xaq_pool == NULL) {
 829                 otx2_err("Unable to create empty mempool.");
 830                 rte_memzone_free(mz);
 831                 return -ENOMEM;
 832         }
 833
 834         rc = rte_mempool_set_ops_byname(dev->xaq_pool,
 835                                         rte_mbuf_platform_mempool_ops(), aura);
 836         if (rc != 0) {
 837                 otx2_err("Unable to set xaqpool ops.");
 838                 goto alloc_fail;
 839         }
 840
 841         rc = rte_mempool_populate_default(dev->xaq_pool);
 842         if (rc < 0) {
 843                 otx2_err("Unable to set populate xaqpool.");
 844                 goto alloc_fail;
 845         }
 846         reconfig_cnt++;
 847         /* When SW does addwork (enqueue) check if there is space in XAQ by
 848          * comparing fc_addr above against the xaq_lmt calculated below.
 849          * There should be a minimum headroom (OTX2_SSO_XAQ_SLACK / 2) for SSO
 850          * to request XAQ to cache them even before enqueue is called.
 851          */
 852         dev->xaq_lmt = xaq_cnt - (OTX2_SSO_XAQ_SLACK / 2 *
 853                                   dev->nb_event_queues);
 854         dev->nb_xaq_cfg = xaq_cnt;
 855
 856         return 0;
 857 alloc_fail:
 858         rte_mempool_free(dev->xaq_pool);
 859         rte_memzone_free(mz);
 860         return rc;
 861 }
 862
 863 static int
 864 sso_ggrp_alloc_xaq(struct otx2_sso_evdev *dev)
 865 {
 866         struct otx2_mbox *mbox = dev->mbox;
 867         struct sso_hw_setconfig *req;
 868
 869         otx2_sso_dbg("Configuring XAQ for GGRPs");
 870         req = otx2_mbox_alloc_msg_sso_hw_setconfig(mbox);
 871         req->npa_pf_func = otx2_npa_pf_func_get();
 872         req->npa_aura_id = npa_lf_aura_handle_to_aura(dev->xaq_pool->pool_id);
 873         req->hwgrps = dev->nb_event_queues;
 874
 875         return otx2_mbox_process(mbox);
 876 }
 877
 878 static void
 879 sso_lf_teardown(struct otx2_sso_evdev *dev,
 880                 enum otx2_sso_lf_type lf_type)
 881 {
 882         uint8_t nb_lf;
 883
 884         switch (lf_type) {
 885         case SSO_LF_GGRP:
 886                 nb_lf = dev->nb_event_queues;
 887                 break;
 888         case SSO_LF_GWS:
 889                 nb_lf = dev->nb_event_ports;
 890                 nb_lf *= dev->dual_ws ? 2 : 1;
 891                 break;
 892         default:
 893                 return;
 894         }
 895
 896         sso_lf_cfg(dev, dev->mbox, lf_type, nb_lf, false);
 897         sso_hw_lf_cfg(dev->mbox, lf_type, nb_lf, false);
 898 }
 899
 900 static int
 901 otx2_sso_configure(const struct rte_eventdev *event_dev)
 902 {
 903         struct rte_event_dev_config *conf = &event_dev->data->dev_conf;
 904         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
 905         uint32_t deq_tmo_ns;
 906         int rc;
 907
 908         sso_func_trace();
 909         deq_tmo_ns = conf->dequeue_timeout_ns;
 910
 911         if (deq_tmo_ns == 0)
 912                 deq_tmo_ns = dev->min_dequeue_timeout_ns;
 913
 914         if (deq_tmo_ns < dev->min_dequeue_timeout_ns ||
 915             deq_tmo_ns > dev->max_dequeue_timeout_ns) {
 916                 otx2_err("Unsupported dequeue timeout requested");
 917                 return -EINVAL;
 918         }
 919
 920         if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
 921                 dev->is_timeout_deq = 1;
 922
 923         dev->deq_tmo_ns = deq_tmo_ns;
 924
 925         if (conf->nb_event_ports > dev->max_event_ports ||
 926             conf->nb_event_queues > dev->max_event_queues) {
 927                 otx2_err("Unsupported event queues/ports requested");
 928                 return -EINVAL;
 929         }
 930
 931         if (conf->nb_event_port_dequeue_depth > 1) {
 932                 otx2_err("Unsupported event port deq depth requested");
 933                 return -EINVAL;
 934         }
 935
 936         if (conf->nb_event_port_enqueue_depth > 1) {
 937                 otx2_err("Unsupported event port enq depth requested");
 938                 return -EINVAL;
 939         }
 940
 941         if (dev->configured)
 942                 sso_unregister_irqs(event_dev);
 943
 944         if (dev->nb_event_queues) {
 945                 /* Finit any previous queues. */
 946                 sso_lf_teardown(dev, SSO_LF_GGRP);
 947         }
 948         if (dev->nb_event_ports) {
 949                 /* Finit any previous ports. */
 950                 sso_lf_teardown(dev, SSO_LF_GWS);
 951         }
 952
 953         dev->nb_event_queues = conf->nb_event_queues;
 954         dev->nb_event_ports = conf->nb_event_ports;
 955
 956         if (dev->dual_ws)
 957                 rc = sso_configure_dual_ports(event_dev);
 958         else
 959                 rc = sso_configure_ports(event_dev);
 960
 961         if (rc < 0) {
 962                 otx2_err("Failed to configure event ports");
 963                 return -ENODEV;
 964         }
 965
 966         if (sso_configure_queues(event_dev) < 0) {
 967                 otx2_err("Failed to configure event queues");
 968                 rc = -ENODEV;
 969                 goto teardown_hws;
 970         }
 971
 972         if (sso_xaq_allocate(dev) < 0) {
 973                 rc = -ENOMEM;
 974                 goto teardown_hwggrp;
 975         }
 976
 977         /* Clear any prior port-queue mapping. */
 978         sso_clr_links(event_dev);
 979         rc = sso_ggrp_alloc_xaq(dev);
 980         if (rc < 0) {
 981                 otx2_err("Failed to alloc xaq to ggrp %d", rc);
 982                 goto teardown_hwggrp;
 983         }
 984
 985         rc = sso_get_msix_offsets(event_dev);
 986         if (rc < 0) {
 987                 otx2_err("Failed to get msix offsets %d", rc);
 988                 goto teardown_hwggrp;
 989         }
 990
 991         rc = sso_register_irqs(event_dev);
 992         if (rc < 0) {
 993                 otx2_err("Failed to register irq %d", rc);
 994                 goto teardown_hwggrp;
 995         }
 996
 997         dev->configured = 1;
 998         rte_mb();
 999
1000         return 0;
1001 teardown_hwggrp:
1002         sso_lf_teardown(dev, SSO_LF_GGRP);
1003 teardown_hws:
1004         sso_lf_teardown(dev, SSO_LF_GWS);
1005         dev->nb_event_queues = 0;
1006         dev->nb_event_ports = 0;
1007         dev->configured = 0;
1008         return rc;
1009 }
1010
1011 static void
1012 otx2_sso_queue_def_conf(struct rte_eventdev *event_dev, uint8_t queue_id,
1013                         struct rte_event_queue_conf *queue_conf)
1014 {
1015         RTE_SET_USED(event_dev);
1016         RTE_SET_USED(queue_id);
1017
1018         queue_conf->nb_atomic_flows = (1ULL << 20);
1019         queue_conf->nb_atomic_order_sequences = (1ULL << 20);
1020         queue_conf->event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES;
1021         queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
1022 }
1023
1024 static int
1025 otx2_sso_queue_setup(struct rte_eventdev *event_dev, uint8_t queue_id,
1026                      const struct rte_event_queue_conf *queue_conf)
1027 {
1028         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1029         struct otx2_mbox *mbox = dev->mbox;
1030         struct sso_grp_priority *req;
1031         int rc;
1032
1033         sso_func_trace("Queue=%d prio=%d", queue_id, queue_conf->priority);
1034
1035         req = otx2_mbox_alloc_msg_sso_grp_set_priority(dev->mbox);
1036         req->grp = queue_id;
1037         req->weight = 0xFF;
1038         req->affinity = 0xFF;
1039         /* Normalize <0-255> to <0-7> */
1040         req->priority = queue_conf->priority / 32;
1041
1042         rc = otx2_mbox_process(mbox);
1043         if (rc < 0) {
1044                 otx2_err("Failed to set priority queue=%d", queue_id);
1045                 return rc;
1046         }
1047
1048         return 0;
1049 }
1050
1051 static void
1052 otx2_sso_port_def_conf(struct rte_eventdev *event_dev, uint8_t port_id,
1053                        struct rte_event_port_conf *port_conf)
1054 {
1055         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1056
1057         RTE_SET_USED(port_id);
1058         port_conf->new_event_threshold = dev->max_num_events;
1059         port_conf->dequeue_depth = 1;
1060         port_conf->enqueue_depth = 1;
1061 }
1062
1063 static int
1064 otx2_sso_port_setup(struct rte_eventdev *event_dev, uint8_t port_id,
1065                     const struct rte_event_port_conf *port_conf)
1066 {
1067         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1068         uintptr_t grps_base[OTX2_SSO_MAX_VHGRP] = {0};
1069         uint64_t val;
1070         uint16_t q;
1071
1072         sso_func_trace("Port=%d", port_id);
1073         RTE_SET_USED(port_conf);
1074
1075         if (event_dev->data->ports[port_id] == NULL) {
1076                 otx2_err("Invalid port Id %d", port_id);
1077                 return -EINVAL;
1078         }
1079
1080         for (q = 0; q < dev->nb_event_queues; q++) {
1081                 grps_base[q] = dev->bar2 + (RVU_BLOCK_ADDR_SSO << 20 | q << 12);
1082                 if (grps_base[q] == 0) {
1083                         otx2_err("Failed to get grp[%d] base addr", q);
1084                         return -EINVAL;
1085                 }
1086         }
1087
1088         /* Set get_work timeout for HWS */
1089         val = NSEC2USEC(dev->deq_tmo_ns) - 1;
1090
1091         if (dev->dual_ws) {
1092                 struct otx2_ssogws_dual *ws = event_dev->data->ports[port_id];
1093
1094                 rte_memcpy(ws->grps_base, grps_base,
1095                            sizeof(uintptr_t) * OTX2_SSO_MAX_VHGRP);
1096                 ws->fc_mem = dev->fc_mem;
1097                 ws->xaq_lmt = dev->xaq_lmt;
1098                 otx2_write64(val, OTX2_SSOW_GET_BASE_ADDR(
1099                              ws->ws_state[0].getwrk_op) + SSOW_LF_GWS_NW_TIM);
1100                 otx2_write64(val, OTX2_SSOW_GET_BASE_ADDR(
1101                              ws->ws_state[1].getwrk_op) + SSOW_LF_GWS_NW_TIM);
1102         } else {
1103                 struct otx2_ssogws *ws = event_dev->data->ports[port_id];
1104                 uintptr_t base = OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op);
1105
1106                 rte_memcpy(ws->grps_base, grps_base,
1107                            sizeof(uintptr_t) * OTX2_SSO_MAX_VHGRP);
1108                 ws->fc_mem = dev->fc_mem;
1109                 ws->xaq_lmt = dev->xaq_lmt;
1110                 otx2_write64(val, base + SSOW_LF_GWS_NW_TIM);
1111         }
1112
1113         otx2_sso_dbg("Port=%d ws=%p", port_id, event_dev->data->ports[port_id]);
1114
1115         return 0;
1116 }
1117
1118 static int
1119 otx2_sso_timeout_ticks(struct rte_eventdev *event_dev, uint64_t ns,
1120                        uint64_t *tmo_ticks)
1121 {
1122         RTE_SET_USED(event_dev);
1123         *tmo_ticks = NSEC2TICK(ns, rte_get_timer_hz());
1124
1125         return 0;
1126 }
1127
1128 static void
1129 ssogws_dump(struct otx2_ssogws *ws, FILE *f)
1130 {
1131         uintptr_t base = OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op);
1132
1133         fprintf(f, "SSOW_LF_GWS Base addr   0x%" PRIx64 "\n", (uint64_t)base);
1134         fprintf(f, "SSOW_LF_GWS_LINKS       0x%" PRIx64 "\n",
1135                 otx2_read64(base + SSOW_LF_GWS_LINKS));
1136         fprintf(f, "SSOW_LF_GWS_PENDWQP     0x%" PRIx64 "\n",
1137                 otx2_read64(base + SSOW_LF_GWS_PENDWQP));
1138         fprintf(f, "SSOW_LF_GWS_PENDSTATE   0x%" PRIx64 "\n",
1139                 otx2_read64(base + SSOW_LF_GWS_PENDSTATE));
1140         fprintf(f, "SSOW_LF_GWS_NW_TIM      0x%" PRIx64 "\n",
1141                 otx2_read64(base + SSOW_LF_GWS_NW_TIM));
1142         fprintf(f, "SSOW_LF_GWS_TAG         0x%" PRIx64 "\n",
1143                 otx2_read64(base + SSOW_LF_GWS_TAG));
1144         fprintf(f, "SSOW_LF_GWS_WQP         0x%" PRIx64 "\n",
1145                 otx2_read64(base + SSOW_LF_GWS_TAG));
1146         fprintf(f, "SSOW_LF_GWS_SWTP        0x%" PRIx64 "\n",
1147                 otx2_read64(base + SSOW_LF_GWS_SWTP));
1148         fprintf(f, "SSOW_LF_GWS_PENDTAG     0x%" PRIx64 "\n",
1149                 otx2_read64(base + SSOW_LF_GWS_PENDTAG));
1150 }
1151
1152 static void
1153 ssoggrp_dump(uintptr_t base, FILE *f)
1154 {
1155         fprintf(f, "SSO_LF_GGRP Base addr   0x%" PRIx64 "\n", (uint64_t)base);
1156         fprintf(f, "SSO_LF_GGRP_QCTL        0x%" PRIx64 "\n",
1157                 otx2_read64(base + SSO_LF_GGRP_QCTL));
1158         fprintf(f, "SSO_LF_GGRP_XAQ_CNT     0x%" PRIx64 "\n",
1159                 otx2_read64(base + SSO_LF_GGRP_XAQ_CNT));
1160         fprintf(f, "SSO_LF_GGRP_INT_THR     0x%" PRIx64 "\n",
1161                 otx2_read64(base + SSO_LF_GGRP_INT_THR));
1162         fprintf(f, "SSO_LF_GGRP_INT_CNT     0x%" PRIX64 "\n",
1163                 otx2_read64(base + SSO_LF_GGRP_INT_CNT));
1164         fprintf(f, "SSO_LF_GGRP_AQ_CNT      0x%" PRIX64 "\n",
1165                 otx2_read64(base + SSO_LF_GGRP_AQ_CNT));
1166         fprintf(f, "SSO_LF_GGRP_AQ_THR      0x%" PRIX64 "\n",
1167                 otx2_read64(base + SSO_LF_GGRP_AQ_THR));
1168         fprintf(f, "SSO_LF_GGRP_MISC_CNT    0x%" PRIx64 "\n",
1169                 otx2_read64(base + SSO_LF_GGRP_MISC_CNT));
1170 }
1171
1172 static void
1173 otx2_sso_dump(struct rte_eventdev *event_dev, FILE *f)
1174 {
1175         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1176         uint8_t queue;
1177         uint8_t port;
1178
1179         fprintf(f, "[%s] SSO running in [%s] mode\n", __func__, dev->dual_ws ?
1180                 "dual_ws" : "single_ws");
1181         /* Dump SSOW registers */
1182         for (port = 0; port < dev->nb_event_ports; port++) {
1183                 if (dev->dual_ws) {
1184                         struct otx2_ssogws_dual *ws =
1185                                 event_dev->data->ports[port];
1186
1187                         fprintf(f, "[%s] SSO dual workslot[%d] vws[%d] dump\n",
1188                                 __func__, port, 0);
1189                         ssogws_dump((struct otx2_ssogws *)&ws->ws_state[0], f);
1190                         fprintf(f, "[%s]SSO dual workslot[%d] vws[%d] dump\n",
1191                                 __func__, port, 1);
1192                         ssogws_dump((struct otx2_ssogws *)&ws->ws_state[1], f);
1193                 } else {
1194                         fprintf(f, "[%s]SSO single workslot[%d] dump\n",
1195                                 __func__, port);
1196                         ssogws_dump(event_dev->data->ports[port], f);
1197                 }
1198         }
1199
1200         /* Dump SSO registers */
1201         for (queue = 0; queue < dev->nb_event_queues; queue++) {
1202                 fprintf(f, "[%s]SSO group[%d] dump\n", __func__, queue);
1203                 if (dev->dual_ws) {
1204                         struct otx2_ssogws_dual *ws = event_dev->data->ports[0];
1205                         ssoggrp_dump(ws->grps_base[queue], f);
1206                 } else {
1207                         struct otx2_ssogws *ws = event_dev->data->ports[0];
1208                         ssoggrp_dump(ws->grps_base[queue], f);
1209                 }
1210         }
1211 }
1212
1213 static void
1214 otx2_handle_event(void *arg, struct rte_event event)
1215 {
1216         struct rte_eventdev *event_dev = arg;
1217
1218         if (event_dev->dev_ops->dev_stop_flush != NULL)
1219                 event_dev->dev_ops->dev_stop_flush(event_dev->data->dev_id,
1220                                 event, event_dev->data->dev_stop_flush_arg);
1221 }
1222
1223 static void
1224 sso_qos_cfg(struct rte_eventdev *event_dev)
1225 {
1226         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1227         struct sso_grp_qos_cfg *req;
1228         uint16_t i;
1229
1230         for (i = 0; i < dev->qos_queue_cnt; i++) {
1231                 uint8_t xaq_prcnt = dev->qos_parse_data[i].xaq_prcnt;
1232                 uint8_t iaq_prcnt = dev->qos_parse_data[i].iaq_prcnt;
1233                 uint8_t taq_prcnt = dev->qos_parse_data[i].taq_prcnt;
1234
1235                 if (dev->qos_parse_data[i].queue >= dev->nb_event_queues)
1236                         continue;
1237
1238                 req = otx2_mbox_alloc_msg_sso_grp_qos_config(dev->mbox);
1239                 req->xaq_limit = (dev->nb_xaq_cfg *
1240                                   (xaq_prcnt ? xaq_prcnt : 100)) / 100;
1241                 req->taq_thr = (SSO_HWGRP_IAQ_MAX_THR_MASK *
1242                                 (iaq_prcnt ? iaq_prcnt : 100)) / 100;
1243                 req->iaq_thr = (SSO_HWGRP_TAQ_MAX_THR_MASK *
1244                                 (taq_prcnt ? taq_prcnt : 100)) / 100;
1245         }
1246
1247         if (dev->qos_queue_cnt)
1248                 otx2_mbox_process(dev->mbox);
1249 }
1250
1251 static void
1252 sso_cleanup(struct rte_eventdev *event_dev, uint8_t enable)
1253 {
1254         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1255         uint16_t i;
1256
1257         for (i = 0; i < dev->nb_event_ports; i++) {
1258                 if (dev->dual_ws) {
1259                         struct otx2_ssogws_dual *ws;
1260
1261                         ws = event_dev->data->ports[i];
1262                         ssogws_reset((struct otx2_ssogws *)&ws->ws_state[0]);
1263                         ssogws_reset((struct otx2_ssogws *)&ws->ws_state[1]);
1264                         ws->swtag_req = 0;
1265                         ws->vws = 0;
1266                         ws->ws_state[0].cur_grp = 0;
1267                         ws->ws_state[0].cur_tt = SSO_SYNC_EMPTY;
1268                         ws->ws_state[1].cur_grp = 0;
1269                         ws->ws_state[1].cur_tt = SSO_SYNC_EMPTY;
1270                 } else {
1271                         struct otx2_ssogws *ws;
1272
1273                         ws = event_dev->data->ports[i];
1274                         ssogws_reset(ws);
1275                         ws->swtag_req = 0;
1276                         ws->cur_grp = 0;
1277                         ws->cur_tt = SSO_SYNC_EMPTY;
1278                 }
1279         }
1280
1281         rte_mb();
1282         if (dev->dual_ws) {
1283                 struct otx2_ssogws_dual *ws = event_dev->data->ports[0];
1284                 struct otx2_ssogws temp_ws;
1285
1286                 memcpy(&temp_ws, &ws->ws_state[0],
1287                        sizeof(struct otx2_ssogws_state));
1288                 for (i = 0; i < dev->nb_event_queues; i++) {
1289                         /* Consume all the events through HWS0 */
1290                         ssogws_flush_events(&temp_ws, i, ws->grps_base[i],
1291                                             otx2_handle_event, event_dev);
1292                         /* Enable/Disable SSO GGRP */
1293                         otx2_write64(enable, ws->grps_base[i] +
1294                                      SSO_LF_GGRP_QCTL);
1295                 }
1296                 ws->ws_state[0].cur_grp = 0;
1297                 ws->ws_state[0].cur_tt = SSO_SYNC_EMPTY;
1298         } else {
1299                 struct otx2_ssogws *ws = event_dev->data->ports[0];
1300
1301                 for (i = 0; i < dev->nb_event_queues; i++) {
1302                         /* Consume all the events through HWS0 */
1303                         ssogws_flush_events(ws, i, ws->grps_base[i],
1304                                             otx2_handle_event, event_dev);
1305                         /* Enable/Disable SSO GGRP */
1306                         otx2_write64(enable, ws->grps_base[i] +
1307                                      SSO_LF_GGRP_QCTL);
1308                 }
1309                 ws->cur_grp = 0;
1310                 ws->cur_tt = SSO_SYNC_EMPTY;
1311         }
1312
1313         /* reset SSO GWS cache */
1314         otx2_mbox_alloc_msg_sso_ws_cache_inv(dev->mbox);
1315         otx2_mbox_process(dev->mbox);
1316 }
1317
1318 int
1319 sso_xae_reconfigure(struct rte_eventdev *event_dev)
1320 {
1321         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1322         struct rte_mempool *prev_xaq_pool;
1323         int rc = 0;
1324
1325         if (event_dev->data->dev_started)
1326                 sso_cleanup(event_dev, 0);
1327
1328         prev_xaq_pool = dev->xaq_pool;
1329         dev->xaq_pool = NULL;
1330         sso_xaq_allocate(dev);
1331         rc = sso_ggrp_alloc_xaq(dev);
1332         if (rc < 0) {
1333                 otx2_err("Failed to alloc xaq to ggrp %d", rc);
1334                 rte_mempool_free(prev_xaq_pool);
1335                 return rc;
1336         }
1337
1338         rte_mempool_free(prev_xaq_pool);
1339         rte_mb();
1340         if (event_dev->data->dev_started)
1341                 sso_cleanup(event_dev, 1);
1342
1343         return 0;
1344 }
1345
1346 static int
1347 otx2_sso_start(struct rte_eventdev *event_dev)
1348 {
1349         sso_func_trace();
1350         sso_qos_cfg(event_dev);
1351         sso_cleanup(event_dev, 1);
1352         sso_fastpath_fns_set(event_dev);
1353
1354         return 0;
1355 }
1356
1357 static void
1358 otx2_sso_stop(struct rte_eventdev *event_dev)
1359 {
1360         sso_func_trace();
1361         sso_cleanup(event_dev, 0);
1362         rte_mb();
1363 }
1364
1365 static int
1366 otx2_sso_close(struct rte_eventdev *event_dev)
1367 {
1368         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1369         uint8_t all_queues[RTE_EVENT_MAX_QUEUES_PER_DEV];
1370         uint16_t i;
1371
1372         if (!dev->configured)
1373                 return 0;
1374
1375         sso_unregister_irqs(event_dev);
1376
1377         for (i = 0; i < dev->nb_event_queues; i++)
1378                 all_queues[i] = i;
1379
1380         for (i = 0; i < dev->nb_event_ports; i++)
1381                 otx2_sso_port_unlink(event_dev, event_dev->data->ports[i],
1382                                      all_queues, dev->nb_event_queues);
1383
1384         sso_lf_teardown(dev, SSO_LF_GGRP);
1385         sso_lf_teardown(dev, SSO_LF_GWS);
1386         dev->nb_event_ports = 0;
1387         dev->nb_event_queues = 0;
1388         rte_mempool_free(dev->xaq_pool);
1389         rte_memzone_free(rte_memzone_lookup(OTX2_SSO_FC_NAME));
1390
1391         return 0;
1392 }
1393
1394 /* Initialize and register event driver with DPDK Application */
1395 static struct rte_eventdev_ops otx2_sso_ops = {
1396         .dev_infos_get    = otx2_sso_info_get,
1397         .dev_configure    = otx2_sso_configure,
1398         .queue_def_conf   = otx2_sso_queue_def_conf,
1399         .queue_setup      = otx2_sso_queue_setup,
1400         .queue_release    = otx2_sso_queue_release,
1401         .port_def_conf    = otx2_sso_port_def_conf,
1402         .port_setup       = otx2_sso_port_setup,
1403         .port_release     = otx2_sso_port_release,
1404         .port_link        = otx2_sso_port_link,
1405         .port_unlink      = otx2_sso_port_unlink,
1406         .timeout_ticks    = otx2_sso_timeout_ticks,
1407
1408         .eth_rx_adapter_caps_get  = otx2_sso_rx_adapter_caps_get,
1409         .eth_rx_adapter_queue_add = otx2_sso_rx_adapter_queue_add,
1410         .eth_rx_adapter_queue_del = otx2_sso_rx_adapter_queue_del,
1411         .eth_rx_adapter_start = otx2_sso_rx_adapter_start,
1412         .eth_rx_adapter_stop = otx2_sso_rx_adapter_stop,
1413
1414         .timer_adapter_caps_get = otx2_tim_caps_get,
1415
1416         .xstats_get       = otx2_sso_xstats_get,
1417         .xstats_reset     = otx2_sso_xstats_reset,
1418         .xstats_get_names = otx2_sso_xstats_get_names,
1419
1420         .dump             = otx2_sso_dump,
1421         .dev_start        = otx2_sso_start,
1422         .dev_stop         = otx2_sso_stop,
1423         .dev_close        = otx2_sso_close,
1424         .dev_selftest     = otx2_sso_selftest,
1425 };
1426
1427 #define OTX2_SSO_XAE_CNT        "xae_cnt"
1428 #define OTX2_SSO_SINGLE_WS      "single_ws"
1429 #define OTX2_SSO_GGRP_QOS       "qos"
1430 #define OTX2_SSO_SELFTEST       "selftest"
1431
1432 static void
1433 parse_queue_param(char *value, void *opaque)
1434 {
1435         struct otx2_sso_qos queue_qos = {0};
1436         uint8_t *val = (uint8_t *)&queue_qos;
1437         struct otx2_sso_evdev *dev = opaque;
1438         char *tok = strtok(value, "-");
1439
1440         if (!strlen(value))
1441                 return;
1442
1443         while (tok != NULL) {
1444                 *val = atoi(tok);
1445                 tok = strtok(NULL, "-");
1446                 val++;
1447         }
1448
1449         if (val != (&queue_qos.iaq_prcnt + 1)) {
1450                 otx2_err("Invalid QoS parameter expected [Qx-XAQ-TAQ-IAQ]");
1451                 return;
1452         }
1453
1454         dev->qos_queue_cnt++;
1455         dev->qos_parse_data = rte_realloc(dev->qos_parse_data,
1456                                           sizeof(struct otx2_sso_qos) *
1457                                           dev->qos_queue_cnt, 0);
1458         dev->qos_parse_data[dev->qos_queue_cnt - 1] = queue_qos;
1459 }
1460
1461 static void
1462 parse_qos_list(const char *value, void *opaque)
1463 {
1464         char *s = strdup(value);
1465         char *start = NULL;
1466         char *end = NULL;
1467         char *f = s;
1468
1469         while (*s) {
1470                 if (*s == '[')
1471                         start = s;
1472                 else if (*s == ']')
1473                         end = s;
1474
1475                 if (start < end && *start) {
1476                         *end = 0;
1477                         parse_queue_param(start + 1, opaque);
1478                         s = end;
1479                         start = end;
1480                 }
1481                 s++;
1482         }
1483
1484         free(f);
1485 }
1486
1487 static int
1488 parse_sso_kvargs_dict(const char *key, const char *value, void *opaque)
1489 {
1490         RTE_SET_USED(key);
1491
1492         /* Dict format [Qx-XAQ-TAQ-IAQ][Qz-XAQ-TAQ-IAQ] use '-' cause ','
1493          * isn't allowed. Everything is expressed in percentages, 0 represents
1494          * default.
1495          */
1496         parse_qos_list(value, opaque);
1497
1498         return 0;
1499 }
1500
1501 static void
1502 sso_parse_devargs(struct otx2_sso_evdev *dev, struct rte_devargs *devargs)
1503 {
1504         struct rte_kvargs *kvlist;
1505         uint8_t single_ws = 0;
1506
1507         if (devargs == NULL)
1508                 return;
1509         kvlist = rte_kvargs_parse(devargs->args, NULL);
1510         if (kvlist == NULL)
1511                 return;
1512
1513         rte_kvargs_process(kvlist, OTX2_SSO_SELFTEST, &parse_kvargs_flag,
1514                            &dev->selftest);
1515         rte_kvargs_process(kvlist, OTX2_SSO_XAE_CNT, &parse_kvargs_value,
1516                            &dev->xae_cnt);
1517         rte_kvargs_process(kvlist, OTX2_SSO_SINGLE_WS, &parse_kvargs_flag,
1518                            &single_ws);
1519         rte_kvargs_process(kvlist, OTX2_SSO_GGRP_QOS, &parse_sso_kvargs_dict,
1520                            dev);
1521
1522         dev->dual_ws = !single_ws;
1523         rte_kvargs_free(kvlist);
1524 }
1525
1526 static int
1527 otx2_sso_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
1528 {
1529         return rte_event_pmd_pci_probe(pci_drv, pci_dev,
1530                                        sizeof(struct otx2_sso_evdev),
1531                                        otx2_sso_init);
1532 }
1533
1534 static int
1535 otx2_sso_remove(struct rte_pci_device *pci_dev)
1536 {
1537         return rte_event_pmd_pci_remove(pci_dev, otx2_sso_fini);
1538 }
1539
1540 static const struct rte_pci_id pci_sso_map[] = {
1541         {
1542                 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
1543                                PCI_DEVID_OCTEONTX2_RVU_SSO_TIM_PF)
1544         },
1545         {
1546                 .vendor_id = 0,
1547         },
1548 };
1549
1550 static struct rte_pci_driver pci_sso = {
1551         .id_table = pci_sso_map,
1552         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA,
1553         .probe = otx2_sso_probe,
1554         .remove = otx2_sso_remove,
1555 };
1556
1557 int
1558 otx2_sso_init(struct rte_eventdev *event_dev)
1559 {
1560         struct free_rsrcs_rsp *rsrc_cnt;
1561         struct rte_pci_device *pci_dev;
1562         struct otx2_sso_evdev *dev;
1563         int rc;
1564
1565         event_dev->dev_ops = &otx2_sso_ops;
1566         /* For secondary processes, the primary has done all the work */
1567         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1568                 sso_fastpath_fns_set(event_dev);
1569                 return 0;
1570         }
1571
1572         dev = sso_pmd_priv(event_dev);
1573
1574         pci_dev = container_of(event_dev->dev, struct rte_pci_device, device);
1575
1576         /* Initialize the base otx2_dev object */
1577         rc = otx2_dev_init(pci_dev, dev);
1578         if (rc < 0) {
1579                 otx2_err("Failed to initialize otx2_dev rc=%d", rc);
1580                 goto error;
1581         }
1582
1583         /* Get SSO and SSOW MSIX rsrc cnt */
1584         otx2_mbox_alloc_msg_free_rsrc_cnt(dev->mbox);
1585         rc = otx2_mbox_process_msg(dev->mbox, (void *)&rsrc_cnt);
1586         if (rc < 0) {
1587                 otx2_err("Unable to get free rsrc count");
1588                 goto otx2_dev_uninit;
1589         }
1590         otx2_sso_dbg("SSO %d SSOW %d NPA %d provisioned", rsrc_cnt->sso,
1591                      rsrc_cnt->ssow, rsrc_cnt->npa);
1592
1593         dev->max_event_ports = RTE_MIN(rsrc_cnt->ssow, OTX2_SSO_MAX_VHWS);
1594         dev->max_event_queues = RTE_MIN(rsrc_cnt->sso, OTX2_SSO_MAX_VHGRP);
1595         /* Grab the NPA LF if required */
1596         rc = otx2_npa_lf_init(pci_dev, dev);
1597         if (rc < 0) {
1598                 otx2_err("Unable to init NPA lf. It might not be provisioned");
1599                 goto otx2_dev_uninit;
1600         }
1601
1602         dev->drv_inited = true;
1603         dev->is_timeout_deq = 0;
1604         dev->min_dequeue_timeout_ns = USEC2NSEC(1);
1605         dev->max_dequeue_timeout_ns = USEC2NSEC(0x3FF);
1606         dev->max_num_events = -1;
1607         dev->nb_event_queues = 0;
1608         dev->nb_event_ports = 0;
1609
1610         if (!dev->max_event_ports || !dev->max_event_queues) {
1611                 otx2_err("Not enough eventdev resource queues=%d ports=%d",
1612                          dev->max_event_queues, dev->max_event_ports);
1613                 rc = -ENODEV;
1614                 goto otx2_npa_lf_uninit;
1615         }
1616
1617         dev->dual_ws = 1;
1618         sso_parse_devargs(dev, pci_dev->device.devargs);
1619         if (dev->dual_ws) {
1620                 otx2_sso_dbg("Using dual workslot mode");
1621                 dev->max_event_ports = dev->max_event_ports / 2;
1622         } else {
1623                 otx2_sso_dbg("Using single workslot mode");
1624         }
1625
1626         otx2_sso_pf_func_set(dev->pf_func);
1627         otx2_sso_dbg("Initializing %s max_queues=%d max_ports=%d",
1628                      event_dev->data->name, dev->max_event_queues,
1629                      dev->max_event_ports);
1630         if (dev->selftest) {
1631                 event_dev->dev->driver = &pci_sso.driver;
1632                 event_dev->dev_ops->dev_selftest();
1633         }
1634
1635         otx2_tim_init(pci_dev, (struct otx2_dev *)dev);
1636
1637         return 0;
1638
1639 otx2_npa_lf_uninit:
1640         otx2_npa_lf_fini();
1641 otx2_dev_uninit:
1642         otx2_dev_fini(pci_dev, dev);
1643 error:
1644         return rc;
1645 }
1646
1647 int
1648 otx2_sso_fini(struct rte_eventdev *event_dev)
1649 {
1650         struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev);
1651         struct rte_pci_device *pci_dev;
1652
1653         /* For secondary processes, nothing to be done */
1654         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1655                 return 0;
1656
1657         pci_dev = container_of(event_dev->dev, struct rte_pci_device, device);
1658
1659         if (!dev->drv_inited)
1660                 goto dev_fini;
1661
1662         dev->drv_inited = false;
1663         otx2_npa_lf_fini();
1664
1665 dev_fini:
1666         if (otx2_npa_lf_active(dev)) {
1667                 otx2_info("Common resource in use by other devices");
1668                 return -EAGAIN;
1669         }
1670
1671         otx2_tim_fini();
1672         otx2_dev_fini(pci_dev, dev);
1673
1674         return 0;
1675 }
1676
1677 RTE_PMD_REGISTER_PCI(event_octeontx2, pci_sso);
1678 RTE_PMD_REGISTER_PCI_TABLE(event_octeontx2, pci_sso_map);
1679 RTE_PMD_REGISTER_KMOD_DEP(event_octeontx2, "vfio-pci");
1680 RTE_PMD_REGISTER_PARAM_STRING(event_octeontx2, OTX2_SSO_XAE_CNT "=<int>"
1681                               OTX2_SSO_SINGLE_WS "=1"
1682                               OTX2_SSO_GGRP_QOS "=<string>"
1683                               OTX2_SSO_SELFTEST "=1");