drivers/event/cnxk/cn9k_eventdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include "cnxk_eventdev.h"
   6
   7 #define CN9K_DUAL_WS_NB_WS          2
   8 #define CN9K_DUAL_WS_PAIR_ID(x, id) (((x)*CN9K_DUAL_WS_NB_WS) + id)
   9
  10 static void
  11 cn9k_init_hws_ops(struct cn9k_sso_hws_state *ws, uintptr_t base)
  12 {
  13         ws->tag_op = base + SSOW_LF_GWS_TAG;
  14         ws->wqp_op = base + SSOW_LF_GWS_WQP;
  15         ws->getwrk_op = base + SSOW_LF_GWS_OP_GET_WORK0;
  16         ws->swtag_flush_op = base + SSOW_LF_GWS_OP_SWTAG_FLUSH;
  17         ws->swtag_norm_op = base + SSOW_LF_GWS_OP_SWTAG_NORM;
  18         ws->swtag_desched_op = base + SSOW_LF_GWS_OP_SWTAG_DESCHED;
  19 }
  20
  21 static int
  22 cn9k_sso_hws_link(void *arg, void *port, uint16_t *map, uint16_t nb_link)
  23 {
  24         struct cnxk_sso_evdev *dev = arg;
  25         struct cn9k_sso_hws_dual *dws;
  26         struct cn9k_sso_hws *ws;
  27         int rc;
  28
  29         if (dev->dual_ws) {
  30                 dws = port;
  31                 rc = roc_sso_hws_link(&dev->sso,
  32                                       CN9K_DUAL_WS_PAIR_ID(dws->hws_id, 0), map,
  33                                       nb_link);
  34                 rc |= roc_sso_hws_link(&dev->sso,
  35                                        CN9K_DUAL_WS_PAIR_ID(dws->hws_id, 1),
  36                                        map, nb_link);
  37         } else {
  38                 ws = port;
  39                 rc = roc_sso_hws_link(&dev->sso, ws->hws_id, map, nb_link);
  40         }
  41
  42         return rc;
  43 }
  44
  45 static int
  46 cn9k_sso_hws_unlink(void *arg, void *port, uint16_t *map, uint16_t nb_link)
  47 {
  48         struct cnxk_sso_evdev *dev = arg;
  49         struct cn9k_sso_hws_dual *dws;
  50         struct cn9k_sso_hws *ws;
  51         int rc;
  52
  53         if (dev->dual_ws) {
  54                 dws = port;
  55                 rc = roc_sso_hws_unlink(&dev->sso,
  56                                         CN9K_DUAL_WS_PAIR_ID(dws->hws_id, 0),
  57                                         map, nb_link);
  58                 rc |= roc_sso_hws_unlink(&dev->sso,
  59                                          CN9K_DUAL_WS_PAIR_ID(dws->hws_id, 1),
  60                                          map, nb_link);
  61         } else {
  62                 ws = port;
  63                 rc = roc_sso_hws_unlink(&dev->sso, ws->hws_id, map, nb_link);
  64         }
  65
  66         return rc;
  67 }
  68
  69 static void
  70 cn9k_sso_hws_setup(void *arg, void *hws, uintptr_t *grps_base)
  71 {
  72         struct cnxk_sso_evdev *dev = arg;
  73         struct cn9k_sso_hws_dual *dws;
  74         struct cn9k_sso_hws *ws;
  75         uint64_t val;
  76
  77         /* Set get_work tmo for HWS */
  78         val = NSEC2USEC(dev->deq_tmo_ns) - 1;
  79         if (dev->dual_ws) {
  80                 dws = hws;
  81                 rte_memcpy(dws->grps_base, grps_base,
  82                            sizeof(uintptr_t) * CNXK_SSO_MAX_HWGRP);
  83                 dws->fc_mem = dev->fc_mem;
  84                 dws->xaq_lmt = dev->xaq_lmt;
  85
  86                 plt_write64(val, dws->base[0] + SSOW_LF_GWS_NW_TIM);
  87                 plt_write64(val, dws->base[1] + SSOW_LF_GWS_NW_TIM);
  88         } else {
  89                 ws = hws;
  90                 rte_memcpy(ws->grps_base, grps_base,
  91                            sizeof(uintptr_t) * CNXK_SSO_MAX_HWGRP);
  92                 ws->fc_mem = dev->fc_mem;
  93                 ws->xaq_lmt = dev->xaq_lmt;
  94
  95                 plt_write64(val, ws->base + SSOW_LF_GWS_NW_TIM);
  96         }
  97 }
  98
  99 static void
 100 cn9k_sso_hws_release(void *arg, void *hws)
 101 {
 102         struct cnxk_sso_evdev *dev = arg;
 103         struct cn9k_sso_hws_dual *dws;
 104         struct cn9k_sso_hws *ws;
 105         int i;
 106
 107         if (dev->dual_ws) {
 108                 dws = hws;
 109                 for (i = 0; i < dev->nb_event_queues; i++) {
 110                         roc_sso_hws_unlink(&dev->sso,
 111                                            CN9K_DUAL_WS_PAIR_ID(dws->hws_id, 0),
 112                                            (uint16_t *)&i, 1);
 113                         roc_sso_hws_unlink(&dev->sso,
 114                                            CN9K_DUAL_WS_PAIR_ID(dws->hws_id, 1),
 115                                            (uint16_t *)&i, 1);
 116                 }
 117                 memset(dws, 0, sizeof(*dws));
 118         } else {
 119                 ws = hws;
 120                 for (i = 0; i < dev->nb_event_queues; i++)
 121                         roc_sso_hws_unlink(&dev->sso, ws->hws_id,
 122                                            (uint16_t *)&i, 1);
 123                 memset(ws, 0, sizeof(*ws));
 124         }
 125 }
 126
 127 static void
 128 cn9k_sso_set_rsrc(void *arg)
 129 {
 130         struct cnxk_sso_evdev *dev = arg;
 131
 132         if (dev->dual_ws)
 133                 dev->max_event_ports = dev->sso.max_hws / CN9K_DUAL_WS_NB_WS;
 134         else
 135                 dev->max_event_ports = dev->sso.max_hws;
 136         dev->max_event_queues =
 137                 dev->sso.max_hwgrp > RTE_EVENT_MAX_QUEUES_PER_DEV ?
 138                               RTE_EVENT_MAX_QUEUES_PER_DEV :
 139                               dev->sso.max_hwgrp;
 140 }
 141
 142 static int
 143 cn9k_sso_rsrc_init(void *arg, uint8_t hws, uint8_t hwgrp)
 144 {
 145         struct cnxk_sso_evdev *dev = arg;
 146
 147         if (dev->dual_ws)
 148                 hws = hws * CN9K_DUAL_WS_NB_WS;
 149
 150         return roc_sso_rsrc_init(&dev->sso, hws, hwgrp);
 151 }
 152
 153 static void *
 154 cn9k_sso_init_hws_mem(void *arg, uint8_t port_id)
 155 {
 156         struct cnxk_sso_evdev *dev = arg;
 157         struct cn9k_sso_hws_dual *dws;
 158         struct cn9k_sso_hws *ws;
 159         void *data;
 160
 161         if (dev->dual_ws) {
 162                 dws = rte_zmalloc("cn9k_dual_ws",
 163                                   sizeof(struct cn9k_sso_hws_dual) +
 164                                           RTE_CACHE_LINE_SIZE,
 165                                   RTE_CACHE_LINE_SIZE);
 166                 if (dws == NULL) {
 167                         plt_err("Failed to alloc memory for port=%d", port_id);
 168                         return NULL;
 169                 }
 170
 171                 dws = RTE_PTR_ADD(dws, sizeof(struct cnxk_sso_hws_cookie));
 172                 dws->base[0] = roc_sso_hws_base_get(
 173                         &dev->sso, CN9K_DUAL_WS_PAIR_ID(port_id, 0));
 174                 dws->base[1] = roc_sso_hws_base_get(
 175                         &dev->sso, CN9K_DUAL_WS_PAIR_ID(port_id, 1));
 176                 cn9k_init_hws_ops(&dws->ws_state[0], dws->base[0]);
 177                 cn9k_init_hws_ops(&dws->ws_state[1], dws->base[1]);
 178                 dws->hws_id = port_id;
 179                 dws->swtag_req = 0;
 180                 dws->vws = 0;
 181
 182                 data = dws;
 183         } else {
 184                 /* Allocate event port memory */
 185                 ws = rte_zmalloc("cn9k_ws",
 186                                  sizeof(struct cn9k_sso_hws) +
 187                                          RTE_CACHE_LINE_SIZE,
 188                                  RTE_CACHE_LINE_SIZE);
 189                 if (ws == NULL) {
 190                         plt_err("Failed to alloc memory for port=%d", port_id);
 191                         return NULL;
 192                 }
 193
 194                 /* First cache line is reserved for cookie */
 195                 ws = RTE_PTR_ADD(ws, sizeof(struct cnxk_sso_hws_cookie));
 196                 ws->base = roc_sso_hws_base_get(&dev->sso, port_id);
 197                 cn9k_init_hws_ops((struct cn9k_sso_hws_state *)ws, ws->base);
 198                 ws->hws_id = port_id;
 199                 ws->swtag_req = 0;
 200
 201                 data = ws;
 202         }
 203
 204         return data;
 205 }
 206
 207 static void
 208 cn9k_sso_info_get(struct rte_eventdev *event_dev,
 209                   struct rte_event_dev_info *dev_info)
 210 {
 211         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 212
 213         dev_info->driver_name = RTE_STR(EVENTDEV_NAME_CN9K_PMD);
 214         cnxk_sso_info_get(dev, dev_info);
 215 }
 216
 217 static int
 218 cn9k_sso_dev_configure(const struct rte_eventdev *event_dev)
 219 {
 220         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 221         int rc;
 222
 223         rc = cnxk_sso_dev_validate(event_dev);
 224         if (rc < 0) {
 225                 plt_err("Invalid event device configuration");
 226                 return -EINVAL;
 227         }
 228
 229         roc_sso_rsrc_fini(&dev->sso);
 230
 231         rc = cn9k_sso_rsrc_init(dev, dev->nb_event_ports, dev->nb_event_queues);
 232         if (rc < 0) {
 233                 plt_err("Failed to initialize SSO resources");
 234                 return -ENODEV;
 235         }
 236
 237         rc = cnxk_sso_xaq_allocate(dev);
 238         if (rc < 0)
 239                 goto cnxk_rsrc_fini;
 240
 241         rc = cnxk_setup_event_ports(event_dev, cn9k_sso_init_hws_mem,
 242                                     cn9k_sso_hws_setup);
 243         if (rc < 0)
 244                 goto cnxk_rsrc_fini;
 245
 246         /* Restore any prior port-queue mapping. */
 247         cnxk_sso_restore_links(event_dev, cn9k_sso_hws_link);
 248
 249         dev->configured = 1;
 250         rte_mb();
 251
 252         return 0;
 253 cnxk_rsrc_fini:
 254         roc_sso_rsrc_fini(&dev->sso);
 255         dev->nb_event_ports = 0;
 256         return rc;
 257 }
 258
 259 static int
 260 cn9k_sso_port_setup(struct rte_eventdev *event_dev, uint8_t port_id,
 261                     const struct rte_event_port_conf *port_conf)
 262 {
 263
 264         RTE_SET_USED(port_conf);
 265         return cnxk_sso_port_setup(event_dev, port_id, cn9k_sso_hws_setup);
 266 }
 267
 268 static void
 269 cn9k_sso_port_release(void *port)
 270 {
 271         struct cnxk_sso_hws_cookie *gws_cookie = cnxk_sso_hws_get_cookie(port);
 272         struct cnxk_sso_evdev *dev;
 273
 274         if (port == NULL)
 275                 return;
 276
 277         dev = cnxk_sso_pmd_priv(gws_cookie->event_dev);
 278         if (!gws_cookie->configured)
 279                 goto free;
 280
 281         cn9k_sso_hws_release(dev, port);
 282         memset(gws_cookie, 0, sizeof(*gws_cookie));
 283 free:
 284         rte_free(gws_cookie);
 285 }
 286
 287 static int
 288 cn9k_sso_port_link(struct rte_eventdev *event_dev, void *port,
 289                    const uint8_t queues[], const uint8_t priorities[],
 290                    uint16_t nb_links)
 291 {
 292         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 293         uint16_t hwgrp_ids[nb_links];
 294         uint16_t link;
 295
 296         RTE_SET_USED(priorities);
 297         for (link = 0; link < nb_links; link++)
 298                 hwgrp_ids[link] = queues[link];
 299         nb_links = cn9k_sso_hws_link(dev, port, hwgrp_ids, nb_links);
 300
 301         return (int)nb_links;
 302 }
 303
 304 static int
 305 cn9k_sso_port_unlink(struct rte_eventdev *event_dev, void *port,
 306                      uint8_t queues[], uint16_t nb_unlinks)
 307 {
 308         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 309         uint16_t hwgrp_ids[nb_unlinks];
 310         uint16_t unlink;
 311
 312         for (unlink = 0; unlink < nb_unlinks; unlink++)
 313                 hwgrp_ids[unlink] = queues[unlink];
 314         nb_unlinks = cn9k_sso_hws_unlink(dev, port, hwgrp_ids, nb_unlinks);
 315
 316         return (int)nb_unlinks;
 317 }
 318
 319 static struct rte_eventdev_ops cn9k_sso_dev_ops = {
 320         .dev_infos_get = cn9k_sso_info_get,
 321         .dev_configure = cn9k_sso_dev_configure,
 322         .queue_def_conf = cnxk_sso_queue_def_conf,
 323         .queue_setup = cnxk_sso_queue_setup,
 324         .queue_release = cnxk_sso_queue_release,
 325         .port_def_conf = cnxk_sso_port_def_conf,
 326         .port_setup = cn9k_sso_port_setup,
 327         .port_release = cn9k_sso_port_release,
 328         .port_link = cn9k_sso_port_link,
 329         .port_unlink = cn9k_sso_port_unlink,
 330         .timeout_ticks = cnxk_sso_timeout_ticks,
 331 };
 332
 333 static int
 334 cn9k_sso_init(struct rte_eventdev *event_dev)
 335 {
 336         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 337         int rc;
 338
 339         if (RTE_CACHE_LINE_SIZE != 128) {
 340                 plt_err("Driver not compiled for CN9K");
 341                 return -EFAULT;
 342         }
 343
 344         rc = roc_plt_init();
 345         if (rc < 0) {
 346                 plt_err("Failed to initialize platform model");
 347                 return rc;
 348         }
 349
 350         event_dev->dev_ops = &cn9k_sso_dev_ops;
 351         /* For secondary processes, the primary has done all the work */
 352         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 353                 return 0;
 354
 355         rc = cnxk_sso_init(event_dev);
 356         if (rc < 0)
 357                 return rc;
 358
 359         cn9k_sso_set_rsrc(cnxk_sso_pmd_priv(event_dev));
 360         if (!dev->max_event_ports || !dev->max_event_queues) {
 361                 plt_err("Not enough eventdev resource queues=%d ports=%d",
 362                         dev->max_event_queues, dev->max_event_ports);
 363                 cnxk_sso_fini(event_dev);
 364                 return -ENODEV;
 365         }
 366
 367         plt_sso_dbg("Initializing %s max_queues=%d max_ports=%d",
 368                     event_dev->data->name, dev->max_event_queues,
 369                     dev->max_event_ports);
 370
 371         return 0;
 372 }
 373
 374 static int
 375 cn9k_sso_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 376 {
 377         return rte_event_pmd_pci_probe(
 378                 pci_drv, pci_dev, sizeof(struct cnxk_sso_evdev), cn9k_sso_init);
 379 }
 380
 381 static const struct rte_pci_id cn9k_pci_sso_map[] = {
 382         {
 383                 .vendor_id = 0,
 384         },
 385 };
 386
 387 static struct rte_pci_driver cn9k_pci_sso = {
 388         .id_table = cn9k_pci_sso_map,
 389         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
 390         .probe = cn9k_sso_probe,
 391         .remove = cnxk_sso_remove,
 392 };
 393
 394 RTE_PMD_REGISTER_PCI(event_cn9k, cn9k_pci_sso);
 395 RTE_PMD_REGISTER_PCI_TABLE(event_cn9k, cn9k_pci_sso_map);
 396 RTE_PMD_REGISTER_KMOD_DEP(event_cn9k, "vfio-pci");
 397 RTE_PMD_REGISTER_PARAM_STRING(event_cn9k, CNXK_SSO_XAE_CNT "=<int>"
 398                               CNXK_SSO_GGRP_QOS "=<string>"
 399                               CN9K_SSO_SINGLE_WS "=1");