drivers/bus/dpaa/dpaa_bus.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright 2017 NXP
   4  *
   5  */
   6 /* System headers */
   7 #include <stdio.h>
   8 #include <inttypes.h>
   9 #include <unistd.h>
  10 #include <limits.h>
  11 #include <sched.h>
  12 #include <signal.h>
  13 #include <pthread.h>
  14 #include <sys/types.h>
  15 #include <sys/syscall.h>
  16
  17 #include <rte_byteorder.h>
  18 #include <rte_common.h>
  19 #include <rte_interrupts.h>
  20 #include <rte_log.h>
  21 #include <rte_debug.h>
  22 #include <rte_pci.h>
  23 #include <rte_atomic.h>
  24 #include <rte_branch_prediction.h>
  25 #include <rte_memory.h>
  26 #include <rte_tailq.h>
  27 #include <rte_eal.h>
  28 #include <rte_alarm.h>
  29 #include <rte_ether.h>
  30 #include <rte_ethdev_driver.h>
  31 #include <rte_malloc.h>
  32 #include <rte_ring.h>
  33 #include <rte_bus.h>
  34 #include <rte_mbuf_pool_ops.h>
  35
  36 #include <rte_dpaa_bus.h>
  37 #include <rte_dpaa_logs.h>
  38
  39 #include <fsl_usd.h>
  40 #include <fsl_qman.h>
  41 #include <fsl_bman.h>
  42 #include <of.h>
  43 #include <netcfg.h>
  44
  45 int dpaa_logtype_bus;
  46 int dpaa_logtype_mempool;
  47 int dpaa_logtype_pmd;
  48 int dpaa_logtype_eventdev;
  49
  50 struct rte_dpaa_bus rte_dpaa_bus;
  51 struct netcfg_info *dpaa_netcfg;
  52
  53 /* define a variable to hold the portal_key, once created.*/
  54 pthread_key_t dpaa_portal_key;
  55
  56 unsigned int dpaa_svr_family;
  57
  58 RTE_DEFINE_PER_LCORE(bool, _dpaa_io);
  59 RTE_DEFINE_PER_LCORE(struct dpaa_portal_dqrr, held_bufs);
  60
  61 static inline void
  62 dpaa_add_to_device_list(struct rte_dpaa_device *dev)
  63 {
  64         TAILQ_INSERT_TAIL(&rte_dpaa_bus.device_list, dev, next);
  65 }
  66
  67 static inline void
  68 dpaa_remove_from_device_list(struct rte_dpaa_device *dev)
  69 {
  70         TAILQ_INSERT_TAIL(&rte_dpaa_bus.device_list, dev, next);
  71 }
  72
  73 /*
  74  * Reads the SEC device from DTS
  75  * Returns -1 if SEC devices not available, 0 otherwise
  76  */
  77 static inline int
  78 dpaa_sec_available(void)
  79 {
  80         const struct device_node *caam_node;
  81
  82         for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
  83                 return 0;
  84         }
  85
  86         return -1;
  87 }
  88
  89 static void dpaa_clean_device_list(void);
  90
  91 static int
  92 dpaa_create_device_list(void)
  93 {
  94         int i;
  95         int ret;
  96         struct rte_dpaa_device *dev;
  97         struct fm_eth_port_cfg *cfg;
  98         struct fman_if *fman_intf;
  99
 100         /* Creating Ethernet Devices */
 101         for (i = 0; i < dpaa_netcfg->num_ethports; i++) {
 102                 dev = calloc(1, sizeof(struct rte_dpaa_device));
 103                 if (!dev) {
 104                         DPAA_BUS_LOG(ERR, "Failed to allocate ETH devices");
 105                         ret = -ENOMEM;
 106                         goto cleanup;
 107                 }
 108
 109                 cfg = &dpaa_netcfg->port_cfg[i];
 110                 fman_intf = cfg->fman_if;
 111
 112                 /* Device identifiers */
 113                 dev->id.fman_id = fman_intf->fman_idx + 1;
 114                 dev->id.mac_id = fman_intf->mac_idx;
 115                 dev->device_type = FSL_DPAA_ETH;
 116                 dev->id.dev_id = i;
 117
 118                 /* Create device name */
 119                 memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN);
 120                 sprintf(dev->name, "fm%d-mac%d", (fman_intf->fman_idx + 1),
 121                         fman_intf->mac_idx);
 122                 DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name);
 123                 dev->device.name = dev->name;
 124
 125                 dpaa_add_to_device_list(dev);
 126         }
 127
 128         rte_dpaa_bus.device_count = i;
 129
 130         /* Unlike case of ETH, RTE_LIBRTE_DPAA_MAX_CRYPTODEV SEC devices are
 131          * constantly created only if "sec" property is found in the device
 132          * tree. Logically there is no limit for number of devices (QI
 133          * interfaces) that can be created.
 134          */
 135
 136         if (dpaa_sec_available()) {
 137                 DPAA_BUS_LOG(INFO, "DPAA SEC devices are not available");
 138                 return 0;
 139         }
 140
 141         /* Creating SEC Devices */
 142         for (i = 0; i < RTE_LIBRTE_DPAA_MAX_CRYPTODEV; i++) {
 143                 dev = calloc(1, sizeof(struct rte_dpaa_device));
 144                 if (!dev) {
 145                         DPAA_BUS_LOG(ERR, "Failed to allocate SEC devices");
 146                         ret = -1;
 147                         goto cleanup;
 148                 }
 149
 150                 dev->device_type = FSL_DPAA_CRYPTO;
 151                 dev->id.dev_id = rte_dpaa_bus.device_count + i;
 152
 153                 /* Even though RTE_CRYPTODEV_NAME_MAX_LEN is valid length of
 154                  * crypto PMD, using RTE_ETH_NAME_MAX_LEN as that is the size
 155                  * allocated for dev->name/
 156                  */
 157                 memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN);
 158                 sprintf(dev->name, "dpaa-sec%d", i);
 159                 DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name);
 160
 161                 dpaa_add_to_device_list(dev);
 162         }
 163
 164         rte_dpaa_bus.device_count += i;
 165
 166         return 0;
 167
 168 cleanup:
 169         dpaa_clean_device_list();
 170         return ret;
 171 }
 172
 173 static void
 174 dpaa_clean_device_list(void)
 175 {
 176         struct rte_dpaa_device *dev = NULL;
 177         struct rte_dpaa_device *tdev = NULL;
 178
 179         TAILQ_FOREACH_SAFE(dev, &rte_dpaa_bus.device_list, next, tdev) {
 180                 TAILQ_REMOVE(&rte_dpaa_bus.device_list, dev, next);
 181                 free(dev);
 182                 dev = NULL;
 183         }
 184 }
 185
 186 /** XXX move this function into a separate file */
 187 static int
 188 _dpaa_portal_init(void *arg)
 189 {
 190         cpu_set_t cpuset;
 191         pthread_t id;
 192         uint32_t cpu = rte_lcore_id();
 193         int ret;
 194         struct dpaa_portal *dpaa_io_portal;
 195
 196         BUS_INIT_FUNC_TRACE();
 197
 198         if ((uint64_t)arg == 1 || cpu == LCORE_ID_ANY)
 199                 cpu = rte_get_master_lcore();
 200         /* if the core id is not supported */
 201         else
 202                 if (cpu >= RTE_MAX_LCORE)
 203                         return -1;
 204
 205         /* Set CPU affinity for this thread */
 206         CPU_ZERO(&cpuset);
 207         CPU_SET(cpu, &cpuset);
 208         id = pthread_self();
 209         ret = pthread_setaffinity_np(id, sizeof(cpu_set_t), &cpuset);
 210         if (ret) {
 211                 DPAA_BUS_LOG(ERR, "pthread_setaffinity_np failed on "
 212                         "core :%d with ret: %d", cpu, ret);
 213                 return ret;
 214         }
 215
 216         /* Initialise bman thread portals */
 217         ret = bman_thread_init();
 218         if (ret) {
 219                 DPAA_BUS_LOG(ERR, "bman_thread_init failed on "
 220                         "core %d with ret: %d", cpu, ret);
 221                 return ret;
 222         }
 223
 224         DPAA_BUS_LOG(DEBUG, "BMAN thread initialized");
 225
 226         /* Initialise qman thread portals */
 227         ret = qman_thread_init();
 228         if (ret) {
 229                 DPAA_BUS_LOG(ERR, "bman_thread_init failed on "
 230                         "core %d with ret: %d", cpu, ret);
 231                 bman_thread_finish();
 232                 return ret;
 233         }
 234
 235         DPAA_BUS_LOG(DEBUG, "QMAN thread initialized");
 236
 237         dpaa_io_portal = rte_malloc(NULL, sizeof(struct dpaa_portal),
 238                                     RTE_CACHE_LINE_SIZE);
 239         if (!dpaa_io_portal) {
 240                 DPAA_BUS_LOG(ERR, "Unable to allocate memory");
 241                 bman_thread_finish();
 242                 qman_thread_finish();
 243                 return -ENOMEM;
 244         }
 245
 246         dpaa_io_portal->qman_idx = qman_get_portal_index();
 247         dpaa_io_portal->bman_idx = bman_get_portal_index();
 248         dpaa_io_portal->tid = syscall(SYS_gettid);
 249
 250         ret = pthread_setspecific(dpaa_portal_key, (void *)dpaa_io_portal);
 251         if (ret) {
 252                 DPAA_BUS_LOG(ERR, "pthread_setspecific failed on "
 253                             "core %d with ret: %d", cpu, ret);
 254                 dpaa_portal_finish(NULL);
 255
 256                 return ret;
 257         }
 258
 259         RTE_PER_LCORE(_dpaa_io) = true;
 260
 261         DPAA_BUS_LOG(DEBUG, "QMAN thread initialized");
 262
 263         return 0;
 264 }
 265
 266 /*
 267  * rte_dpaa_portal_init - Wrapper over _dpaa_portal_init with thread level check
 268  * XXX Complete this
 269  */
 270 int rte_dpaa_portal_init(void *arg)
 271 {
 272         if (unlikely(!RTE_PER_LCORE(_dpaa_io)))
 273                 return _dpaa_portal_init(arg);
 274
 275         return 0;
 276 }
 277
 278 int
 279 rte_dpaa_portal_fq_init(void *arg, struct qman_fq *fq)
 280 {
 281         /* Affine above created portal with channel*/
 282         u32 sdqcr;
 283         struct qman_portal *qp;
 284
 285         if (unlikely(!RTE_PER_LCORE(_dpaa_io)))
 286                 _dpaa_portal_init(arg);
 287
 288         /* Initialise qman specific portals */
 289         qp = fsl_qman_portal_create();
 290         if (!qp) {
 291                 DPAA_BUS_LOG(ERR, "Unable to alloc fq portal");
 292                 return -1;
 293         }
 294         fq->qp = qp;
 295         sdqcr = QM_SDQCR_CHANNELS_POOL_CONV(fq->ch_id);
 296         qman_static_dequeue_add(sdqcr, qp);
 297
 298         return 0;
 299 }
 300
 301 int rte_dpaa_portal_fq_close(struct qman_fq *fq)
 302 {
 303         return fsl_qman_portal_destroy(fq->qp);
 304 }
 305
 306 void
 307 dpaa_portal_finish(void *arg)
 308 {
 309         struct dpaa_portal *dpaa_io_portal = (struct dpaa_portal *)arg;
 310
 311         if (!dpaa_io_portal) {
 312                 DPAA_BUS_LOG(DEBUG, "Portal already cleaned");
 313                 return;
 314         }
 315
 316         bman_thread_finish();
 317         qman_thread_finish();
 318
 319         pthread_setspecific(dpaa_portal_key, NULL);
 320
 321         rte_free(dpaa_io_portal);
 322         dpaa_io_portal = NULL;
 323
 324         RTE_PER_LCORE(_dpaa_io) = false;
 325 }
 326
 327 #define DPAA_DEV_PATH1 "/sys/devices/platform/soc/soc:fsl,dpaa"
 328 #define DPAA_DEV_PATH2 "/sys/devices/platform/fsl,dpaa"
 329
 330 static int
 331 rte_dpaa_bus_scan(void)
 332 {
 333         int ret;
 334
 335         BUS_INIT_FUNC_TRACE();
 336
 337         if ((access(DPAA_DEV_PATH1, F_OK) != 0) &&
 338             (access(DPAA_DEV_PATH2, F_OK) != 0)) {
 339                 RTE_LOG(DEBUG, EAL, "DPAA Bus not present. Skipping.\n");
 340                 return 0;
 341         }
 342
 343         /* Load the device-tree driver */
 344         ret = of_init();
 345         if (ret) {
 346                 DPAA_BUS_LOG(ERR, "of_init failed with ret: %d", ret);
 347                 return -1;
 348         }
 349
 350         /* Get the interface configurations from device-tree */
 351         dpaa_netcfg = netcfg_acquire();
 352         if (!dpaa_netcfg) {
 353                 DPAA_BUS_LOG(ERR, "netcfg_acquire failed");
 354                 return -EINVAL;
 355         }
 356
 357         RTE_LOG(NOTICE, EAL, "DPAA Bus Detected\n");
 358
 359         if (!dpaa_netcfg->num_ethports) {
 360                 DPAA_BUS_LOG(INFO, "no network interfaces available");
 361                 /* This is not an error */
 362                 return 0;
 363         }
 364
 365         DPAA_BUS_LOG(DEBUG, "Bus: Address of netcfg=%p, Ethports=%d",
 366                      dpaa_netcfg, dpaa_netcfg->num_ethports);
 367
 368 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
 369         dump_netcfg(dpaa_netcfg);
 370 #endif
 371
 372         DPAA_BUS_LOG(DEBUG, "Number of devices = %d\n",
 373                      dpaa_netcfg->num_ethports);
 374         ret = dpaa_create_device_list();
 375         if (ret) {
 376                 DPAA_BUS_LOG(ERR, "Unable to create device list. (%d)", ret);
 377                 return ret;
 378         }
 379
 380         /* create the key, supplying a function that'll be invoked
 381          * when a portal affined thread will be deleted.
 382          */
 383         ret = pthread_key_create(&dpaa_portal_key, dpaa_portal_finish);
 384         if (ret) {
 385                 DPAA_BUS_LOG(DEBUG, "Unable to create pthread key. (%d)", ret);
 386                 dpaa_clean_device_list();
 387                 return ret;
 388         }
 389
 390         DPAA_BUS_LOG(DEBUG, "dpaa_portal_key=%u, ret=%d\n",
 391                     (unsigned int)dpaa_portal_key, ret);
 392
 393         return 0;
 394 }
 395
 396 /* register a dpaa bus based dpaa driver */
 397 void
 398 rte_dpaa_driver_register(struct rte_dpaa_driver *driver)
 399 {
 400         RTE_VERIFY(driver);
 401
 402         BUS_INIT_FUNC_TRACE();
 403
 404         TAILQ_INSERT_TAIL(&rte_dpaa_bus.driver_list, driver, next);
 405         /* Update Bus references */
 406         driver->dpaa_bus = &rte_dpaa_bus;
 407 }
 408
 409 /* un-register a dpaa bus based dpaa driver */
 410 void
 411 rte_dpaa_driver_unregister(struct rte_dpaa_driver *driver)
 412 {
 413         struct rte_dpaa_bus *dpaa_bus;
 414
 415         BUS_INIT_FUNC_TRACE();
 416
 417         dpaa_bus = driver->dpaa_bus;
 418
 419         TAILQ_REMOVE(&dpaa_bus->driver_list, driver, next);
 420         /* Update Bus references */
 421         driver->dpaa_bus = NULL;
 422 }
 423
 424 static int
 425 rte_dpaa_device_match(struct rte_dpaa_driver *drv,
 426                       struct rte_dpaa_device *dev)
 427 {
 428         int ret = -1;
 429
 430         BUS_INIT_FUNC_TRACE();
 431
 432         if (!drv || !dev) {
 433                 DPAA_BUS_DEBUG("Invalid drv or dev received.");
 434                 return ret;
 435         }
 436
 437         if (drv->drv_type == dev->device_type) {
 438                 DPAA_BUS_INFO("Device: %s matches for driver: %s",
 439                               dev->name, drv->driver.name);
 440                 ret = 0; /* Found a match */
 441         }
 442
 443         return ret;
 444 }
 445
 446 static int
 447 rte_dpaa_bus_probe(void)
 448 {
 449         int ret = -1;
 450         struct rte_dpaa_device *dev;
 451         struct rte_dpaa_driver *drv;
 452         FILE *svr_file = NULL;
 453         unsigned int svr_ver;
 454
 455         BUS_INIT_FUNC_TRACE();
 456
 457         /* For each registered driver, and device, call the driver->probe */
 458         TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) {
 459                 TAILQ_FOREACH(drv, &rte_dpaa_bus.driver_list, next) {
 460                         ret = rte_dpaa_device_match(drv, dev);
 461                         if (ret)
 462                                 continue;
 463
 464                         if (!drv->probe)
 465                                 continue;
 466
 467                         ret = drv->probe(drv, dev);
 468                         if (ret)
 469                                 DPAA_BUS_ERR("Unable to probe.\n");
 470                         break;
 471                 }
 472         }
 473         rte_mbuf_set_platform_mempool_ops(DPAA_MEMPOOL_OPS_NAME);
 474
 475         svr_file = fopen(DPAA_SOC_ID_FILE, "r");
 476         if (svr_file) {
 477                 if (fscanf(svr_file, "svr:%x", &svr_ver) > 0)
 478                         dpaa_svr_family = svr_ver & SVR_MASK;
 479                 fclose(svr_file);
 480         }
 481
 482         return 0;
 483 }
 484
 485 static struct rte_device *
 486 rte_dpaa_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 487                      const void *data)
 488 {
 489         struct rte_dpaa_device *dev;
 490
 491         TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) {
 492                 if (start && &dev->device == start) {
 493                         start = NULL;  /* starting point found */
 494                         continue;
 495                 }
 496
 497                 if (cmp(&dev->device, data) == 0)
 498                         return &dev->device;
 499         }
 500
 501         return NULL;
 502 }
 503
 504 /*
 505  * Get iommu class of DPAA2 devices on the bus.
 506  */
 507 static enum rte_iova_mode
 508 rte_dpaa_get_iommu_class(void)
 509 {
 510         return RTE_IOVA_PA;
 511 }
 512
 513 struct rte_dpaa_bus rte_dpaa_bus = {
 514         .bus = {
 515                 .scan = rte_dpaa_bus_scan,
 516                 .probe = rte_dpaa_bus_probe,
 517                 .find_device = rte_dpaa_find_device,
 518                 .get_iommu_class = rte_dpaa_get_iommu_class,
 519         },
 520         .device_list = TAILQ_HEAD_INITIALIZER(rte_dpaa_bus.device_list),
 521         .driver_list = TAILQ_HEAD_INITIALIZER(rte_dpaa_bus.driver_list),
 522         .device_count = 0,
 523 };
 524
 525 RTE_REGISTER_BUS(FSL_DPAA_BUS_NAME, rte_dpaa_bus.bus);
 526
 527 RTE_INIT(dpaa_init_log);
 528 static void
 529 dpaa_init_log(void)
 530 {
 531         dpaa_logtype_bus = rte_log_register("bus.dpaa");
 532         if (dpaa_logtype_bus >= 0)
 533                 rte_log_set_level(dpaa_logtype_bus, RTE_LOG_NOTICE);
 534
 535         dpaa_logtype_mempool = rte_log_register("mempool.dpaa");
 536         if (dpaa_logtype_mempool >= 0)
 537                 rte_log_set_level(dpaa_logtype_mempool, RTE_LOG_NOTICE);
 538
 539         dpaa_logtype_pmd = rte_log_register("pmd.dpaa");
 540         if (dpaa_logtype_pmd >= 0)
 541                 rte_log_set_level(dpaa_logtype_pmd, RTE_LOG_NOTICE);
 542
 543         dpaa_logtype_eventdev = rte_log_register("eventdev.dpaa");
 544         if (dpaa_logtype_eventdev >= 0)
 545                 rte_log_set_level(dpaa_logtype_eventdev, RTE_LOG_NOTICE);
 546 }