drivers/event/dlb2/dlb2.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2020 Intel Corporation
   3  */
   4
   5 #include <assert.h>
   6 #include <errno.h>
   7 #include <nmmintrin.h>
   8 #include <pthread.h>
   9 #include <stdint.h>
  10 #include <stdbool.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <sys/mman.h>
  14 #include <sys/fcntl.h>
  15
  16 #include <rte_common.h>
  17 #include <rte_config.h>
  18 #include <rte_cycles.h>
  19 #include <rte_debug.h>
  20 #include <rte_dev.h>
  21 #include <rte_errno.h>
  22 #include <rte_eventdev.h>
  23 #include <rte_eventdev_pmd.h>
  24 #include <rte_io.h>
  25 #include <rte_kvargs.h>
  26 #include <rte_log.h>
  27 #include <rte_malloc.h>
  28 #include <rte_mbuf.h>
  29 #include <rte_prefetch.h>
  30 #include <rte_ring.h>
  31 #include <rte_string_fns.h>
  32
  33 #include "dlb2_priv.h"
  34 #include "dlb2_iface.h"
  35 #include "dlb2_inline_fns.h"
  36
  37 /*
  38  * Resources exposed to eventdev. Some values overridden at runtime using
  39  * values returned by the DLB kernel driver.
  40  */
  41 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
  42 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
  43 #endif
  44 static struct rte_event_dev_info evdev_dlb2_default_info = {
  45         .driver_name = "", /* probe will set */
  46         .min_dequeue_timeout_ns = DLB2_MIN_DEQUEUE_TIMEOUT_NS,
  47         .max_dequeue_timeout_ns = DLB2_MAX_DEQUEUE_TIMEOUT_NS,
  48 #if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB2_MAX_NUM_LDB_QUEUES)
  49         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
  50 #else
  51         .max_event_queues = DLB2_MAX_NUM_LDB_QUEUES,
  52 #endif
  53         .max_event_queue_flows = DLB2_MAX_NUM_FLOWS,
  54         .max_event_queue_priority_levels = DLB2_QID_PRIORITIES,
  55         .max_event_priority_levels = DLB2_QID_PRIORITIES,
  56         .max_event_ports = DLB2_MAX_NUM_LDB_PORTS,
  57         .max_event_port_dequeue_depth = DLB2_MAX_CQ_DEPTH,
  58         .max_event_port_enqueue_depth = DLB2_MAX_ENQUEUE_DEPTH,
  59         .max_event_port_links = DLB2_MAX_NUM_QIDS_PER_LDB_CQ,
  60         .max_num_events = DLB2_MAX_NUM_LDB_CREDITS,
  61         .max_single_link_event_port_queue_pairs = DLB2_MAX_NUM_DIR_PORTS,
  62         .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
  63                           RTE_EVENT_DEV_CAP_EVENT_QOS |
  64                           RTE_EVENT_DEV_CAP_BURST_MODE |
  65                           RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
  66                           RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
  67                           RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
  68 };
  69
  70 struct process_local_port_data
  71 dlb2_port[DLB2_MAX_NUM_PORTS][DLB2_NUM_PORT_TYPES];
  72
  73 /*
  74  * DUMMY - added so that xstats path will compile/link.
  75  * Will be replaced by real version in a subsequent
  76  * patch.
  77  */
  78 uint32_t
  79 dlb2_get_queue_depth(struct dlb2_eventdev *dlb2,
  80                      struct dlb2_eventdev_queue *queue)
  81 {
  82         RTE_SET_USED(dlb2);
  83         RTE_SET_USED(queue);
  84
  85         return 0;
  86 }
  87
  88 /* override defaults with value(s) provided on command line */
  89 static void
  90 dlb2_init_queue_depth_thresholds(struct dlb2_eventdev *dlb2,
  91                                  int *qid_depth_thresholds)
  92 {
  93         int q;
  94
  95         for (q = 0; q < DLB2_MAX_NUM_QUEUES; q++) {
  96                 if (qid_depth_thresholds[q] != 0)
  97                         dlb2->ev_queues[q].depth_threshold =
  98                                 qid_depth_thresholds[q];
  99         }
 100 }
 101
 102 static int
 103 dlb2_hw_query_resources(struct dlb2_eventdev *dlb2)
 104 {
 105         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
 106         struct dlb2_hw_resource_info *dlb2_info = &handle->info;
 107         int ret;
 108
 109         /* Query driver resources provisioned for this device */
 110
 111         ret = dlb2_iface_get_num_resources(handle,
 112                                            &dlb2->hw_rsrc_query_results);
 113         if (ret) {
 114                 DLB2_LOG_ERR("ioctl get dlb2 num resources, err=%d\n", ret);
 115                 return ret;
 116         }
 117
 118         /* Complete filling in device resource info returned to evdev app,
 119          * overriding any default values.
 120          * The capabilities (CAPs) were set at compile time.
 121          */
 122
 123         evdev_dlb2_default_info.max_event_queues =
 124                 dlb2->hw_rsrc_query_results.num_ldb_queues;
 125
 126         evdev_dlb2_default_info.max_event_ports =
 127                 dlb2->hw_rsrc_query_results.num_ldb_ports;
 128
 129         evdev_dlb2_default_info.max_num_events =
 130                 dlb2->hw_rsrc_query_results.num_ldb_credits;
 131
 132         /* Save off values used when creating the scheduling domain. */
 133
 134         handle->info.num_sched_domains =
 135                 dlb2->hw_rsrc_query_results.num_sched_domains;
 136
 137         handle->info.hw_rsrc_max.nb_events_limit =
 138                 dlb2->hw_rsrc_query_results.num_ldb_credits;
 139
 140         handle->info.hw_rsrc_max.num_queues =
 141                 dlb2->hw_rsrc_query_results.num_ldb_queues +
 142                 dlb2->hw_rsrc_query_results.num_dir_ports;
 143
 144         handle->info.hw_rsrc_max.num_ldb_queues =
 145                 dlb2->hw_rsrc_query_results.num_ldb_queues;
 146
 147         handle->info.hw_rsrc_max.num_ldb_ports =
 148                 dlb2->hw_rsrc_query_results.num_ldb_ports;
 149
 150         handle->info.hw_rsrc_max.num_dir_ports =
 151                 dlb2->hw_rsrc_query_results.num_dir_ports;
 152
 153         handle->info.hw_rsrc_max.reorder_window_size =
 154                 dlb2->hw_rsrc_query_results.num_hist_list_entries;
 155
 156         rte_memcpy(dlb2_info, &handle->info.hw_rsrc_max, sizeof(*dlb2_info));
 157
 158         return 0;
 159 }
 160
 161 #define DLB2_BASE_10 10
 162
 163 static int
 164 dlb2_string_to_int(int *result, const char *str)
 165 {
 166         long ret;
 167         char *endptr;
 168
 169         if (str == NULL || result == NULL)
 170                 return -EINVAL;
 171
 172         errno = 0;
 173         ret = strtol(str, &endptr, DLB2_BASE_10);
 174         if (errno)
 175                 return -errno;
 176
 177         /* long int and int may be different width for some architectures */
 178         if (ret < INT_MIN || ret > INT_MAX || endptr == str)
 179                 return -EINVAL;
 180
 181         *result = ret;
 182         return 0;
 183 }
 184
 185 static int
 186 set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 187 {
 188         int *socket_id = opaque;
 189         int ret;
 190
 191         ret = dlb2_string_to_int(socket_id, value);
 192         if (ret < 0)
 193                 return ret;
 194
 195         if (*socket_id > RTE_MAX_NUMA_NODES)
 196                 return -EINVAL;
 197         return 0;
 198 }
 199
 200 static int
 201 set_max_num_events(const char *key __rte_unused,
 202                    const char *value,
 203                    void *opaque)
 204 {
 205         int *max_num_events = opaque;
 206         int ret;
 207
 208         if (value == NULL || opaque == NULL) {
 209                 DLB2_LOG_ERR("NULL pointer\n");
 210                 return -EINVAL;
 211         }
 212
 213         ret = dlb2_string_to_int(max_num_events, value);
 214         if (ret < 0)
 215                 return ret;
 216
 217         if (*max_num_events < 0 || *max_num_events >
 218                         DLB2_MAX_NUM_LDB_CREDITS) {
 219                 DLB2_LOG_ERR("dlb2: max_num_events must be between 0 and %d\n",
 220                              DLB2_MAX_NUM_LDB_CREDITS);
 221                 return -EINVAL;
 222         }
 223
 224         return 0;
 225 }
 226
 227 static int
 228 set_num_dir_credits(const char *key __rte_unused,
 229                     const char *value,
 230                     void *opaque)
 231 {
 232         int *num_dir_credits = opaque;
 233         int ret;
 234
 235         if (value == NULL || opaque == NULL) {
 236                 DLB2_LOG_ERR("NULL pointer\n");
 237                 return -EINVAL;
 238         }
 239
 240         ret = dlb2_string_to_int(num_dir_credits, value);
 241         if (ret < 0)
 242                 return ret;
 243
 244         if (*num_dir_credits < 0 ||
 245             *num_dir_credits > DLB2_MAX_NUM_DIR_CREDITS) {
 246                 DLB2_LOG_ERR("dlb2: num_dir_credits must be between 0 and %d\n",
 247                              DLB2_MAX_NUM_DIR_CREDITS);
 248                 return -EINVAL;
 249         }
 250
 251         return 0;
 252 }
 253
 254 static int
 255 set_dev_id(const char *key __rte_unused,
 256            const char *value,
 257            void *opaque)
 258 {
 259         int *dev_id = opaque;
 260         int ret;
 261
 262         if (value == NULL || opaque == NULL) {
 263                 DLB2_LOG_ERR("NULL pointer\n");
 264                 return -EINVAL;
 265         }
 266
 267         ret = dlb2_string_to_int(dev_id, value);
 268         if (ret < 0)
 269                 return ret;
 270
 271         return 0;
 272 }
 273
 274 static int
 275 set_cos(const char *key __rte_unused,
 276         const char *value,
 277         void *opaque)
 278 {
 279         enum dlb2_cos *cos_id = opaque;
 280         int x = 0;
 281         int ret;
 282
 283         if (value == NULL || opaque == NULL) {
 284                 DLB2_LOG_ERR("NULL pointer\n");
 285                 return -EINVAL;
 286         }
 287
 288         ret = dlb2_string_to_int(&x, value);
 289         if (ret < 0)
 290                 return ret;
 291
 292         if (x != DLB2_COS_DEFAULT && (x < DLB2_COS_0 || x > DLB2_COS_3)) {
 293                 DLB2_LOG_ERR(
 294                         "COS %d out of range, must be DLB2_COS_DEFAULT or 0-3\n",
 295                         x);
 296                 return -EINVAL;
 297         }
 298
 299         *cos_id = x;
 300
 301         return 0;
 302 }
 303
 304
 305 static int
 306 set_qid_depth_thresh(const char *key __rte_unused,
 307                      const char *value,
 308                      void *opaque)
 309 {
 310         struct dlb2_qid_depth_thresholds *qid_thresh = opaque;
 311         int first, last, thresh, i;
 312
 313         if (value == NULL || opaque == NULL) {
 314                 DLB2_LOG_ERR("NULL pointer\n");
 315                 return -EINVAL;
 316         }
 317
 318         /* command line override may take one of the following 3 forms:
 319          * qid_depth_thresh=all:<threshold_value> ... all queues
 320          * qid_depth_thresh=qidA-qidB:<threshold_value> ... a range of queues
 321          * qid_depth_thresh=qid:<threshold_value> ... just one queue
 322          */
 323         if (sscanf(value, "all:%d", &thresh) == 1) {
 324                 first = 0;
 325                 last = DLB2_MAX_NUM_QUEUES - 1;
 326         } else if (sscanf(value, "%d-%d:%d", &first, &last, &thresh) == 3) {
 327                 /* we have everything we need */
 328         } else if (sscanf(value, "%d:%d", &first, &thresh) == 2) {
 329                 last = first;
 330         } else {
 331                 DLB2_LOG_ERR("Error parsing qid depth devarg. Should be all:val, qid-qid:val, or qid:val\n");
 332                 return -EINVAL;
 333         }
 334
 335         if (first > last || first < 0 || last >= DLB2_MAX_NUM_QUEUES) {
 336                 DLB2_LOG_ERR("Error parsing qid depth devarg, invalid qid value\n");
 337                 return -EINVAL;
 338         }
 339
 340         if (thresh < 0 || thresh > DLB2_MAX_QUEUE_DEPTH_THRESHOLD) {
 341                 DLB2_LOG_ERR("Error parsing qid depth devarg, threshold > %d\n",
 342                              DLB2_MAX_QUEUE_DEPTH_THRESHOLD);
 343                 return -EINVAL;
 344         }
 345
 346         for (i = first; i <= last; i++)
 347                 qid_thresh->val[i] = thresh; /* indexed by qid */
 348
 349         return 0;
 350 }
 351
 352 static void
 353 dlb2_entry_points_init(struct rte_eventdev *dev)
 354 {
 355         /* Expose PMD's eventdev interface */
 356         static struct rte_eventdev_ops dlb2_eventdev_entry_ops = {
 357                 .dump             = dlb2_eventdev_dump,
 358                 .xstats_get       = dlb2_eventdev_xstats_get,
 359                 .xstats_get_names = dlb2_eventdev_xstats_get_names,
 360                 .xstats_get_by_name = dlb2_eventdev_xstats_get_by_name,
 361                 .xstats_reset       = dlb2_eventdev_xstats_reset,
 362         };
 363
 364         dev->dev_ops = &dlb2_eventdev_entry_ops;
 365 }
 366
 367 int
 368 dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
 369                             const char *name,
 370                             struct dlb2_devargs *dlb2_args)
 371 {
 372         struct dlb2_eventdev *dlb2;
 373         int err;
 374
 375         dlb2 = dev->data->dev_private;
 376
 377         dlb2->event_dev = dev; /* backlink */
 378
 379         evdev_dlb2_default_info.driver_name = name;
 380
 381         dlb2->max_num_events_override = dlb2_args->max_num_events;
 382         dlb2->num_dir_credits_override = dlb2_args->num_dir_credits_override;
 383         dlb2->qm_instance.cos_id = dlb2_args->cos_id;
 384
 385         err = dlb2_iface_open(&dlb2->qm_instance, name);
 386         if (err < 0) {
 387                 DLB2_LOG_ERR("could not open event hardware device, err=%d\n",
 388                              err);
 389                 return err;
 390         }
 391
 392         err = dlb2_iface_get_device_version(&dlb2->qm_instance,
 393                                             &dlb2->revision);
 394         if (err < 0) {
 395                 DLB2_LOG_ERR("dlb2: failed to get the device version, err=%d\n",
 396                              err);
 397                 return err;
 398         }
 399
 400         err = dlb2_hw_query_resources(dlb2);
 401         if (err) {
 402                 DLB2_LOG_ERR("get resources err=%d for %s\n",
 403                              err, name);
 404                 return err;
 405         }
 406
 407         dlb2_iface_hardware_init(&dlb2->qm_instance);
 408
 409         err = dlb2_iface_get_cq_poll_mode(&dlb2->qm_instance, &dlb2->poll_mode);
 410         if (err < 0) {
 411                 DLB2_LOG_ERR("dlb2: failed to get the poll mode, err=%d\n",
 412                              err);
 413                 return err;
 414         }
 415
 416         /* Complete xtstats runtime initialization */
 417         err = dlb2_xstats_init(dlb2);
 418         if (err) {
 419                 DLB2_LOG_ERR("dlb2: failed to init xstats, err=%d\n", err);
 420                 return err;
 421         }
 422
 423         rte_spinlock_init(&dlb2->qm_instance.resource_lock);
 424
 425         dlb2_iface_low_level_io_init();
 426
 427         dlb2_entry_points_init(dev);
 428
 429         dlb2_init_queue_depth_thresholds(dlb2,
 430                                          dlb2_args->qid_depth_thresholds.val);
 431
 432         return 0;
 433 }
 434
 435 int
 436 dlb2_secondary_eventdev_probe(struct rte_eventdev *dev,
 437                               const char *name)
 438 {
 439         struct dlb2_eventdev *dlb2;
 440         int err;
 441
 442         dlb2 = dev->data->dev_private;
 443
 444         evdev_dlb2_default_info.driver_name = name;
 445
 446         err = dlb2_iface_open(&dlb2->qm_instance, name);
 447         if (err < 0) {
 448                 DLB2_LOG_ERR("could not open event hardware device, err=%d\n",
 449                              err);
 450                 return err;
 451         }
 452
 453         err = dlb2_hw_query_resources(dlb2);
 454         if (err) {
 455                 DLB2_LOG_ERR("get resources err=%d for %s\n",
 456                              err, name);
 457                 return err;
 458         }
 459
 460         dlb2_iface_low_level_io_init();
 461
 462         dlb2_entry_points_init(dev);
 463
 464         return 0;
 465 }
 466
 467 int
 468 dlb2_parse_params(const char *params,
 469                   const char *name,
 470                   struct dlb2_devargs *dlb2_args)
 471 {
 472         int ret = 0;
 473         static const char * const args[] = { NUMA_NODE_ARG,
 474                                              DLB2_MAX_NUM_EVENTS,
 475                                              DLB2_NUM_DIR_CREDITS,
 476                                              DEV_ID_ARG,
 477                                              DLB2_QID_DEPTH_THRESH_ARG,
 478                                              DLB2_COS_ARG,
 479                                              NULL };
 480
 481         if (params != NULL && params[0] != '\0') {
 482                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
 483
 484                 if (kvlist == NULL) {
 485                         RTE_LOG(INFO, PMD,
 486                                 "Ignoring unsupported parameters when creating device '%s'\n",
 487                                 name);
 488                 } else {
 489                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
 490                                                      set_numa_node,
 491                                                      &dlb2_args->socket_id);
 492                         if (ret != 0) {
 493                                 DLB2_LOG_ERR("%s: Error parsing numa node parameter",
 494                                              name);
 495                                 rte_kvargs_free(kvlist);
 496                                 return ret;
 497                         }
 498
 499                         ret = rte_kvargs_process(kvlist, DLB2_MAX_NUM_EVENTS,
 500                                                  set_max_num_events,
 501                                                  &dlb2_args->max_num_events);
 502                         if (ret != 0) {
 503                                 DLB2_LOG_ERR("%s: Error parsing max_num_events parameter",
 504                                              name);
 505                                 rte_kvargs_free(kvlist);
 506                                 return ret;
 507                         }
 508
 509                         ret = rte_kvargs_process(kvlist,
 510                                         DLB2_NUM_DIR_CREDITS,
 511                                         set_num_dir_credits,
 512                                         &dlb2_args->num_dir_credits_override);
 513                         if (ret != 0) {
 514                                 DLB2_LOG_ERR("%s: Error parsing num_dir_credits parameter",
 515                                              name);
 516                                 rte_kvargs_free(kvlist);
 517                                 return ret;
 518                         }
 519
 520                         ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
 521                                                  set_dev_id,
 522                                                  &dlb2_args->dev_id);
 523                         if (ret != 0) {
 524                                 DLB2_LOG_ERR("%s: Error parsing dev_id parameter",
 525                                              name);
 526                                 rte_kvargs_free(kvlist);
 527                                 return ret;
 528                         }
 529
 530                         ret = rte_kvargs_process(
 531                                         kvlist,
 532                                         DLB2_QID_DEPTH_THRESH_ARG,
 533                                         set_qid_depth_thresh,
 534                                         &dlb2_args->qid_depth_thresholds);
 535                         if (ret != 0) {
 536                                 DLB2_LOG_ERR("%s: Error parsing qid_depth_thresh parameter",
 537                                              name);
 538                                 rte_kvargs_free(kvlist);
 539                                 return ret;
 540                         }
 541
 542                         ret = rte_kvargs_process(kvlist, DLB2_COS_ARG,
 543                                                  set_cos,
 544                                                  &dlb2_args->cos_id);
 545                         if (ret != 0) {
 546                                 DLB2_LOG_ERR("%s: Error parsing cos parameter",
 547                                              name);
 548                                 rte_kvargs_free(kvlist);
 549                                 return ret;
 550                         }
 551
 552                         rte_kvargs_free(kvlist);
 553                 }
 554         }
 555         return ret;
 556 }
 557 RTE_LOG_REGISTER(eventdev_dlb2_log_level, pmd.event.dlb2, NOTICE);