drivers/event/dlb2/dlb2.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2020 Intel Corporation
   3  */
   4
   5 #include <assert.h>
   6 #include <errno.h>
   7 #include <nmmintrin.h>
   8 #include <pthread.h>
   9 #include <stdint.h>
  10 #include <stdbool.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <sys/mman.h>
  14 #include <sys/fcntl.h>
  15
  16 #include <rte_common.h>
  17 #include <rte_config.h>
  18 #include <rte_cycles.h>
  19 #include <rte_debug.h>
  20 #include <rte_dev.h>
  21 #include <rte_errno.h>
  22 #include <rte_eventdev.h>
  23 #include <rte_eventdev_pmd.h>
  24 #include <rte_io.h>
  25 #include <rte_kvargs.h>
  26 #include <rte_log.h>
  27 #include <rte_malloc.h>
  28 #include <rte_mbuf.h>
  29 #include <rte_prefetch.h>
  30 #include <rte_ring.h>
  31 #include <rte_string_fns.h>
  32
  33 #include "dlb2_priv.h"
  34 #include "dlb2_iface.h"
  35 #include "dlb2_inline_fns.h"
  36
  37 /*
  38  * Resources exposed to eventdev. Some values overridden at runtime using
  39  * values returned by the DLB kernel driver.
  40  */
  41 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
  42 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
  43 #endif
  44 static struct rte_event_dev_info evdev_dlb2_default_info = {
  45         .driver_name = "", /* probe will set */
  46         .min_dequeue_timeout_ns = DLB2_MIN_DEQUEUE_TIMEOUT_NS,
  47         .max_dequeue_timeout_ns = DLB2_MAX_DEQUEUE_TIMEOUT_NS,
  48 #if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB2_MAX_NUM_LDB_QUEUES)
  49         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
  50 #else
  51         .max_event_queues = DLB2_MAX_NUM_LDB_QUEUES,
  52 #endif
  53         .max_event_queue_flows = DLB2_MAX_NUM_FLOWS,
  54         .max_event_queue_priority_levels = DLB2_QID_PRIORITIES,
  55         .max_event_priority_levels = DLB2_QID_PRIORITIES,
  56         .max_event_ports = DLB2_MAX_NUM_LDB_PORTS,
  57         .max_event_port_dequeue_depth = DLB2_MAX_CQ_DEPTH,
  58         .max_event_port_enqueue_depth = DLB2_MAX_ENQUEUE_DEPTH,
  59         .max_event_port_links = DLB2_MAX_NUM_QIDS_PER_LDB_CQ,
  60         .max_num_events = DLB2_MAX_NUM_LDB_CREDITS,
  61         .max_single_link_event_port_queue_pairs = DLB2_MAX_NUM_DIR_PORTS,
  62         .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
  63                           RTE_EVENT_DEV_CAP_EVENT_QOS |
  64                           RTE_EVENT_DEV_CAP_BURST_MODE |
  65                           RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
  66                           RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
  67                           RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
  68 };
  69
  70 struct process_local_port_data
  71 dlb2_port[DLB2_MAX_NUM_PORTS][DLB2_NUM_PORT_TYPES];
  72
  73 /* override defaults with value(s) provided on command line */
  74 static void
  75 dlb2_init_queue_depth_thresholds(struct dlb2_eventdev *dlb2,
  76                                  int *qid_depth_thresholds)
  77 {
  78         int q;
  79
  80         for (q = 0; q < DLB2_MAX_NUM_QUEUES; q++) {
  81                 if (qid_depth_thresholds[q] != 0)
  82                         dlb2->ev_queues[q].depth_threshold =
  83                                 qid_depth_thresholds[q];
  84         }
  85 }
  86
  87 static int
  88 dlb2_hw_query_resources(struct dlb2_eventdev *dlb2)
  89 {
  90         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
  91         struct dlb2_hw_resource_info *dlb2_info = &handle->info;
  92         int ret;
  93
  94         /* Query driver resources provisioned for this device */
  95
  96         ret = dlb2_iface_get_num_resources(handle,
  97                                            &dlb2->hw_rsrc_query_results);
  98         if (ret) {
  99                 DLB2_LOG_ERR("ioctl get dlb2 num resources, err=%d\n", ret);
 100                 return ret;
 101         }
 102
 103         /* Complete filling in device resource info returned to evdev app,
 104          * overriding any default values.
 105          * The capabilities (CAPs) were set at compile time.
 106          */
 107
 108         evdev_dlb2_default_info.max_event_queues =
 109                 dlb2->hw_rsrc_query_results.num_ldb_queues;
 110
 111         evdev_dlb2_default_info.max_event_ports =
 112                 dlb2->hw_rsrc_query_results.num_ldb_ports;
 113
 114         evdev_dlb2_default_info.max_num_events =
 115                 dlb2->hw_rsrc_query_results.num_ldb_credits;
 116
 117         /* Save off values used when creating the scheduling domain. */
 118
 119         handle->info.num_sched_domains =
 120                 dlb2->hw_rsrc_query_results.num_sched_domains;
 121
 122         handle->info.hw_rsrc_max.nb_events_limit =
 123                 dlb2->hw_rsrc_query_results.num_ldb_credits;
 124
 125         handle->info.hw_rsrc_max.num_queues =
 126                 dlb2->hw_rsrc_query_results.num_ldb_queues +
 127                 dlb2->hw_rsrc_query_results.num_dir_ports;
 128
 129         handle->info.hw_rsrc_max.num_ldb_queues =
 130                 dlb2->hw_rsrc_query_results.num_ldb_queues;
 131
 132         handle->info.hw_rsrc_max.num_ldb_ports =
 133                 dlb2->hw_rsrc_query_results.num_ldb_ports;
 134
 135         handle->info.hw_rsrc_max.num_dir_ports =
 136                 dlb2->hw_rsrc_query_results.num_dir_ports;
 137
 138         handle->info.hw_rsrc_max.reorder_window_size =
 139                 dlb2->hw_rsrc_query_results.num_hist_list_entries;
 140
 141         rte_memcpy(dlb2_info, &handle->info.hw_rsrc_max, sizeof(*dlb2_info));
 142
 143         return 0;
 144 }
 145
 146 #define DLB2_BASE_10 10
 147
 148 static int
 149 dlb2_string_to_int(int *result, const char *str)
 150 {
 151         long ret;
 152         char *endptr;
 153
 154         if (str == NULL || result == NULL)
 155                 return -EINVAL;
 156
 157         errno = 0;
 158         ret = strtol(str, &endptr, DLB2_BASE_10);
 159         if (errno)
 160                 return -errno;
 161
 162         /* long int and int may be different width for some architectures */
 163         if (ret < INT_MIN || ret > INT_MAX || endptr == str)
 164                 return -EINVAL;
 165
 166         *result = ret;
 167         return 0;
 168 }
 169
 170 static int
 171 set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 172 {
 173         int *socket_id = opaque;
 174         int ret;
 175
 176         ret = dlb2_string_to_int(socket_id, value);
 177         if (ret < 0)
 178                 return ret;
 179
 180         if (*socket_id > RTE_MAX_NUMA_NODES)
 181                 return -EINVAL;
 182         return 0;
 183 }
 184
 185 static int
 186 set_max_num_events(const char *key __rte_unused,
 187                    const char *value,
 188                    void *opaque)
 189 {
 190         int *max_num_events = opaque;
 191         int ret;
 192
 193         if (value == NULL || opaque == NULL) {
 194                 DLB2_LOG_ERR("NULL pointer\n");
 195                 return -EINVAL;
 196         }
 197
 198         ret = dlb2_string_to_int(max_num_events, value);
 199         if (ret < 0)
 200                 return ret;
 201
 202         if (*max_num_events < 0 || *max_num_events >
 203                         DLB2_MAX_NUM_LDB_CREDITS) {
 204                 DLB2_LOG_ERR("dlb2: max_num_events must be between 0 and %d\n",
 205                              DLB2_MAX_NUM_LDB_CREDITS);
 206                 return -EINVAL;
 207         }
 208
 209         return 0;
 210 }
 211
 212 static int
 213 set_num_dir_credits(const char *key __rte_unused,
 214                     const char *value,
 215                     void *opaque)
 216 {
 217         int *num_dir_credits = opaque;
 218         int ret;
 219
 220         if (value == NULL || opaque == NULL) {
 221                 DLB2_LOG_ERR("NULL pointer\n");
 222                 return -EINVAL;
 223         }
 224
 225         ret = dlb2_string_to_int(num_dir_credits, value);
 226         if (ret < 0)
 227                 return ret;
 228
 229         if (*num_dir_credits < 0 ||
 230             *num_dir_credits > DLB2_MAX_NUM_DIR_CREDITS) {
 231                 DLB2_LOG_ERR("dlb2: num_dir_credits must be between 0 and %d\n",
 232                              DLB2_MAX_NUM_DIR_CREDITS);
 233                 return -EINVAL;
 234         }
 235
 236         return 0;
 237 }
 238
 239 static int
 240 set_dev_id(const char *key __rte_unused,
 241            const char *value,
 242            void *opaque)
 243 {
 244         int *dev_id = opaque;
 245         int ret;
 246
 247         if (value == NULL || opaque == NULL) {
 248                 DLB2_LOG_ERR("NULL pointer\n");
 249                 return -EINVAL;
 250         }
 251
 252         ret = dlb2_string_to_int(dev_id, value);
 253         if (ret < 0)
 254                 return ret;
 255
 256         return 0;
 257 }
 258
 259 static int
 260 set_cos(const char *key __rte_unused,
 261         const char *value,
 262         void *opaque)
 263 {
 264         enum dlb2_cos *cos_id = opaque;
 265         int x = 0;
 266         int ret;
 267
 268         if (value == NULL || opaque == NULL) {
 269                 DLB2_LOG_ERR("NULL pointer\n");
 270                 return -EINVAL;
 271         }
 272
 273         ret = dlb2_string_to_int(&x, value);
 274         if (ret < 0)
 275                 return ret;
 276
 277         if (x != DLB2_COS_DEFAULT && (x < DLB2_COS_0 || x > DLB2_COS_3)) {
 278                 DLB2_LOG_ERR(
 279                         "COS %d out of range, must be DLB2_COS_DEFAULT or 0-3\n",
 280                         x);
 281                 return -EINVAL;
 282         }
 283
 284         *cos_id = x;
 285
 286         return 0;
 287 }
 288
 289
 290 static int
 291 set_qid_depth_thresh(const char *key __rte_unused,
 292                      const char *value,
 293                      void *opaque)
 294 {
 295         struct dlb2_qid_depth_thresholds *qid_thresh = opaque;
 296         int first, last, thresh, i;
 297
 298         if (value == NULL || opaque == NULL) {
 299                 DLB2_LOG_ERR("NULL pointer\n");
 300                 return -EINVAL;
 301         }
 302
 303         /* command line override may take one of the following 3 forms:
 304          * qid_depth_thresh=all:<threshold_value> ... all queues
 305          * qid_depth_thresh=qidA-qidB:<threshold_value> ... a range of queues
 306          * qid_depth_thresh=qid:<threshold_value> ... just one queue
 307          */
 308         if (sscanf(value, "all:%d", &thresh) == 1) {
 309                 first = 0;
 310                 last = DLB2_MAX_NUM_QUEUES - 1;
 311         } else if (sscanf(value, "%d-%d:%d", &first, &last, &thresh) == 3) {
 312                 /* we have everything we need */
 313         } else if (sscanf(value, "%d:%d", &first, &thresh) == 2) {
 314                 last = first;
 315         } else {
 316                 DLB2_LOG_ERR("Error parsing qid depth devarg. Should be all:val, qid-qid:val, or qid:val\n");
 317                 return -EINVAL;
 318         }
 319
 320         if (first > last || first < 0 || last >= DLB2_MAX_NUM_QUEUES) {
 321                 DLB2_LOG_ERR("Error parsing qid depth devarg, invalid qid value\n");
 322                 return -EINVAL;
 323         }
 324
 325         if (thresh < 0 || thresh > DLB2_MAX_QUEUE_DEPTH_THRESHOLD) {
 326                 DLB2_LOG_ERR("Error parsing qid depth devarg, threshold > %d\n",
 327                              DLB2_MAX_QUEUE_DEPTH_THRESHOLD);
 328                 return -EINVAL;
 329         }
 330
 331         for (i = first; i <= last; i++)
 332                 qid_thresh->val[i] = thresh; /* indexed by qid */
 333
 334         return 0;
 335 }
 336
 337 static void
 338 dlb2_entry_points_init(struct rte_eventdev *dev)
 339 {
 340         RTE_SET_USED(dev);
 341
 342         /* Eventdev PMD entry points */
 343 }
 344
 345 int
 346 dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
 347                             const char *name,
 348                             struct dlb2_devargs *dlb2_args)
 349 {
 350         struct dlb2_eventdev *dlb2;
 351         int err;
 352
 353         dlb2 = dev->data->dev_private;
 354
 355         dlb2->event_dev = dev; /* backlink */
 356
 357         evdev_dlb2_default_info.driver_name = name;
 358
 359         dlb2->max_num_events_override = dlb2_args->max_num_events;
 360         dlb2->num_dir_credits_override = dlb2_args->num_dir_credits_override;
 361         dlb2->qm_instance.cos_id = dlb2_args->cos_id;
 362
 363         err = dlb2_iface_open(&dlb2->qm_instance, name);
 364         if (err < 0) {
 365                 DLB2_LOG_ERR("could not open event hardware device, err=%d\n",
 366                              err);
 367                 return err;
 368         }
 369
 370         err = dlb2_iface_get_device_version(&dlb2->qm_instance,
 371                                             &dlb2->revision);
 372         if (err < 0) {
 373                 DLB2_LOG_ERR("dlb2: failed to get the device version, err=%d\n",
 374                              err);
 375                 return err;
 376         }
 377
 378         err = dlb2_hw_query_resources(dlb2);
 379         if (err) {
 380                 DLB2_LOG_ERR("get resources err=%d for %s\n",
 381                              err, name);
 382                 return err;
 383         }
 384
 385         dlb2_iface_hardware_init(&dlb2->qm_instance);
 386
 387         err = dlb2_iface_get_cq_poll_mode(&dlb2->qm_instance, &dlb2->poll_mode);
 388         if (err < 0) {
 389                 DLB2_LOG_ERR("dlb2: failed to get the poll mode, err=%d\n",
 390                              err);
 391                 return err;
 392         }
 393
 394         rte_spinlock_init(&dlb2->qm_instance.resource_lock);
 395
 396         dlb2_iface_low_level_io_init();
 397
 398         dlb2_entry_points_init(dev);
 399
 400         dlb2_init_queue_depth_thresholds(dlb2,
 401                                          dlb2_args->qid_depth_thresholds.val);
 402
 403         return 0;
 404 }
 405
 406 int
 407 dlb2_secondary_eventdev_probe(struct rte_eventdev *dev,
 408                               const char *name)
 409 {
 410         struct dlb2_eventdev *dlb2;
 411         int err;
 412
 413         dlb2 = dev->data->dev_private;
 414
 415         evdev_dlb2_default_info.driver_name = name;
 416
 417         err = dlb2_iface_open(&dlb2->qm_instance, name);
 418         if (err < 0) {
 419                 DLB2_LOG_ERR("could not open event hardware device, err=%d\n",
 420                              err);
 421                 return err;
 422         }
 423
 424         err = dlb2_hw_query_resources(dlb2);
 425         if (err) {
 426                 DLB2_LOG_ERR("get resources err=%d for %s\n",
 427                              err, name);
 428                 return err;
 429         }
 430
 431         dlb2_iface_low_level_io_init();
 432
 433         dlb2_entry_points_init(dev);
 434
 435         return 0;
 436 }
 437
 438 int
 439 dlb2_parse_params(const char *params,
 440                   const char *name,
 441                   struct dlb2_devargs *dlb2_args)
 442 {
 443         int ret = 0;
 444         static const char * const args[] = { NUMA_NODE_ARG,
 445                                              DLB2_MAX_NUM_EVENTS,
 446                                              DLB2_NUM_DIR_CREDITS,
 447                                              DEV_ID_ARG,
 448                                              DLB2_QID_DEPTH_THRESH_ARG,
 449                                              DLB2_COS_ARG,
 450                                              NULL };
 451
 452         if (params != NULL && params[0] != '\0') {
 453                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
 454
 455                 if (kvlist == NULL) {
 456                         RTE_LOG(INFO, PMD,
 457                                 "Ignoring unsupported parameters when creating device '%s'\n",
 458                                 name);
 459                 } else {
 460                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
 461                                                      set_numa_node,
 462                                                      &dlb2_args->socket_id);
 463                         if (ret != 0) {
 464                                 DLB2_LOG_ERR("%s: Error parsing numa node parameter",
 465                                              name);
 466                                 rte_kvargs_free(kvlist);
 467                                 return ret;
 468                         }
 469
 470                         ret = rte_kvargs_process(kvlist, DLB2_MAX_NUM_EVENTS,
 471                                                  set_max_num_events,
 472                                                  &dlb2_args->max_num_events);
 473                         if (ret != 0) {
 474                                 DLB2_LOG_ERR("%s: Error parsing max_num_events parameter",
 475                                              name);
 476                                 rte_kvargs_free(kvlist);
 477                                 return ret;
 478                         }
 479
 480                         ret = rte_kvargs_process(kvlist,
 481                                         DLB2_NUM_DIR_CREDITS,
 482                                         set_num_dir_credits,
 483                                         &dlb2_args->num_dir_credits_override);
 484                         if (ret != 0) {
 485                                 DLB2_LOG_ERR("%s: Error parsing num_dir_credits parameter",
 486                                              name);
 487                                 rte_kvargs_free(kvlist);
 488                                 return ret;
 489                         }
 490
 491                         ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
 492                                                  set_dev_id,
 493                                                  &dlb2_args->dev_id);
 494                         if (ret != 0) {
 495                                 DLB2_LOG_ERR("%s: Error parsing dev_id parameter",
 496                                              name);
 497                                 rte_kvargs_free(kvlist);
 498                                 return ret;
 499                         }
 500
 501                         ret = rte_kvargs_process(
 502                                         kvlist,
 503                                         DLB2_QID_DEPTH_THRESH_ARG,
 504                                         set_qid_depth_thresh,
 505                                         &dlb2_args->qid_depth_thresholds);
 506                         if (ret != 0) {
 507                                 DLB2_LOG_ERR("%s: Error parsing qid_depth_thresh parameter",
 508                                              name);
 509                                 rte_kvargs_free(kvlist);
 510                                 return ret;
 511                         }
 512
 513                         ret = rte_kvargs_process(kvlist, DLB2_COS_ARG,
 514                                                  set_cos,
 515                                                  &dlb2_args->cos_id);
 516                         if (ret != 0) {
 517                                 DLB2_LOG_ERR("%s: Error parsing cos parameter",
 518                                              name);
 519                                 rte_kvargs_free(kvlist);
 520                                 return ret;
 521                         }
 522
 523                         rte_kvargs_free(kvlist);
 524                 }
 525         }
 526         return ret;
 527 }
 528 RTE_LOG_REGISTER(eventdev_dlb2_log_level, pmd.event.dlb2, NOTICE);