drivers/event/dlb/dlb.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2020 Intel Corporation
   3  */
   4
   5 #include <assert.h>
   6 #include <errno.h>
   7 #include <nmmintrin.h>
   8 #include <pthread.h>
   9 #include <stdbool.h>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <sys/fcntl.h>
  14 #include <sys/mman.h>
  15 #include <unistd.h>
  16
  17 #include <rte_common.h>
  18 #include <rte_config.h>
  19 #include <rte_cycles.h>
  20 #include <rte_debug.h>
  21 #include <rte_dev.h>
  22 #include <rte_errno.h>
  23 #include <rte_io.h>
  24 #include <rte_kvargs.h>
  25 #include <rte_log.h>
  26 #include <rte_malloc.h>
  27 #include <rte_mbuf.h>
  28 #include <rte_prefetch.h>
  29 #include <rte_ring.h>
  30 #include <rte_string_fns.h>
  31
  32 #include <rte_eventdev.h>
  33 #include <rte_eventdev_pmd.h>
  34
  35 #include "dlb_priv.h"
  36 #include "dlb_iface.h"
  37 #include "dlb_inline_fns.h"
  38
  39 /*
  40  * Resources exposed to eventdev.
  41  */
  42 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
  43 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
  44 #endif
  45 static struct rte_event_dev_info evdev_dlb_default_info = {
  46         .driver_name = "", /* probe will set */
  47         .min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
  48         .max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
  49 #if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
  50         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
  51 #else
  52         .max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
  53 #endif
  54         .max_event_queue_flows = DLB_MAX_NUM_FLOWS,
  55         .max_event_queue_priority_levels = DLB_QID_PRIORITIES,
  56         .max_event_priority_levels = DLB_QID_PRIORITIES,
  57         .max_event_ports = DLB_MAX_NUM_LDB_PORTS,
  58         .max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
  59         .max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
  60         .max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
  61         .max_num_events = DLB_MAX_NUM_LDB_CREDITS,
  62         .max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
  63         .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
  64                           RTE_EVENT_DEV_CAP_EVENT_QOS |
  65                           RTE_EVENT_DEV_CAP_BURST_MODE |
  66                           RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
  67                           RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
  68                           RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
  69 };
  70
  71 struct process_local_port_data
  72 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
  73
  74 uint32_t
  75 dlb_get_queue_depth(struct dlb_eventdev *dlb,
  76                     struct dlb_eventdev_queue *queue)
  77 {
  78         /* DUMMY FOR NOW So "xstats" patch compiles */
  79         RTE_SET_USED(dlb);
  80         RTE_SET_USED(queue);
  81
  82         return 0;
  83 }
  84
  85 static int
  86 dlb_hw_query_resources(struct dlb_eventdev *dlb)
  87 {
  88         struct dlb_hw_dev *handle = &dlb->qm_instance;
  89         struct dlb_hw_resource_info *dlb_info = &handle->info;
  90         int ret;
  91
  92         ret = dlb_iface_get_num_resources(handle,
  93                                           &dlb->hw_rsrc_query_results);
  94         if (ret) {
  95                 DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
  96                 return ret;
  97         }
  98
  99         /* Complete filling in device resource info returned to evdev app,
 100          * overriding any default values.
 101          * The capabilities (CAPs) were set at compile time.
 102          */
 103
 104         evdev_dlb_default_info.max_event_queues =
 105                 dlb->hw_rsrc_query_results.num_ldb_queues;
 106
 107         evdev_dlb_default_info.max_event_ports =
 108                 dlb->hw_rsrc_query_results.num_ldb_ports;
 109
 110         evdev_dlb_default_info.max_num_events =
 111                 dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
 112
 113         /* Save off values used when creating the scheduling domain. */
 114
 115         handle->info.num_sched_domains =
 116                 dlb->hw_rsrc_query_results.num_sched_domains;
 117
 118         handle->info.hw_rsrc_max.nb_events_limit =
 119                 dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
 120
 121         handle->info.hw_rsrc_max.num_queues =
 122                 dlb->hw_rsrc_query_results.num_ldb_queues +
 123                 dlb->hw_rsrc_query_results.num_dir_ports;
 124
 125         handle->info.hw_rsrc_max.num_ldb_queues =
 126                 dlb->hw_rsrc_query_results.num_ldb_queues;
 127
 128         handle->info.hw_rsrc_max.num_ldb_ports =
 129                 dlb->hw_rsrc_query_results.num_ldb_ports;
 130
 131         handle->info.hw_rsrc_max.num_dir_ports =
 132                 dlb->hw_rsrc_query_results.num_dir_ports;
 133
 134         handle->info.hw_rsrc_max.reorder_window_size =
 135                 dlb->hw_rsrc_query_results.num_hist_list_entries;
 136
 137         rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
 138
 139         return 0;
 140 }
 141
 142 static void
 143 dlb_free_qe_mem(struct dlb_port *qm_port)
 144 {
 145         if (qm_port == NULL)
 146                 return;
 147
 148         rte_free(qm_port->qe4);
 149         qm_port->qe4 = NULL;
 150
 151         rte_free(qm_port->consume_qe);
 152         qm_port->consume_qe = NULL;
 153 }
 154
 155 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
 156  * unsafe.
 157  */
 158 #define DLB_BASE_10 10
 159
 160 static int
 161 dlb_string_to_int(int *result, const char *str)
 162 {
 163         long ret;
 164         char *endstr;
 165
 166         if (str == NULL || result == NULL)
 167                 return -EINVAL;
 168
 169         errno = 0;
 170         ret = strtol(str, &endstr, DLB_BASE_10);
 171         if (errno)
 172                 return -errno;
 173
 174         /* long int and int may be different width for some architectures */
 175         if (ret < INT_MIN || ret > INT_MAX || endstr == str)
 176                 return -EINVAL;
 177
 178         *result = ret;
 179         return 0;
 180 }
 181
 182 static int
 183 set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 184 {
 185         int *socket_id = opaque;
 186         int ret;
 187
 188         ret = dlb_string_to_int(socket_id, value);
 189         if (ret < 0)
 190                 return ret;
 191
 192         if (*socket_id > RTE_MAX_NUMA_NODES)
 193                 return -EINVAL;
 194
 195         return 0;
 196 }
 197
 198 static int
 199 set_max_num_events(const char *key __rte_unused,
 200                    const char *value,
 201                    void *opaque)
 202 {
 203         int *max_num_events = opaque;
 204         int ret;
 205
 206         if (value == NULL || opaque == NULL) {
 207                 DLB_LOG_ERR("NULL pointer\n");
 208                 return -EINVAL;
 209         }
 210
 211         ret = dlb_string_to_int(max_num_events, value);
 212         if (ret < 0)
 213                 return ret;
 214
 215         if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
 216                 DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
 217                             DLB_MAX_NUM_LDB_CREDITS);
 218                 return -EINVAL;
 219         }
 220
 221         return 0;
 222 }
 223
 224 static int
 225 set_num_dir_credits(const char *key __rte_unused,
 226                     const char *value,
 227                     void *opaque)
 228 {
 229         int *num_dir_credits = opaque;
 230         int ret;
 231
 232         if (value == NULL || opaque == NULL) {
 233                 DLB_LOG_ERR("NULL pointer\n");
 234                 return -EINVAL;
 235         }
 236
 237         ret = dlb_string_to_int(num_dir_credits, value);
 238         if (ret < 0)
 239                 return ret;
 240
 241         if (*num_dir_credits < 0 ||
 242             *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
 243                 DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
 244                             DLB_MAX_NUM_DIR_CREDITS);
 245                 return -EINVAL;
 246         }
 247         return 0;
 248 }
 249
 250 /* VDEV-only notes:
 251  * This function first unmaps all memory mappings and closes the
 252  * domain's file descriptor, which causes the driver to reset the
 253  * scheduling domain. Once that completes (when close() returns), we
 254  * can safely free the dynamically allocated memory used by the
 255  * scheduling domain.
 256  *
 257  * PF-only notes:
 258  * We will maintain a use count and use that to determine when
 259  * a reset is required.  In PF mode, we never mmap, or munmap
 260  * device memory,  and we own the entire physical PCI device.
 261  */
 262
 263 static void
 264 dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
 265 {
 266         struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
 267         enum dlb_configuration_state config_state;
 268         int i, j;
 269
 270         /* Close and reset the domain */
 271         dlb_iface_domain_close(dlb);
 272
 273         /* Free all dynamically allocated port memory */
 274         for (i = 0; i < dlb->num_ports; i++)
 275                 dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
 276
 277         /* If reconfiguring, mark the device's queues and ports as "previously
 278          * configured." If the user does not reconfigure them, the PMD will
 279          * reapply their previous configuration when the device is started.
 280          */
 281         config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
 282
 283         for (i = 0; i < dlb->num_ports; i++) {
 284                 dlb->ev_ports[i].qm_port.config_state = config_state;
 285                 /* Reset setup_done so ports can be reconfigured */
 286                 dlb->ev_ports[i].setup_done = false;
 287                 for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
 288                         dlb->ev_ports[i].link[j].mapped = false;
 289         }
 290
 291         for (i = 0; i < dlb->num_queues; i++)
 292                 dlb->ev_queues[i].qm_queue.config_state = config_state;
 293
 294         for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
 295                 dlb->ev_queues[i].setup_done = false;
 296
 297         dlb->num_ports = 0;
 298         dlb->num_ldb_ports = 0;
 299         dlb->num_dir_ports = 0;
 300         dlb->num_queues = 0;
 301         dlb->num_ldb_queues = 0;
 302         dlb->num_dir_queues = 0;
 303         dlb->configured = false;
 304 }
 305
 306 static int
 307 dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
 308 {
 309         struct dlb_create_ldb_pool_args cfg;
 310         struct dlb_cmd_response response;
 311         int ret;
 312
 313         if (handle == NULL)
 314                 return -EINVAL;
 315
 316         if (!handle->cfg.resources.num_ldb_credits) {
 317                 handle->cfg.ldb_credit_pool_id = 0;
 318                 handle->cfg.num_ldb_credits = 0;
 319                 return 0;
 320         }
 321
 322         cfg.response = (uintptr_t)&response;
 323         cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
 324
 325         ret = dlb_iface_ldb_credit_pool_create(handle,
 326                                                &cfg);
 327         if (ret < 0) {
 328                 DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
 329                             ret, dlb_error_strings[response.status]);
 330         }
 331
 332         handle->cfg.ldb_credit_pool_id = response.id;
 333         handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
 334
 335         return ret;
 336 }
 337
 338 static int
 339 dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
 340 {
 341         struct dlb_create_dir_pool_args cfg;
 342         struct dlb_cmd_response response;
 343         int ret;
 344
 345         if (handle == NULL)
 346                 return -EINVAL;
 347
 348         if (!handle->cfg.resources.num_dir_credits) {
 349                 handle->cfg.dir_credit_pool_id = 0;
 350                 handle->cfg.num_dir_credits = 0;
 351                 return 0;
 352         }
 353
 354         cfg.response = (uintptr_t)&response;
 355         cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
 356
 357         ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
 358         if (ret < 0)
 359                 DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
 360                             ret, dlb_error_strings[response.status]);
 361
 362         handle->cfg.dir_credit_pool_id = response.id;
 363         handle->cfg.num_dir_credits = cfg.num_dir_credits;
 364
 365         return ret;
 366 }
 367
 368 static int
 369 dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
 370                            struct dlb_eventdev *dlb,
 371                            const struct dlb_hw_rsrcs *resources_asked)
 372 {
 373         int ret = 0;
 374         struct dlb_create_sched_domain_args *config_params;
 375         struct dlb_cmd_response response;
 376
 377         if (resources_asked == NULL) {
 378                 DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
 379                 ret = EINVAL;
 380                 goto error_exit;
 381         }
 382
 383         /* Map generic qm resources to dlb resources */
 384         config_params = &handle->cfg.resources;
 385
 386         config_params->response = (uintptr_t)&response;
 387
 388         /* DIR ports and queues */
 389
 390         config_params->num_dir_ports =
 391                 resources_asked->num_dir_ports;
 392
 393         config_params->num_dir_credits =
 394                 resources_asked->num_dir_credits;
 395
 396         /* LDB ports and queues */
 397
 398         config_params->num_ldb_queues =
 399                 resources_asked->num_ldb_queues;
 400
 401         config_params->num_ldb_ports =
 402                 resources_asked->num_ldb_ports;
 403
 404         config_params->num_ldb_credits =
 405                 resources_asked->num_ldb_credits;
 406
 407         config_params->num_atomic_inflights =
 408                 dlb->num_atm_inflights_per_queue *
 409                 config_params->num_ldb_queues;
 410
 411         config_params->num_hist_list_entries = config_params->num_ldb_ports *
 412                 DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
 413
 414         /* dlb limited to 1 credit pool per queue type */
 415         config_params->num_ldb_credit_pools = 1;
 416         config_params->num_dir_credit_pools = 1;
 417
 418         DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
 419                     config_params->num_ldb_queues,
 420                     config_params->num_ldb_ports,
 421                     config_params->num_dir_ports,
 422                     config_params->num_atomic_inflights,
 423                     config_params->num_hist_list_entries,
 424                     config_params->num_ldb_credits,
 425                     config_params->num_dir_credits,
 426                     config_params->num_ldb_credit_pools,
 427                     config_params->num_dir_credit_pools);
 428
 429         /* Configure the QM */
 430
 431         ret = dlb_iface_sched_domain_create(handle, config_params);
 432         if (ret < 0) {
 433                 DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
 434                             handle->device_id,
 435                             ret,
 436                             dlb_error_strings[response.status]);
 437                 goto error_exit;
 438         }
 439
 440         handle->domain_id = response.id;
 441         handle->domain_id_valid = 1;
 442
 443         config_params->response = 0;
 444
 445         ret = dlb_ldb_credit_pool_create(handle);
 446         if (ret < 0) {
 447                 DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
 448                 goto error_exit2;
 449         }
 450
 451         ret = dlb_dir_credit_pool_create(handle);
 452         if (ret < 0) {
 453                 DLB_LOG_ERR("dlb: create dir credit pool failed\n");
 454                 goto error_exit2;
 455         }
 456
 457         handle->cfg.configured = true;
 458
 459         return 0;
 460
 461 error_exit2:
 462         dlb_iface_domain_close(dlb);
 463
 464 error_exit:
 465         return ret;
 466 }
 467
 468 /* End HW specific */
 469 static void
 470 dlb_eventdev_info_get(struct rte_eventdev *dev,
 471                       struct rte_event_dev_info *dev_info)
 472 {
 473         struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
 474         int ret;
 475
 476         ret = dlb_hw_query_resources(dlb);
 477         if (ret) {
 478                 const struct rte_eventdev_data *data = dev->data;
 479
 480                 DLB_LOG_ERR("get resources err=%d, devid=%d\n",
 481                             ret, data->dev_id);
 482                 /* fn is void, so fall through and return values set up in
 483                  * probe
 484                  */
 485         }
 486
 487         /* Add num resources currently owned by this domain.
 488          * These would become available if the scheduling domain were reset due
 489          * to the application recalling eventdev_configure to *reconfigure* the
 490          * domain.
 491          */
 492         evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
 493         evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
 494         evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
 495
 496         /* In DLB A-stepping hardware, applications are limited to 128
 497          * configured ports (load-balanced or directed). The reported number of
 498          * available ports must reflect this.
 499          */
 500         if (dlb->revision < DLB_REV_B0) {
 501                 int used_ports;
 502
 503                 used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
 504                         dlb->hw_rsrc_query_results.num_ldb_ports -
 505                         dlb->hw_rsrc_query_results.num_dir_ports;
 506
 507                 evdev_dlb_default_info.max_event_ports =
 508                         RTE_MIN(evdev_dlb_default_info.max_event_ports,
 509                                 128 - used_ports);
 510         }
 511
 512         evdev_dlb_default_info.max_event_queues =
 513                 RTE_MIN(evdev_dlb_default_info.max_event_queues,
 514                         RTE_EVENT_MAX_QUEUES_PER_DEV);
 515
 516         evdev_dlb_default_info.max_num_events =
 517                 RTE_MIN(evdev_dlb_default_info.max_num_events,
 518                         dlb->max_num_events_override);
 519
 520         *dev_info = evdev_dlb_default_info;
 521 }
 522
 523 /* Note: 1 QM instance per QM device, QM instance/device == event device */
 524 static int
 525 dlb_eventdev_configure(const struct rte_eventdev *dev)
 526 {
 527         struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
 528         struct dlb_hw_dev *handle = &dlb->qm_instance;
 529         struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
 530         const struct rte_eventdev_data *data = dev->data;
 531         const struct rte_event_dev_config *config = &data->dev_conf;
 532         int ret;
 533
 534         /* If this eventdev is already configured, we must release the current
 535          * scheduling domain before attempting to configure a new one.
 536          */
 537         if (dlb->configured) {
 538                 dlb_hw_reset_sched_domain(dev, true);
 539
 540                 ret = dlb_hw_query_resources(dlb);
 541                 if (ret) {
 542                         DLB_LOG_ERR("get resources err=%d, devid=%d\n",
 543                                     ret, data->dev_id);
 544                         return ret;
 545                 }
 546         }
 547
 548         if (config->nb_event_queues > rsrcs->num_queues) {
 549                 DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
 550                             config->nb_event_queues,
 551                             rsrcs->num_queues);
 552                 return -EINVAL;
 553         }
 554         if (config->nb_event_ports > (rsrcs->num_ldb_ports
 555                         + rsrcs->num_dir_ports)) {
 556                 DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
 557                             config->nb_event_ports,
 558                             (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
 559                 return -EINVAL;
 560         }
 561         if (config->nb_events_limit > rsrcs->nb_events_limit) {
 562                 DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
 563                             config->nb_events_limit,
 564                             rsrcs->nb_events_limit);
 565                 return -EINVAL;
 566         }
 567
 568         if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
 569                 dlb->global_dequeue_wait = false;
 570         else {
 571                 uint32_t timeout32;
 572
 573                 dlb->global_dequeue_wait = true;
 574
 575                 timeout32 = config->dequeue_timeout_ns;
 576
 577                 dlb->global_dequeue_wait_ticks =
 578                         timeout32 * (rte_get_timer_hz() / 1E9);
 579         }
 580
 581         /* Does this platform support umonitor/umwait? */
 582         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
 583                 if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
 584                     RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
 585                         DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
 586                                     RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
 587                         return -EINVAL;
 588                 }
 589                 dlb->umwait_allowed = true;
 590         }
 591
 592         rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
 593         rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
 594         /* 1 dir queue per dir port */
 595         rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
 596
 597         /* Scale down nb_events_limit by 4 for directed credits, since there
 598          * are 4x as many load-balanced credits.
 599          */
 600         rsrcs->num_ldb_credits = 0;
 601         rsrcs->num_dir_credits = 0;
 602
 603         if (rsrcs->num_ldb_queues)
 604                 rsrcs->num_ldb_credits = config->nb_events_limit;
 605         if (rsrcs->num_dir_ports)
 606                 rsrcs->num_dir_credits = config->nb_events_limit / 4;
 607         if (dlb->num_dir_credits_override != -1)
 608                 rsrcs->num_dir_credits = dlb->num_dir_credits_override;
 609
 610         if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
 611                 DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
 612                 return -ENODEV;
 613         }
 614
 615         dlb->new_event_limit = config->nb_events_limit;
 616         __atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
 617
 618         /* Save number of ports/queues for this event dev */
 619         dlb->num_ports = config->nb_event_ports;
 620         dlb->num_queues = config->nb_event_queues;
 621         dlb->num_dir_ports = rsrcs->num_dir_ports;
 622         dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
 623         dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
 624         dlb->num_dir_queues = dlb->num_dir_ports;
 625         dlb->num_ldb_credits = rsrcs->num_ldb_credits;
 626         dlb->num_dir_credits = rsrcs->num_dir_credits;
 627
 628         dlb->configured = true;
 629
 630         return 0;
 631 }
 632
 633 static void
 634 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 635                                    uint8_t port_id,
 636                                    struct rte_event_port_conf *port_conf)
 637 {
 638         RTE_SET_USED(port_id);
 639         struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
 640
 641         port_conf->new_event_threshold = dlb->new_event_limit;
 642         port_conf->dequeue_depth = 32;
 643         port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
 644         port_conf->event_port_cfg = 0;
 645 }
 646
 647 static void
 648 dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 649                                     uint8_t queue_id,
 650                                     struct rte_event_queue_conf *queue_conf)
 651 {
 652         RTE_SET_USED(dev);
 653         RTE_SET_USED(queue_id);
 654         queue_conf->nb_atomic_flows = 1024;
 655         queue_conf->nb_atomic_order_sequences = 32;
 656         queue_conf->event_queue_cfg = 0;
 657         queue_conf->priority = 0;
 658 }
 659
 660 static int
 661 set_dev_id(const char *key __rte_unused,
 662            const char *value,
 663            void *opaque)
 664 {
 665         int *dev_id = opaque;
 666         int ret;
 667
 668         if (value == NULL || opaque == NULL) {
 669                 DLB_LOG_ERR("NULL pointer\n");
 670                 return -EINVAL;
 671         }
 672
 673         ret = dlb_string_to_int(dev_id, value);
 674         if (ret < 0)
 675                 return ret;
 676
 677         return 0;
 678 }
 679
 680 static int
 681 set_defer_sched(const char *key __rte_unused,
 682                 const char *value,
 683                 void *opaque)
 684 {
 685         int *defer_sched = opaque;
 686
 687         if (value == NULL || opaque == NULL) {
 688                 DLB_LOG_ERR("NULL pointer\n");
 689                 return -EINVAL;
 690         }
 691
 692         if (strncmp(value, "on", 2) != 0) {
 693                 DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
 694                             value);
 695                 return -EINVAL;
 696         }
 697
 698         *defer_sched = 1;
 699
 700         return 0;
 701 }
 702
 703 static int
 704 set_num_atm_inflights(const char *key __rte_unused,
 705                       const char *value,
 706                       void *opaque)
 707 {
 708         int *num_atm_inflights = opaque;
 709         int ret;
 710
 711         if (value == NULL || opaque == NULL) {
 712                 DLB_LOG_ERR("NULL pointer\n");
 713                 return -EINVAL;
 714         }
 715
 716         ret = dlb_string_to_int(num_atm_inflights, value);
 717         if (ret < 0)
 718                 return ret;
 719
 720         if (*num_atm_inflights < 0 ||
 721             *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
 722                 DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
 723                             DLB_MAX_NUM_ATM_INFLIGHTS);
 724                 return -EINVAL;
 725         }
 726
 727         return 0;
 728 }
 729
 730 void
 731 dlb_entry_points_init(struct rte_eventdev *dev)
 732 {
 733         static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 734                 .dev_infos_get    = dlb_eventdev_info_get,
 735                 .dev_configure    = dlb_eventdev_configure,
 736                 .queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 737                 .port_def_conf    = dlb_eventdev_port_default_conf_get,
 738                 .dump             = dlb_eventdev_dump,
 739                 .xstats_get       = dlb_eventdev_xstats_get,
 740                 .xstats_get_names = dlb_eventdev_xstats_get_names,
 741                 .xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 742                 .xstats_reset       = dlb_eventdev_xstats_reset,
 743         };
 744
 745         /* Expose PMD's eventdev interface */
 746         dev->dev_ops = &dlb_eventdev_entry_ops;
 747 }
 748
 749 int
 750 dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 751                            const char *name,
 752                            struct dlb_devargs *dlb_args)
 753 {
 754         struct dlb_eventdev *dlb;
 755         int err;
 756
 757         dlb = dev->data->dev_private;
 758
 759         dlb->event_dev = dev; /* backlink */
 760
 761         evdev_dlb_default_info.driver_name = name;
 762
 763         dlb->max_num_events_override = dlb_args->max_num_events;
 764         dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
 765         dlb->defer_sched = dlb_args->defer_sched;
 766         dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
 767
 768         /* Open the interface.
 769          * For vdev mode, this means open the dlb kernel module.
 770          */
 771         err = dlb_iface_open(&dlb->qm_instance, name);
 772         if (err < 0) {
 773                 DLB_LOG_ERR("could not open event hardware device, err=%d\n",
 774                             err);
 775                 return err;
 776         }
 777
 778         err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
 779         if (err < 0) {
 780                 DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
 781                             err);
 782                 return err;
 783         }
 784
 785         err = dlb_hw_query_resources(dlb);
 786         if (err) {
 787                 DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
 788                 return err;
 789         }
 790
 791         err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
 792         if (err < 0) {
 793                 DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
 794                 return err;
 795         }
 796
 797         /* Complete xtstats runtime initialization */
 798         err = dlb_xstats_init(dlb);
 799         if (err) {
 800                 DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
 801                 return err;
 802         }
 803
 804         rte_spinlock_init(&dlb->qm_instance.resource_lock);
 805
 806         dlb_iface_low_level_io_init(dlb);
 807
 808         dlb_entry_points_init(dev);
 809
 810         return 0;
 811 }
 812
 813 int
 814 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 815                              const char *name)
 816 {
 817         struct dlb_eventdev *dlb;
 818         int err;
 819
 820         dlb = dev->data->dev_private;
 821
 822         evdev_dlb_default_info.driver_name = name;
 823
 824         err = dlb_iface_open(&dlb->qm_instance, name);
 825         if (err < 0) {
 826                 DLB_LOG_ERR("could not open event hardware device, err=%d\n",
 827                             err);
 828                 return err;
 829         }
 830
 831         err = dlb_hw_query_resources(dlb);
 832         if (err) {
 833                 DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
 834                 return err;
 835         }
 836
 837         dlb_iface_low_level_io_init(dlb);
 838
 839         dlb_entry_points_init(dev);
 840
 841         return 0;
 842 }
 843
 844 int
 845 dlb_parse_params(const char *params,
 846                  const char *name,
 847                  struct dlb_devargs *dlb_args)
 848 {
 849         int ret = 0;
 850         static const char * const args[] = { NUMA_NODE_ARG,
 851                                              DLB_MAX_NUM_EVENTS,
 852                                              DLB_NUM_DIR_CREDITS,
 853                                              DEV_ID_ARG,
 854                                              DLB_DEFER_SCHED_ARG,
 855                                              DLB_NUM_ATM_INFLIGHTS_ARG,
 856                                              NULL };
 857
 858         if (params && params[0] != '\0') {
 859                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
 860
 861                 if (kvlist == NULL) {
 862                         DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
 863                                      name);
 864                 } else {
 865                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
 866                                                      set_numa_node,
 867                                                      &dlb_args->socket_id);
 868                         if (ret != 0) {
 869                                 DLB_LOG_ERR("%s: Error parsing numa node parameter",
 870                                             name);
 871                                 rte_kvargs_free(kvlist);
 872                                 return ret;
 873                         }
 874
 875                         ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
 876                                                  set_max_num_events,
 877                                                  &dlb_args->max_num_events);
 878                         if (ret != 0) {
 879                                 DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
 880                                             name);
 881                                 rte_kvargs_free(kvlist);
 882                                 return ret;
 883                         }
 884
 885                         ret = rte_kvargs_process(kvlist,
 886                                         DLB_NUM_DIR_CREDITS,
 887                                         set_num_dir_credits,
 888                                         &dlb_args->num_dir_credits_override);
 889                         if (ret != 0) {
 890                                 DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
 891                                             name);
 892                                 rte_kvargs_free(kvlist);
 893                                 return ret;
 894                         }
 895
 896                         ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
 897                                                  set_dev_id,
 898                                                  &dlb_args->dev_id);
 899                         if (ret != 0) {
 900                                 DLB_LOG_ERR("%s: Error parsing dev_id parameter",
 901                                             name);
 902                                 rte_kvargs_free(kvlist);
 903                                 return ret;
 904                         }
 905
 906                         ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
 907                                                  set_defer_sched,
 908                                                  &dlb_args->defer_sched);
 909                         if (ret != 0) {
 910                                 DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
 911                                             name);
 912                                 rte_kvargs_free(kvlist);
 913                                 return ret;
 914                         }
 915
 916                         ret = rte_kvargs_process(kvlist,
 917                                                  DLB_NUM_ATM_INFLIGHTS_ARG,
 918                                                  set_num_atm_inflights,
 919                                                  &dlb_args->num_atm_inflights);
 920                         if (ret != 0) {
 921                                 DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
 922                                             name);
 923                                 rte_kvargs_free(kvlist);
 924                                 return ret;
 925                         }
 926
 927                         rte_kvargs_free(kvlist);
 928                 }
 929         }
 930         return ret;
 931 }
 932 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);