drivers/event/dlb2/dlb2.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2020 Intel Corporation
   3  */
   4
   5 #include <assert.h>
   6 #include <errno.h>
   7 #include <nmmintrin.h>
   8 #include <pthread.h>
   9 #include <stdint.h>
  10 #include <stdbool.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <sys/mman.h>
  14 #include <fcntl.h>
  15
  16 #include <rte_common.h>
  17 #include <rte_config.h>
  18 #include <rte_cycles.h>
  19 #include <rte_debug.h>
  20 #include <rte_dev.h>
  21 #include <rte_errno.h>
  22 #include <rte_eventdev.h>
  23 #include <eventdev_pmd.h>
  24 #include <rte_io.h>
  25 #include <rte_kvargs.h>
  26 #include <rte_log.h>
  27 #include <rte_malloc.h>
  28 #include <rte_mbuf.h>
  29 #include <rte_power_intrinsics.h>
  30 #include <rte_prefetch.h>
  31 #include <rte_ring.h>
  32 #include <rte_string_fns.h>
  33
  34 #include "dlb2_priv.h"
  35 #include "dlb2_iface.h"
  36 #include "dlb2_inline_fns.h"
  37
  38 /*
  39  * Resources exposed to eventdev. Some values overridden at runtime using
  40  * values returned by the DLB kernel driver.
  41  */
  42 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
  43 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
  44 #endif
  45 static struct rte_event_dev_info evdev_dlb2_default_info = {
  46         .driver_name = "", /* probe will set */
  47         .min_dequeue_timeout_ns = DLB2_MIN_DEQUEUE_TIMEOUT_NS,
  48         .max_dequeue_timeout_ns = DLB2_MAX_DEQUEUE_TIMEOUT_NS,
  49 #if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB2_MAX_NUM_LDB_QUEUES)
  50         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
  51 #else
  52         .max_event_queues = DLB2_MAX_NUM_LDB_QUEUES,
  53 #endif
  54         .max_event_queue_flows = DLB2_MAX_NUM_FLOWS,
  55         .max_event_queue_priority_levels = DLB2_QID_PRIORITIES,
  56         .max_event_priority_levels = DLB2_QID_PRIORITIES,
  57         .max_event_ports = DLB2_MAX_NUM_LDB_PORTS,
  58         .max_event_port_dequeue_depth = DLB2_MAX_CQ_DEPTH,
  59         .max_event_port_enqueue_depth = DLB2_MAX_ENQUEUE_DEPTH,
  60         .max_event_port_links = DLB2_MAX_NUM_QIDS_PER_LDB_CQ,
  61         .max_num_events = DLB2_MAX_NUM_LDB_CREDITS,
  62         .max_single_link_event_port_queue_pairs =
  63                 DLB2_MAX_NUM_DIR_PORTS(DLB2_HW_V2),
  64         .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
  65                           RTE_EVENT_DEV_CAP_EVENT_QOS |
  66                           RTE_EVENT_DEV_CAP_BURST_MODE |
  67                           RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
  68                           RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
  69                           RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
  70 };
  71
  72 struct process_local_port_data
  73 dlb2_port[DLB2_MAX_NUM_PORTS_ALL][DLB2_NUM_PORT_TYPES];
  74
  75 static void
  76 dlb2_free_qe_mem(struct dlb2_port *qm_port)
  77 {
  78         if (qm_port == NULL)
  79                 return;
  80
  81         rte_free(qm_port->qe4);
  82         qm_port->qe4 = NULL;
  83
  84         rte_free(qm_port->int_arm_qe);
  85         qm_port->int_arm_qe = NULL;
  86
  87         rte_free(qm_port->consume_qe);
  88         qm_port->consume_qe = NULL;
  89
  90         rte_memzone_free(dlb2_port[qm_port->id][PORT_TYPE(qm_port)].mz);
  91         dlb2_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
  92 }
  93
  94 /* override defaults with value(s) provided on command line */
  95 static void
  96 dlb2_init_queue_depth_thresholds(struct dlb2_eventdev *dlb2,
  97                                  int *qid_depth_thresholds)
  98 {
  99         int q;
 100
 101         for (q = 0; q < DLB2_MAX_NUM_QUEUES(dlb2->version); q++) {
 102                 if (qid_depth_thresholds[q] != 0)
 103                         dlb2->ev_queues[q].depth_threshold =
 104                                 qid_depth_thresholds[q];
 105         }
 106 }
 107
 108 static int
 109 dlb2_hw_query_resources(struct dlb2_eventdev *dlb2)
 110 {
 111         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
 112         struct dlb2_hw_resource_info *dlb2_info = &handle->info;
 113         int ret;
 114
 115         /* Query driver resources provisioned for this device */
 116
 117         ret = dlb2_iface_get_num_resources(handle,
 118                                            &dlb2->hw_rsrc_query_results);
 119         if (ret) {
 120                 DLB2_LOG_ERR("ioctl get dlb2 num resources, err=%d\n", ret);
 121                 return ret;
 122         }
 123
 124         /* Complete filling in device resource info returned to evdev app,
 125          * overriding any default values.
 126          * The capabilities (CAPs) were set at compile time.
 127          */
 128
 129         evdev_dlb2_default_info.max_event_queues =
 130                 dlb2->hw_rsrc_query_results.num_ldb_queues;
 131
 132         evdev_dlb2_default_info.max_event_ports =
 133                 dlb2->hw_rsrc_query_results.num_ldb_ports;
 134
 135         if (dlb2->version == DLB2_HW_V2_5) {
 136                 evdev_dlb2_default_info.max_num_events =
 137                         dlb2->hw_rsrc_query_results.num_credits;
 138         } else {
 139                 evdev_dlb2_default_info.max_num_events =
 140                         dlb2->hw_rsrc_query_results.num_ldb_credits;
 141         }
 142         /* Save off values used when creating the scheduling domain. */
 143
 144         handle->info.num_sched_domains =
 145                 dlb2->hw_rsrc_query_results.num_sched_domains;
 146
 147         if (dlb2->version == DLB2_HW_V2_5) {
 148                 handle->info.hw_rsrc_max.nb_events_limit =
 149                         dlb2->hw_rsrc_query_results.num_credits;
 150         } else {
 151                 handle->info.hw_rsrc_max.nb_events_limit =
 152                         dlb2->hw_rsrc_query_results.num_ldb_credits;
 153         }
 154         handle->info.hw_rsrc_max.num_queues =
 155                 dlb2->hw_rsrc_query_results.num_ldb_queues +
 156                 dlb2->hw_rsrc_query_results.num_dir_ports;
 157
 158         handle->info.hw_rsrc_max.num_ldb_queues =
 159                 dlb2->hw_rsrc_query_results.num_ldb_queues;
 160
 161         handle->info.hw_rsrc_max.num_ldb_ports =
 162                 dlb2->hw_rsrc_query_results.num_ldb_ports;
 163
 164         handle->info.hw_rsrc_max.num_dir_ports =
 165                 dlb2->hw_rsrc_query_results.num_dir_ports;
 166
 167         handle->info.hw_rsrc_max.reorder_window_size =
 168                 dlb2->hw_rsrc_query_results.num_hist_list_entries;
 169
 170         rte_memcpy(dlb2_info, &handle->info.hw_rsrc_max, sizeof(*dlb2_info));
 171
 172         return 0;
 173 }
 174
 175 #define DLB2_BASE_10 10
 176
 177 static int
 178 dlb2_string_to_int(int *result, const char *str)
 179 {
 180         long ret;
 181         char *endptr;
 182
 183         if (str == NULL || result == NULL)
 184                 return -EINVAL;
 185
 186         errno = 0;
 187         ret = strtol(str, &endptr, DLB2_BASE_10);
 188         if (errno)
 189                 return -errno;
 190
 191         /* long int and int may be different width for some architectures */
 192         if (ret < INT_MIN || ret > INT_MAX || endptr == str)
 193                 return -EINVAL;
 194
 195         *result = ret;
 196         return 0;
 197 }
 198
 199 static int
 200 set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 201 {
 202         int *socket_id = opaque;
 203         int ret;
 204
 205         ret = dlb2_string_to_int(socket_id, value);
 206         if (ret < 0)
 207                 return ret;
 208
 209         if (*socket_id > RTE_MAX_NUMA_NODES)
 210                 return -EINVAL;
 211         return 0;
 212 }
 213
 214 static int
 215 set_max_num_events(const char *key __rte_unused,
 216                    const char *value,
 217                    void *opaque)
 218 {
 219         int *max_num_events = opaque;
 220         int ret;
 221
 222         if (value == NULL || opaque == NULL) {
 223                 DLB2_LOG_ERR("NULL pointer\n");
 224                 return -EINVAL;
 225         }
 226
 227         ret = dlb2_string_to_int(max_num_events, value);
 228         if (ret < 0)
 229                 return ret;
 230
 231         if (*max_num_events < 0 || *max_num_events >
 232                         DLB2_MAX_NUM_LDB_CREDITS) {
 233                 DLB2_LOG_ERR("dlb2: max_num_events must be between 0 and %d\n",
 234                              DLB2_MAX_NUM_LDB_CREDITS);
 235                 return -EINVAL;
 236         }
 237
 238         return 0;
 239 }
 240
 241 static int
 242 set_num_dir_credits(const char *key __rte_unused,
 243                     const char *value,
 244                     void *opaque)
 245 {
 246         int *num_dir_credits = opaque;
 247         int ret;
 248
 249         if (value == NULL || opaque == NULL) {
 250                 DLB2_LOG_ERR("NULL pointer\n");
 251                 return -EINVAL;
 252         }
 253
 254         ret = dlb2_string_to_int(num_dir_credits, value);
 255         if (ret < 0)
 256                 return ret;
 257
 258         if (*num_dir_credits < 0 ||
 259             *num_dir_credits > DLB2_MAX_NUM_DIR_CREDITS(DLB2_HW_V2)) {
 260                 DLB2_LOG_ERR("dlb2: num_dir_credits must be between 0 and %d\n",
 261                              DLB2_MAX_NUM_DIR_CREDITS(DLB2_HW_V2));
 262                 return -EINVAL;
 263         }
 264
 265         return 0;
 266 }
 267
 268 static int
 269 set_dev_id(const char *key __rte_unused,
 270            const char *value,
 271            void *opaque)
 272 {
 273         int *dev_id = opaque;
 274         int ret;
 275
 276         if (value == NULL || opaque == NULL) {
 277                 DLB2_LOG_ERR("NULL pointer\n");
 278                 return -EINVAL;
 279         }
 280
 281         ret = dlb2_string_to_int(dev_id, value);
 282         if (ret < 0)
 283                 return ret;
 284
 285         return 0;
 286 }
 287
 288 static int
 289 set_cos(const char *key __rte_unused,
 290         const char *value,
 291         void *opaque)
 292 {
 293         enum dlb2_cos *cos_id = opaque;
 294         int x = 0;
 295         int ret;
 296
 297         if (value == NULL || opaque == NULL) {
 298                 DLB2_LOG_ERR("NULL pointer\n");
 299                 return -EINVAL;
 300         }
 301
 302         ret = dlb2_string_to_int(&x, value);
 303         if (ret < 0)
 304                 return ret;
 305
 306         if (x != DLB2_COS_DEFAULT && (x < DLB2_COS_0 || x > DLB2_COS_3)) {
 307                 DLB2_LOG_ERR(
 308                         "COS %d out of range, must be DLB2_COS_DEFAULT or 0-3\n",
 309                         x);
 310                 return -EINVAL;
 311         }
 312
 313         *cos_id = x;
 314
 315         return 0;
 316 }
 317
 318 static int
 319 set_poll_interval(const char *key __rte_unused,
 320         const char *value,
 321         void *opaque)
 322 {
 323         int *poll_interval = opaque;
 324         int ret;
 325
 326         if (value == NULL || opaque == NULL) {
 327                 DLB2_LOG_ERR("NULL pointer\n");
 328                 return -EINVAL;
 329         }
 330
 331         ret = dlb2_string_to_int(poll_interval, value);
 332         if (ret < 0)
 333                 return ret;
 334
 335         return 0;
 336 }
 337
 338 static int
 339 set_sw_credit_quanta(const char *key __rte_unused,
 340         const char *value,
 341         void *opaque)
 342 {
 343         int *sw_credit_quanta = opaque;
 344         int ret;
 345
 346         if (value == NULL || opaque == NULL) {
 347                 DLB2_LOG_ERR("NULL pointer\n");
 348                 return -EINVAL;
 349         }
 350
 351         ret = dlb2_string_to_int(sw_credit_quanta, value);
 352         if (ret < 0)
 353                 return ret;
 354
 355         return 0;
 356 }
 357
 358 static int
 359 set_hw_credit_quanta(const char *key __rte_unused,
 360         const char *value,
 361         void *opaque)
 362 {
 363         int *hw_credit_quanta = opaque;
 364         int ret;
 365
 366         if (value == NULL || opaque == NULL) {
 367                 DLB2_LOG_ERR("NULL pointer\n");
 368                 return -EINVAL;
 369         }
 370
 371         ret = dlb2_string_to_int(hw_credit_quanta, value);
 372         if (ret < 0)
 373                 return ret;
 374
 375         return 0;
 376 }
 377
 378 static int
 379 set_default_depth_thresh(const char *key __rte_unused,
 380         const char *value,
 381         void *opaque)
 382 {
 383         int *default_depth_thresh = opaque;
 384         int ret;
 385
 386         if (value == NULL || opaque == NULL) {
 387                 DLB2_LOG_ERR("NULL pointer\n");
 388                 return -EINVAL;
 389         }
 390
 391         ret = dlb2_string_to_int(default_depth_thresh, value);
 392         if (ret < 0)
 393                 return ret;
 394
 395         return 0;
 396 }
 397
 398 static int
 399 set_vector_opts_enab(const char *key __rte_unused,
 400         const char *value,
 401         void *opaque)
 402 {
 403         bool *dlb2_vector_opts_enabled = opaque;
 404
 405         if (value == NULL || opaque == NULL) {
 406                 DLB2_LOG_ERR("NULL pointer\n");
 407                 return -EINVAL;
 408         }
 409
 410         if ((*value == 'y') || (*value == 'Y'))
 411                 *dlb2_vector_opts_enabled = true;
 412         else
 413                 *dlb2_vector_opts_enabled = false;
 414
 415         return 0;
 416 }
 417
 418 static int
 419 set_qid_depth_thresh(const char *key __rte_unused,
 420                      const char *value,
 421                      void *opaque)
 422 {
 423         struct dlb2_qid_depth_thresholds *qid_thresh = opaque;
 424         int first, last, thresh, i;
 425
 426         if (value == NULL || opaque == NULL) {
 427                 DLB2_LOG_ERR("NULL pointer\n");
 428                 return -EINVAL;
 429         }
 430
 431         /* command line override may take one of the following 3 forms:
 432          * qid_depth_thresh=all:<threshold_value> ... all queues
 433          * qid_depth_thresh=qidA-qidB:<threshold_value> ... a range of queues
 434          * qid_depth_thresh=qid:<threshold_value> ... just one queue
 435          */
 436         if (sscanf(value, "all:%d", &thresh) == 1) {
 437                 first = 0;
 438                 last = DLB2_MAX_NUM_QUEUES(DLB2_HW_V2) - 1;
 439         } else if (sscanf(value, "%d-%d:%d", &first, &last, &thresh) == 3) {
 440                 /* we have everything we need */
 441         } else if (sscanf(value, "%d:%d", &first, &thresh) == 2) {
 442                 last = first;
 443         } else {
 444                 DLB2_LOG_ERR("Error parsing qid depth devarg. Should be all:val, qid-qid:val, or qid:val\n");
 445                 return -EINVAL;
 446         }
 447
 448         if (first > last || first < 0 ||
 449                 last >= DLB2_MAX_NUM_QUEUES(DLB2_HW_V2)) {
 450                 DLB2_LOG_ERR("Error parsing qid depth devarg, invalid qid value\n");
 451                 return -EINVAL;
 452         }
 453
 454         if (thresh < 0 || thresh > DLB2_MAX_QUEUE_DEPTH_THRESHOLD) {
 455                 DLB2_LOG_ERR("Error parsing qid depth devarg, threshold > %d\n",
 456                              DLB2_MAX_QUEUE_DEPTH_THRESHOLD);
 457                 return -EINVAL;
 458         }
 459
 460         for (i = first; i <= last; i++)
 461                 qid_thresh->val[i] = thresh; /* indexed by qid */
 462
 463         return 0;
 464 }
 465
 466 static int
 467 set_qid_depth_thresh_v2_5(const char *key __rte_unused,
 468                           const char *value,
 469                           void *opaque)
 470 {
 471         struct dlb2_qid_depth_thresholds *qid_thresh = opaque;
 472         int first, last, thresh, i;
 473
 474         if (value == NULL || opaque == NULL) {
 475                 DLB2_LOG_ERR("NULL pointer\n");
 476                 return -EINVAL;
 477         }
 478
 479         /* command line override may take one of the following 3 forms:
 480          * qid_depth_thresh=all:<threshold_value> ... all queues
 481          * qid_depth_thresh=qidA-qidB:<threshold_value> ... a range of queues
 482          * qid_depth_thresh=qid:<threshold_value> ... just one queue
 483          */
 484         if (sscanf(value, "all:%d", &thresh) == 1) {
 485                 first = 0;
 486                 last = DLB2_MAX_NUM_QUEUES(DLB2_HW_V2_5) - 1;
 487         } else if (sscanf(value, "%d-%d:%d", &first, &last, &thresh) == 3) {
 488                 /* we have everything we need */
 489         } else if (sscanf(value, "%d:%d", &first, &thresh) == 2) {
 490                 last = first;
 491         } else {
 492                 DLB2_LOG_ERR("Error parsing qid depth devarg. Should be all:val, qid-qid:val, or qid:val\n");
 493                 return -EINVAL;
 494         }
 495
 496         if (first > last || first < 0 ||
 497                 last >= DLB2_MAX_NUM_QUEUES(DLB2_HW_V2_5)) {
 498                 DLB2_LOG_ERR("Error parsing qid depth devarg, invalid qid value\n");
 499                 return -EINVAL;
 500         }
 501
 502         if (thresh < 0 || thresh > DLB2_MAX_QUEUE_DEPTH_THRESHOLD) {
 503                 DLB2_LOG_ERR("Error parsing qid depth devarg, threshold > %d\n",
 504                              DLB2_MAX_QUEUE_DEPTH_THRESHOLD);
 505                 return -EINVAL;
 506         }
 507
 508         for (i = first; i <= last; i++)
 509                 qid_thresh->val[i] = thresh; /* indexed by qid */
 510
 511         return 0;
 512 }
 513
 514 static void
 515 dlb2_eventdev_info_get(struct rte_eventdev *dev,
 516                        struct rte_event_dev_info *dev_info)
 517 {
 518         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
 519         int ret;
 520
 521         ret = dlb2_hw_query_resources(dlb2);
 522         if (ret) {
 523                 const struct rte_eventdev_data *data = dev->data;
 524
 525                 DLB2_LOG_ERR("get resources err=%d, devid=%d\n",
 526                              ret, data->dev_id);
 527                 /* fn is void, so fall through and return values set up in
 528                  * probe
 529                  */
 530         }
 531
 532         /* Add num resources currently owned by this domain.
 533          * These would become available if the scheduling domain were reset due
 534          * to the application recalling eventdev_configure to *reconfigure* the
 535          * domain.
 536          */
 537         evdev_dlb2_default_info.max_event_ports += dlb2->num_ldb_ports;
 538         evdev_dlb2_default_info.max_event_queues += dlb2->num_ldb_queues;
 539         if (dlb2->version == DLB2_HW_V2_5) {
 540                 evdev_dlb2_default_info.max_num_events +=
 541                         dlb2->max_credits;
 542         } else {
 543                 evdev_dlb2_default_info.max_num_events +=
 544                         dlb2->max_ldb_credits;
 545         }
 546         evdev_dlb2_default_info.max_event_queues =
 547                 RTE_MIN(evdev_dlb2_default_info.max_event_queues,
 548                         RTE_EVENT_MAX_QUEUES_PER_DEV);
 549
 550         evdev_dlb2_default_info.max_num_events =
 551                 RTE_MIN(evdev_dlb2_default_info.max_num_events,
 552                         dlb2->max_num_events_override);
 553
 554         *dev_info = evdev_dlb2_default_info;
 555 }
 556
 557 static int
 558 dlb2_hw_create_sched_domain(struct dlb2_hw_dev *handle,
 559                             const struct dlb2_hw_rsrcs *resources_asked,
 560                             uint8_t device_version)
 561 {
 562         int ret = 0;
 563         struct dlb2_create_sched_domain_args *cfg;
 564
 565         if (resources_asked == NULL) {
 566                 DLB2_LOG_ERR("dlb2: dlb2_create NULL parameter\n");
 567                 ret = EINVAL;
 568                 goto error_exit;
 569         }
 570
 571         /* Map generic qm resources to dlb2 resources */
 572         cfg = &handle->cfg.resources;
 573
 574         /* DIR ports and queues */
 575
 576         cfg->num_dir_ports = resources_asked->num_dir_ports;
 577         if (device_version == DLB2_HW_V2_5)
 578                 cfg->num_credits = resources_asked->num_credits;
 579         else
 580                 cfg->num_dir_credits = resources_asked->num_dir_credits;
 581
 582         /* LDB queues */
 583
 584         cfg->num_ldb_queues = resources_asked->num_ldb_queues;
 585
 586         /* LDB ports */
 587
 588         cfg->cos_strict = 0; /* Best effort */
 589         cfg->num_cos_ldb_ports[0] = 0;
 590         cfg->num_cos_ldb_ports[1] = 0;
 591         cfg->num_cos_ldb_ports[2] = 0;
 592         cfg->num_cos_ldb_ports[3] = 0;
 593
 594         switch (handle->cos_id) {
 595         case DLB2_COS_0:
 596                 cfg->num_ldb_ports = 0; /* no don't care ports */
 597                 cfg->num_cos_ldb_ports[0] =
 598                         resources_asked->num_ldb_ports;
 599                 break;
 600         case DLB2_COS_1:
 601                 cfg->num_ldb_ports = 0; /* no don't care ports */
 602                 cfg->num_cos_ldb_ports[1] = resources_asked->num_ldb_ports;
 603                 break;
 604         case DLB2_COS_2:
 605                 cfg->num_ldb_ports = 0; /* no don't care ports */
 606                 cfg->num_cos_ldb_ports[2] = resources_asked->num_ldb_ports;
 607                 break;
 608         case DLB2_COS_3:
 609                 cfg->num_ldb_ports = 0; /* no don't care ports */
 610                 cfg->num_cos_ldb_ports[3] =
 611                         resources_asked->num_ldb_ports;
 612                 break;
 613         case DLB2_COS_DEFAULT:
 614                 /* all ldb ports are don't care ports from a cos perspective */
 615                 cfg->num_ldb_ports =
 616                         resources_asked->num_ldb_ports;
 617                 break;
 618         }
 619
 620         if (device_version == DLB2_HW_V2)
 621                 cfg->num_ldb_credits = resources_asked->num_ldb_credits;
 622
 623         cfg->num_atomic_inflights =
 624                 DLB2_NUM_ATOMIC_INFLIGHTS_PER_QUEUE *
 625                 cfg->num_ldb_queues;
 626
 627         cfg->num_hist_list_entries = resources_asked->num_ldb_ports *
 628                 DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
 629
 630         if (device_version == DLB2_HW_V2_5) {
 631                 DLB2_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, credits=%d\n",
 632                              cfg->num_ldb_queues,
 633                              resources_asked->num_ldb_ports,
 634                              cfg->num_dir_ports,
 635                              cfg->num_atomic_inflights,
 636                              cfg->num_hist_list_entries,
 637                              cfg->num_credits);
 638         } else {
 639                 DLB2_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d\n",
 640                              cfg->num_ldb_queues,
 641                              resources_asked->num_ldb_ports,
 642                              cfg->num_dir_ports,
 643                              cfg->num_atomic_inflights,
 644                              cfg->num_hist_list_entries,
 645                              cfg->num_ldb_credits,
 646                              cfg->num_dir_credits);
 647         }
 648
 649         /* Configure the QM */
 650
 651         ret = dlb2_iface_sched_domain_create(handle, cfg);
 652         if (ret < 0) {
 653                 DLB2_LOG_ERR("dlb2: domain create failed, ret = %d, extra status: %s\n",
 654                              ret,
 655                              dlb2_error_strings[cfg->response.status]);
 656
 657                 goto error_exit;
 658         }
 659
 660         handle->domain_id = cfg->response.id;
 661         handle->cfg.configured = true;
 662
 663 error_exit:
 664
 665         return ret;
 666 }
 667
 668 static void
 669 dlb2_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
 670 {
 671         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
 672         enum dlb2_configuration_state config_state;
 673         int i, j;
 674
 675         dlb2_iface_domain_reset(dlb2);
 676
 677         /* Free all dynamically allocated port memory */
 678         for (i = 0; i < dlb2->num_ports; i++)
 679                 dlb2_free_qe_mem(&dlb2->ev_ports[i].qm_port);
 680
 681         /* If reconfiguring, mark the device's queues and ports as "previously
 682          * configured." If the user doesn't reconfigure them, the PMD will
 683          * reapply their previous configuration when the device is started.
 684          */
 685         config_state = (reconfig) ? DLB2_PREV_CONFIGURED :
 686                 DLB2_NOT_CONFIGURED;
 687
 688         for (i = 0; i < dlb2->num_ports; i++) {
 689                 dlb2->ev_ports[i].qm_port.config_state = config_state;
 690                 /* Reset setup_done so ports can be reconfigured */
 691                 dlb2->ev_ports[i].setup_done = false;
 692                 for (j = 0; j < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; j++)
 693                         dlb2->ev_ports[i].link[j].mapped = false;
 694         }
 695
 696         for (i = 0; i < dlb2->num_queues; i++)
 697                 dlb2->ev_queues[i].qm_queue.config_state = config_state;
 698
 699         for (i = 0; i < DLB2_MAX_NUM_QUEUES(DLB2_HW_V2_5); i++)
 700                 dlb2->ev_queues[i].setup_done = false;
 701
 702         dlb2->num_ports = 0;
 703         dlb2->num_ldb_ports = 0;
 704         dlb2->num_dir_ports = 0;
 705         dlb2->num_queues = 0;
 706         dlb2->num_ldb_queues = 0;
 707         dlb2->num_dir_queues = 0;
 708         dlb2->configured = false;
 709 }
 710
 711 /* Note: 1 QM instance per QM device, QM instance/device == event device */
 712 static int
 713 dlb2_eventdev_configure(const struct rte_eventdev *dev)
 714 {
 715         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
 716         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
 717         struct dlb2_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
 718         const struct rte_eventdev_data *data = dev->data;
 719         const struct rte_event_dev_config *config = &data->dev_conf;
 720         int ret;
 721
 722         /* If this eventdev is already configured, we must release the current
 723          * scheduling domain before attempting to configure a new one.
 724          */
 725         if (dlb2->configured) {
 726                 dlb2_hw_reset_sched_domain(dev, true);
 727                 ret = dlb2_hw_query_resources(dlb2);
 728                 if (ret) {
 729                         DLB2_LOG_ERR("get resources err=%d, devid=%d\n",
 730                                      ret, data->dev_id);
 731                         return ret;
 732                 }
 733         }
 734
 735         if (config->nb_event_queues > rsrcs->num_queues) {
 736                 DLB2_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
 737                              config->nb_event_queues,
 738                              rsrcs->num_queues);
 739                 return -EINVAL;
 740         }
 741         if (config->nb_event_ports > (rsrcs->num_ldb_ports
 742                         + rsrcs->num_dir_ports)) {
 743                 DLB2_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
 744                              config->nb_event_ports,
 745                              (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
 746                 return -EINVAL;
 747         }
 748         if (config->nb_events_limit > rsrcs->nb_events_limit) {
 749                 DLB2_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
 750                              config->nb_events_limit,
 751                              rsrcs->nb_events_limit);
 752                 return -EINVAL;
 753         }
 754
 755         if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
 756                 dlb2->global_dequeue_wait = false;
 757         else {
 758                 uint32_t timeout32;
 759
 760                 dlb2->global_dequeue_wait = true;
 761
 762                 /* note size mismatch of timeout vals in eventdev lib. */
 763                 timeout32 = config->dequeue_timeout_ns;
 764
 765                 dlb2->global_dequeue_wait_ticks =
 766                         timeout32 * (rte_get_timer_hz() / 1E9);
 767         }
 768
 769         /* Does this platform support umonitor/umwait? */
 770         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG))
 771                 dlb2->umwait_allowed = true;
 772
 773         rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
 774         rsrcs->num_ldb_ports  = config->nb_event_ports - rsrcs->num_dir_ports;
 775         /* 1 dir queue per dir port */
 776         rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
 777
 778         if (dlb2->version == DLB2_HW_V2_5) {
 779                 rsrcs->num_credits = 0;
 780                 if (rsrcs->num_ldb_queues || rsrcs->num_dir_ports)
 781                         rsrcs->num_credits = config->nb_events_limit;
 782         } else {
 783                 /* Scale down nb_events_limit by 4 for directed credits,
 784                  * since there are 4x as many load-balanced credits.
 785                  */
 786                 rsrcs->num_ldb_credits = 0;
 787                 rsrcs->num_dir_credits = 0;
 788
 789                 if (rsrcs->num_ldb_queues)
 790                         rsrcs->num_ldb_credits = config->nb_events_limit;
 791                 if (rsrcs->num_dir_ports)
 792                         rsrcs->num_dir_credits = config->nb_events_limit / 2;
 793                 if (dlb2->num_dir_credits_override != -1)
 794                         rsrcs->num_dir_credits = dlb2->num_dir_credits_override;
 795         }
 796
 797         if (dlb2_hw_create_sched_domain(handle, rsrcs, dlb2->version) < 0) {
 798                 DLB2_LOG_ERR("dlb2_hw_create_sched_domain failed\n");
 799                 return -ENODEV;
 800         }
 801
 802         dlb2->new_event_limit = config->nb_events_limit;
 803         __atomic_store_n(&dlb2->inflights, 0, __ATOMIC_SEQ_CST);
 804
 805         /* Save number of ports/queues for this event dev */
 806         dlb2->num_ports = config->nb_event_ports;
 807         dlb2->num_queues = config->nb_event_queues;
 808         dlb2->num_dir_ports = rsrcs->num_dir_ports;
 809         dlb2->num_ldb_ports = dlb2->num_ports - dlb2->num_dir_ports;
 810         dlb2->num_ldb_queues = dlb2->num_queues - dlb2->num_dir_ports;
 811         dlb2->num_dir_queues = dlb2->num_dir_ports;
 812         if (dlb2->version == DLB2_HW_V2_5) {
 813                 dlb2->credit_pool = rsrcs->num_credits;
 814                 dlb2->max_credits = rsrcs->num_credits;
 815         } else {
 816                 dlb2->ldb_credit_pool = rsrcs->num_ldb_credits;
 817                 dlb2->max_ldb_credits = rsrcs->num_ldb_credits;
 818                 dlb2->dir_credit_pool = rsrcs->num_dir_credits;
 819                 dlb2->max_dir_credits = rsrcs->num_dir_credits;
 820         }
 821
 822         dlb2->configured = true;
 823
 824         return 0;
 825 }
 826
 827 static void
 828 dlb2_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 829                                     uint8_t port_id,
 830                                     struct rte_event_port_conf *port_conf)
 831 {
 832         RTE_SET_USED(port_id);
 833         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
 834
 835         port_conf->new_event_threshold = dlb2->new_event_limit;
 836         port_conf->dequeue_depth = 32;
 837         port_conf->enqueue_depth = DLB2_MAX_ENQUEUE_DEPTH;
 838         port_conf->event_port_cfg = 0;
 839 }
 840
 841 static void
 842 dlb2_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 843                                      uint8_t queue_id,
 844                                      struct rte_event_queue_conf *queue_conf)
 845 {
 846         RTE_SET_USED(dev);
 847         RTE_SET_USED(queue_id);
 848
 849         queue_conf->nb_atomic_flows = 1024;
 850         queue_conf->nb_atomic_order_sequences = 64;
 851         queue_conf->event_queue_cfg = 0;
 852         queue_conf->priority = 0;
 853 }
 854
 855 static int32_t
 856 dlb2_get_sn_allocation(struct dlb2_eventdev *dlb2, int group)
 857 {
 858         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
 859         struct dlb2_get_sn_allocation_args cfg;
 860         int ret;
 861
 862         cfg.group = group;
 863
 864         ret = dlb2_iface_get_sn_allocation(handle, &cfg);
 865         if (ret < 0) {
 866                 DLB2_LOG_ERR("dlb2: get_sn_allocation ret=%d (driver status: %s)\n",
 867                              ret, dlb2_error_strings[cfg.response.status]);
 868                 return ret;
 869         }
 870
 871         return cfg.response.id;
 872 }
 873
 874 static int
 875 dlb2_set_sn_allocation(struct dlb2_eventdev *dlb2, int group, int num)
 876 {
 877         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
 878         struct dlb2_set_sn_allocation_args cfg;
 879         int ret;
 880
 881         cfg.num = num;
 882         cfg.group = group;
 883
 884         ret = dlb2_iface_set_sn_allocation(handle, &cfg);
 885         if (ret < 0) {
 886                 DLB2_LOG_ERR("dlb2: set_sn_allocation ret=%d (driver status: %s)\n",
 887                              ret, dlb2_error_strings[cfg.response.status]);
 888                 return ret;
 889         }
 890
 891         return ret;
 892 }
 893
 894 static int32_t
 895 dlb2_get_sn_occupancy(struct dlb2_eventdev *dlb2, int group)
 896 {
 897         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
 898         struct dlb2_get_sn_occupancy_args cfg;
 899         int ret;
 900
 901         cfg.group = group;
 902
 903         ret = dlb2_iface_get_sn_occupancy(handle, &cfg);
 904         if (ret < 0) {
 905                 DLB2_LOG_ERR("dlb2: get_sn_occupancy ret=%d (driver status: %s)\n",
 906                              ret, dlb2_error_strings[cfg.response.status]);
 907                 return ret;
 908         }
 909
 910         return cfg.response.id;
 911 }
 912
 913 /* Query the current sequence number allocations and, if they conflict with the
 914  * requested LDB queue configuration, attempt to re-allocate sequence numbers.
 915  * This is best-effort; if it fails, the PMD will attempt to configure the
 916  * load-balanced queue and return an error.
 917  */
 918 static void
 919 dlb2_program_sn_allocation(struct dlb2_eventdev *dlb2,
 920                            const struct rte_event_queue_conf *queue_conf)
 921 {
 922         int grp_occupancy[DLB2_NUM_SN_GROUPS];
 923         int grp_alloc[DLB2_NUM_SN_GROUPS];
 924         int i, sequence_numbers;
 925
 926         sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
 927
 928         for (i = 0; i < DLB2_NUM_SN_GROUPS; i++) {
 929                 int total_slots;
 930
 931                 grp_alloc[i] = dlb2_get_sn_allocation(dlb2, i);
 932                 if (grp_alloc[i] < 0)
 933                         return;
 934
 935                 total_slots = DLB2_MAX_LDB_SN_ALLOC / grp_alloc[i];
 936
 937                 grp_occupancy[i] = dlb2_get_sn_occupancy(dlb2, i);
 938                 if (grp_occupancy[i] < 0)
 939                         return;
 940
 941                 /* DLB has at least one available slot for the requested
 942                  * sequence numbers, so no further configuration required.
 943                  */
 944                 if (grp_alloc[i] == sequence_numbers &&
 945                     grp_occupancy[i] < total_slots)
 946                         return;
 947         }
 948
 949         /* None of the sequence number groups are configured for the requested
 950          * sequence numbers, so we have to reconfigure one of them. This is
 951          * only possible if a group is not in use.
 952          */
 953         for (i = 0; i < DLB2_NUM_SN_GROUPS; i++) {
 954                 if (grp_occupancy[i] == 0)
 955                         break;
 956         }
 957
 958         if (i == DLB2_NUM_SN_GROUPS) {
 959                 DLB2_LOG_ERR("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
 960                        __func__, sequence_numbers);
 961                 return;
 962         }
 963
 964         /* Attempt to configure slot i with the requested number of sequence
 965          * numbers. Ignore the return value -- if this fails, the error will be
 966          * caught during subsequent queue configuration.
 967          */
 968         dlb2_set_sn_allocation(dlb2, i, sequence_numbers);
 969 }
 970
 971 static int32_t
 972 dlb2_hw_create_ldb_queue(struct dlb2_eventdev *dlb2,
 973                          struct dlb2_eventdev_queue *ev_queue,
 974                          const struct rte_event_queue_conf *evq_conf)
 975 {
 976         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
 977         struct dlb2_queue *queue = &ev_queue->qm_queue;
 978         struct dlb2_create_ldb_queue_args cfg;
 979         int32_t ret;
 980         uint32_t qm_qid;
 981         int sched_type = -1;
 982
 983         if (evq_conf == NULL)
 984                 return -EINVAL;
 985
 986         if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
 987                 if (evq_conf->nb_atomic_order_sequences != 0)
 988                         sched_type = RTE_SCHED_TYPE_ORDERED;
 989                 else
 990                         sched_type = RTE_SCHED_TYPE_PARALLEL;
 991         } else
 992                 sched_type = evq_conf->schedule_type;
 993
 994         cfg.num_atomic_inflights = DLB2_NUM_ATOMIC_INFLIGHTS_PER_QUEUE;
 995         cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
 996         cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
 997
 998         if (sched_type != RTE_SCHED_TYPE_ORDERED) {
 999                 cfg.num_sequence_numbers = 0;
1000                 cfg.num_qid_inflights = 2048;
1001         }
1002
1003         /* App should set this to the number of hardware flows they want, not
1004          * the overall number of flows they're going to use. E.g. if app is
1005          * using 64 flows and sets compression to 64, best-case they'll get
1006          * 64 unique hashed flows in hardware.
1007          */
1008         switch (evq_conf->nb_atomic_flows) {
1009         /* Valid DLB2 compression levels */
1010         case 64:
1011         case 128:
1012         case 256:
1013         case 512:
1014         case (1 * 1024): /* 1K */
1015         case (2 * 1024): /* 2K */
1016         case (4 * 1024): /* 4K */
1017         case (64 * 1024): /* 64K */
1018                 cfg.lock_id_comp_level = evq_conf->nb_atomic_flows;
1019                 break;
1020         default:
1021                 /* Invalid compression level */
1022                 cfg.lock_id_comp_level = 0; /* no compression */
1023         }
1024
1025         if (ev_queue->depth_threshold == 0) {
1026                 cfg.depth_threshold = dlb2->default_depth_thresh;
1027                 ev_queue->depth_threshold =
1028                         dlb2->default_depth_thresh;
1029         } else
1030                 cfg.depth_threshold = ev_queue->depth_threshold;
1031
1032         ret = dlb2_iface_ldb_queue_create(handle, &cfg);
1033         if (ret < 0) {
1034                 DLB2_LOG_ERR("dlb2: create LB event queue error, ret=%d (driver status: %s)\n",
1035                              ret, dlb2_error_strings[cfg.response.status]);
1036                 return -EINVAL;
1037         }
1038
1039         qm_qid = cfg.response.id;
1040
1041         /* Save off queue config for debug, resource lookups, and reconfig */
1042         queue->num_qid_inflights = cfg.num_qid_inflights;
1043         queue->num_atm_inflights = cfg.num_atomic_inflights;
1044
1045         queue->sched_type = sched_type;
1046         queue->config_state = DLB2_CONFIGURED;
1047
1048         DLB2_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
1049                      qm_qid,
1050                      cfg.num_atomic_inflights,
1051                      cfg.num_sequence_numbers,
1052                      cfg.num_qid_inflights);
1053
1054         return qm_qid;
1055 }
1056
1057 static int
1058 dlb2_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
1059                               struct dlb2_eventdev_queue *ev_queue,
1060                               const struct rte_event_queue_conf *queue_conf)
1061 {
1062         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
1063         int32_t qm_qid;
1064
1065         if (queue_conf->nb_atomic_order_sequences)
1066                 dlb2_program_sn_allocation(dlb2, queue_conf);
1067
1068         qm_qid = dlb2_hw_create_ldb_queue(dlb2, ev_queue, queue_conf);
1069         if (qm_qid < 0) {
1070                 DLB2_LOG_ERR("Failed to create the load-balanced queue\n");
1071
1072                 return qm_qid;
1073         }
1074
1075         dlb2->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
1076
1077         ev_queue->qm_queue.id = qm_qid;
1078
1079         return 0;
1080 }
1081
1082 static int dlb2_num_dir_queues_setup(struct dlb2_eventdev *dlb2)
1083 {
1084         int i, num = 0;
1085
1086         for (i = 0; i < dlb2->num_queues; i++) {
1087                 if (dlb2->ev_queues[i].setup_done &&
1088                     dlb2->ev_queues[i].qm_queue.is_directed)
1089                         num++;
1090         }
1091
1092         return num;
1093 }
1094
1095 static void
1096 dlb2_queue_link_teardown(struct dlb2_eventdev *dlb2,
1097                          struct dlb2_eventdev_queue *ev_queue)
1098 {
1099         struct dlb2_eventdev_port *ev_port;
1100         int i, j;
1101
1102         for (i = 0; i < dlb2->num_ports; i++) {
1103                 ev_port = &dlb2->ev_ports[i];
1104
1105                 for (j = 0; j < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
1106                         if (!ev_port->link[j].valid ||
1107                             ev_port->link[j].queue_id != ev_queue->id)
1108                                 continue;
1109
1110                         ev_port->link[j].valid = false;
1111                         ev_port->num_links--;
1112                 }
1113         }
1114
1115         ev_queue->num_links = 0;
1116 }
1117
1118 static int
1119 dlb2_eventdev_queue_setup(struct rte_eventdev *dev,
1120                           uint8_t ev_qid,
1121                           const struct rte_event_queue_conf *queue_conf)
1122 {
1123         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
1124         struct dlb2_eventdev_queue *ev_queue;
1125         int ret;
1126
1127         if (queue_conf == NULL)
1128                 return -EINVAL;
1129
1130         if (ev_qid >= dlb2->num_queues)
1131                 return -EINVAL;
1132
1133         ev_queue = &dlb2->ev_queues[ev_qid];
1134
1135         ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
1136                 RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
1137         ev_queue->id = ev_qid;
1138         ev_queue->conf = *queue_conf;
1139
1140         if (!ev_queue->qm_queue.is_directed) {
1141                 ret = dlb2_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
1142         } else {
1143                 /* The directed queue isn't setup until link time, at which
1144                  * point we know its directed port ID. Directed queue setup
1145                  * will only fail if this queue is already setup or there are
1146                  * no directed queues left to configure.
1147                  */
1148                 ret = 0;
1149
1150                 ev_queue->qm_queue.config_state = DLB2_NOT_CONFIGURED;
1151
1152                 if (ev_queue->setup_done ||
1153                     dlb2_num_dir_queues_setup(dlb2) == dlb2->num_dir_queues)
1154                         ret = -EINVAL;
1155         }
1156
1157         /* Tear down pre-existing port->queue links */
1158         if (!ret && dlb2->run_state == DLB2_RUN_STATE_STOPPED)
1159                 dlb2_queue_link_teardown(dlb2, ev_queue);
1160
1161         if (!ret)
1162                 ev_queue->setup_done = true;
1163
1164         return ret;
1165 }
1166
1167 static int
1168 dlb2_init_consume_qe(struct dlb2_port *qm_port, char *mz_name)
1169 {
1170         struct dlb2_cq_pop_qe *qe;
1171
1172         qe = rte_zmalloc(mz_name,
1173                         DLB2_NUM_QES_PER_CACHE_LINE *
1174                                 sizeof(struct dlb2_cq_pop_qe),
1175                         RTE_CACHE_LINE_SIZE);
1176
1177         if (qe == NULL) {
1178                 DLB2_LOG_ERR("dlb2: no memory for consume_qe\n");
1179                 return -ENOMEM;
1180         }
1181         qm_port->consume_qe = qe;
1182
1183         qe->qe_valid = 0;
1184         qe->qe_frag = 0;
1185         qe->qe_comp = 0;
1186         qe->cq_token = 1;
1187         /* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
1188          * and so on.
1189          */
1190         qe->tokens = 0; /* set at run time */
1191         qe->meas_lat = 0;
1192         qe->no_dec = 0;
1193         /* Completion IDs are disabled */
1194         qe->cmp_id = 0;
1195
1196         return 0;
1197 }
1198
1199 static int
1200 dlb2_init_int_arm_qe(struct dlb2_port *qm_port, char *mz_name)
1201 {
1202         struct dlb2_enqueue_qe *qe;
1203
1204         qe = rte_zmalloc(mz_name,
1205                         DLB2_NUM_QES_PER_CACHE_LINE *
1206                                 sizeof(struct dlb2_enqueue_qe),
1207                         RTE_CACHE_LINE_SIZE);
1208
1209         if (qe == NULL) {
1210                 DLB2_LOG_ERR("dlb2: no memory for complete_qe\n");
1211                 return -ENOMEM;
1212         }
1213         qm_port->int_arm_qe = qe;
1214
1215         /* V2 - INT ARM is CQ_TOKEN + FRAG */
1216         qe->qe_valid = 0;
1217         qe->qe_frag = 1;
1218         qe->qe_comp = 0;
1219         qe->cq_token = 1;
1220         qe->meas_lat = 0;
1221         qe->no_dec = 0;
1222         /* Completion IDs are disabled */
1223         qe->cmp_id = 0;
1224
1225         return 0;
1226 }
1227
1228 static int
1229 dlb2_init_qe_mem(struct dlb2_port *qm_port, char *mz_name)
1230 {
1231         int ret, sz;
1232
1233         sz = DLB2_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb2_enqueue_qe);
1234
1235         qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
1236
1237         if (qm_port->qe4 == NULL) {
1238                 DLB2_LOG_ERR("dlb2: no qe4 memory\n");
1239                 ret = -ENOMEM;
1240                 goto error_exit;
1241         }
1242
1243         ret = dlb2_init_int_arm_qe(qm_port, mz_name);
1244         if (ret < 0) {
1245                 DLB2_LOG_ERR("dlb2: dlb2_init_int_arm_qe ret=%d\n", ret);
1246                 goto error_exit;
1247         }
1248
1249         ret = dlb2_init_consume_qe(qm_port, mz_name);
1250         if (ret < 0) {
1251                 DLB2_LOG_ERR("dlb2: dlb2_init_consume_qe ret=%d\n", ret);
1252                 goto error_exit;
1253         }
1254
1255         return 0;
1256
1257 error_exit:
1258
1259         dlb2_free_qe_mem(qm_port);
1260
1261         return ret;
1262 }
1263
1264 static inline uint16_t
1265 dlb2_event_enqueue_delayed(void *event_port,
1266                            const struct rte_event events[]);
1267
1268 static inline uint16_t
1269 dlb2_event_enqueue_burst_delayed(void *event_port,
1270                                  const struct rte_event events[],
1271                                  uint16_t num);
1272
1273 static inline uint16_t
1274 dlb2_event_enqueue_new_burst_delayed(void *event_port,
1275                                      const struct rte_event events[],
1276                                      uint16_t num);
1277
1278 static inline uint16_t
1279 dlb2_event_enqueue_forward_burst_delayed(void *event_port,
1280                                          const struct rte_event events[],
1281                                          uint16_t num);
1282
1283 /* Generate the required bitmask for rotate-style expected QE gen bits.
1284  * This requires a pattern of 1's and zeros, starting with expected as
1285  * 1 bits, so when hardware writes 0's they're "new". This requires the
1286  * ring size to be powers of 2 to wrap correctly.
1287  */
1288 static void
1289 dlb2_hw_cq_bitmask_init(struct dlb2_port *qm_port, uint32_t cq_depth)
1290 {
1291         uint64_t cq_build_mask = 0;
1292         uint32_t i;
1293
1294         if (cq_depth > 64)
1295                 return; /* need to fall back to scalar code */
1296
1297         /*
1298          * all 1's in first u64, all zeros in second is correct bit pattern to
1299          * start. Special casing == 64 easier than adapting complex loop logic.
1300          */
1301         if (cq_depth == 64) {
1302                 qm_port->cq_rolling_mask = 0;
1303                 qm_port->cq_rolling_mask_2 = -1;
1304                 return;
1305         }
1306
1307         for (i = 0; i < 64; i += (cq_depth * 2))
1308                 cq_build_mask |= ((1ULL << cq_depth) - 1) << (i + cq_depth);
1309
1310         qm_port->cq_rolling_mask = cq_build_mask;
1311         qm_port->cq_rolling_mask_2 = cq_build_mask;
1312 }
1313
1314 static int
1315 dlb2_hw_create_ldb_port(struct dlb2_eventdev *dlb2,
1316                         struct dlb2_eventdev_port *ev_port,
1317                         uint32_t dequeue_depth,
1318                         uint32_t enqueue_depth)
1319 {
1320         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
1321         struct dlb2_create_ldb_port_args cfg = { {0} };
1322         int ret;
1323         struct dlb2_port *qm_port = NULL;
1324         char mz_name[RTE_MEMZONE_NAMESIZE];
1325         uint32_t qm_port_id;
1326         uint16_t ldb_credit_high_watermark = 0;
1327         uint16_t dir_credit_high_watermark = 0;
1328         uint16_t credit_high_watermark = 0;
1329
1330         if (handle == NULL)
1331                 return -EINVAL;
1332
1333         if (dequeue_depth < DLB2_MIN_CQ_DEPTH) {
1334                 DLB2_LOG_ERR("dlb2: invalid enqueue_depth, must be at least %d\n",
1335                              DLB2_MIN_CQ_DEPTH);
1336                 return -EINVAL;
1337         }
1338
1339         if (enqueue_depth < DLB2_MIN_ENQUEUE_DEPTH) {
1340                 DLB2_LOG_ERR("dlb2: invalid enqueue_depth, must be at least %d\n",
1341                              DLB2_MIN_ENQUEUE_DEPTH);
1342                 return -EINVAL;
1343         }
1344
1345         rte_spinlock_lock(&handle->resource_lock);
1346
1347         /* We round up to the next power of 2 if necessary */
1348         cfg.cq_depth = rte_align32pow2(dequeue_depth);
1349         cfg.cq_depth_threshold = 1;
1350
1351         cfg.cq_history_list_size = DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
1352
1353         if (handle->cos_id == DLB2_COS_DEFAULT)
1354                 cfg.cos_id = 0;
1355         else
1356                 cfg.cos_id = handle->cos_id;
1357
1358         cfg.cos_strict = 0;
1359
1360         /* User controls the LDB high watermark via enqueue depth. The DIR high
1361          * watermark is equal, unless the directed credit pool is too small.
1362          */
1363         if (dlb2->version == DLB2_HW_V2) {
1364                 ldb_credit_high_watermark = enqueue_depth;
1365                 /* If there are no directed ports, the kernel driver will
1366                  * ignore this port's directed credit settings. Don't use
1367                  * enqueue_depth if it would require more directed credits
1368                  * than are available.
1369                  */
1370                 dir_credit_high_watermark =
1371                         RTE_MIN(enqueue_depth,
1372                                 handle->cfg.num_dir_credits / dlb2->num_ports);
1373         } else
1374                 credit_high_watermark = enqueue_depth;
1375
1376         /* Per QM values */
1377
1378         ret = dlb2_iface_ldb_port_create(handle, &cfg,  dlb2->poll_mode);
1379         if (ret < 0) {
1380                 DLB2_LOG_ERR("dlb2: dlb2_ldb_port_create error, ret=%d (driver status: %s)\n",
1381                              ret, dlb2_error_strings[cfg.response.status]);
1382                 goto error_exit;
1383         }
1384
1385         qm_port_id = cfg.response.id;
1386
1387         DLB2_LOG_DBG("dlb2: ev_port %d uses qm LB port %d <<<<<\n",
1388                      ev_port->id, qm_port_id);
1389
1390         qm_port = &ev_port->qm_port;
1391         qm_port->ev_port = ev_port; /* back ptr */
1392         qm_port->dlb2 = dlb2; /* back ptr */
1393         /*
1394          * Allocate and init local qe struct(s).
1395          * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
1396          */
1397
1398         snprintf(mz_name, sizeof(mz_name), "dlb2_ldb_port%d",
1399                  ev_port->id);
1400
1401         ret = dlb2_init_qe_mem(qm_port, mz_name);
1402         if (ret < 0) {
1403                 DLB2_LOG_ERR("dlb2: init_qe_mem failed, ret=%d\n", ret);
1404                 goto error_exit;
1405         }
1406
1407         qm_port->id = qm_port_id;
1408
1409         if (dlb2->version == DLB2_HW_V2) {
1410                 qm_port->cached_ldb_credits = 0;
1411                 qm_port->cached_dir_credits = 0;
1412         } else
1413                 qm_port->cached_credits = 0;
1414
1415         /* CQs with depth < 8 use an 8-entry queue, but withhold credits so
1416          * the effective depth is smaller.
1417          */
1418         qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
1419         qm_port->cq_idx = 0;
1420         qm_port->cq_idx_unmasked = 0;
1421
1422         if (dlb2->poll_mode == DLB2_CQ_POLL_MODE_SPARSE)
1423                 qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
1424         else
1425                 qm_port->cq_depth_mask = qm_port->cq_depth - 1;
1426
1427         qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
1428         /* starting value of gen bit - it toggles at wrap time */
1429         qm_port->gen_bit = 1;
1430
1431         dlb2_hw_cq_bitmask_init(qm_port, qm_port->cq_depth);
1432
1433         qm_port->int_armed = false;
1434
1435         /* Save off for later use in info and lookup APIs. */
1436         qm_port->qid_mappings = &dlb2->qm_ldb_to_ev_queue_id[0];
1437
1438         qm_port->dequeue_depth = dequeue_depth;
1439         qm_port->token_pop_thresh = dequeue_depth;
1440
1441         /* The default enqueue functions do not include delayed-pop support for
1442          * performance reasons.
1443          */
1444         if (qm_port->token_pop_mode == DELAYED_POP) {
1445                 dlb2->event_dev->enqueue = dlb2_event_enqueue_delayed;
1446                 dlb2->event_dev->enqueue_burst =
1447                         dlb2_event_enqueue_burst_delayed;
1448                 dlb2->event_dev->enqueue_new_burst =
1449                         dlb2_event_enqueue_new_burst_delayed;
1450                 dlb2->event_dev->enqueue_forward_burst =
1451                         dlb2_event_enqueue_forward_burst_delayed;
1452         }
1453
1454         qm_port->owed_tokens = 0;
1455         qm_port->issued_releases = 0;
1456
1457         /* Save config message too. */
1458         rte_memcpy(&qm_port->cfg.ldb, &cfg, sizeof(qm_port->cfg.ldb));
1459
1460         /* update state */
1461         qm_port->state = PORT_STARTED; /* enabled at create time */
1462         qm_port->config_state = DLB2_CONFIGURED;
1463
1464         if (dlb2->version == DLB2_HW_V2) {
1465                 qm_port->dir_credits = dir_credit_high_watermark;
1466                 qm_port->ldb_credits = ldb_credit_high_watermark;
1467                 qm_port->credit_pool[DLB2_DIR_QUEUE] = &dlb2->dir_credit_pool;
1468                 qm_port->credit_pool[DLB2_LDB_QUEUE] = &dlb2->ldb_credit_pool;
1469
1470                 DLB2_LOG_DBG("dlb2: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
1471                              qm_port_id,
1472                              dequeue_depth,
1473                              qm_port->ldb_credits,
1474                              qm_port->dir_credits);
1475         } else {
1476                 qm_port->credits = credit_high_watermark;
1477                 qm_port->credit_pool[DLB2_COMBINED_POOL] = &dlb2->credit_pool;
1478
1479                 DLB2_LOG_DBG("dlb2: created ldb port %d, depth = %d, credits=%d\n",
1480                              qm_port_id,
1481                              dequeue_depth,
1482                              qm_port->credits);
1483         }
1484
1485         qm_port->use_scalar = false;
1486
1487 #if (!defined RTE_ARCH_X86_64)
1488         qm_port->use_scalar = true;
1489 #else
1490         if ((qm_port->cq_depth > 64) ||
1491             (!rte_is_power_of_2(qm_port->cq_depth)) ||
1492             (dlb2->vector_opts_enabled == false))
1493                 qm_port->use_scalar = true;
1494 #endif
1495
1496         rte_spinlock_unlock(&handle->resource_lock);
1497
1498         return 0;
1499
1500 error_exit:
1501
1502         if (qm_port)
1503                 dlb2_free_qe_mem(qm_port);
1504
1505         rte_spinlock_unlock(&handle->resource_lock);
1506
1507         DLB2_LOG_ERR("dlb2: create ldb port failed!\n");
1508
1509         return ret;
1510 }
1511
1512 static void
1513 dlb2_port_link_teardown(struct dlb2_eventdev *dlb2,
1514                         struct dlb2_eventdev_port *ev_port)
1515 {
1516         struct dlb2_eventdev_queue *ev_queue;
1517         int i;
1518
1519         for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
1520                 if (!ev_port->link[i].valid)
1521                         continue;
1522
1523                 ev_queue = &dlb2->ev_queues[ev_port->link[i].queue_id];
1524
1525                 ev_port->link[i].valid = false;
1526                 ev_port->num_links--;
1527                 ev_queue->num_links--;
1528         }
1529 }
1530
1531 static int
1532 dlb2_hw_create_dir_port(struct dlb2_eventdev *dlb2,
1533                         struct dlb2_eventdev_port *ev_port,
1534                         uint32_t dequeue_depth,
1535                         uint32_t enqueue_depth)
1536 {
1537         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
1538         struct dlb2_create_dir_port_args cfg = { {0} };
1539         int ret;
1540         struct dlb2_port *qm_port = NULL;
1541         char mz_name[RTE_MEMZONE_NAMESIZE];
1542         uint32_t qm_port_id;
1543         uint16_t ldb_credit_high_watermark = 0;
1544         uint16_t dir_credit_high_watermark = 0;
1545         uint16_t credit_high_watermark = 0;
1546
1547         if (dlb2 == NULL || handle == NULL)
1548                 return -EINVAL;
1549
1550         if (dequeue_depth < DLB2_MIN_CQ_DEPTH) {
1551                 DLB2_LOG_ERR("dlb2: invalid dequeue_depth, must be %d-%d\n",
1552                              DLB2_MIN_CQ_DEPTH, DLB2_MAX_INPUT_QUEUE_DEPTH);
1553                 return -EINVAL;
1554         }
1555
1556         if (enqueue_depth < DLB2_MIN_ENQUEUE_DEPTH) {
1557                 DLB2_LOG_ERR("dlb2: invalid enqueue_depth, must be at least %d\n",
1558                              DLB2_MIN_ENQUEUE_DEPTH);
1559                 return -EINVAL;
1560         }
1561
1562         rte_spinlock_lock(&handle->resource_lock);
1563
1564         /* Directed queues are configured at link time. */
1565         cfg.queue_id = -1;
1566
1567         /* We round up to the next power of 2 if necessary */
1568         cfg.cq_depth = rte_align32pow2(dequeue_depth);
1569         cfg.cq_depth_threshold = 1;
1570
1571         /* User controls the LDB high watermark via enqueue depth. The DIR high
1572          * watermark is equal, unless the directed credit pool is too small.
1573          */
1574         if (dlb2->version == DLB2_HW_V2) {
1575                 ldb_credit_high_watermark = enqueue_depth;
1576                 /* Don't use enqueue_depth if it would require more directed
1577                  * credits than are available.
1578                  */
1579                 dir_credit_high_watermark =
1580                         RTE_MIN(enqueue_depth,
1581                                 handle->cfg.num_dir_credits / dlb2->num_ports);
1582         } else
1583                 credit_high_watermark = enqueue_depth;
1584
1585         /* Per QM values */
1586
1587         ret = dlb2_iface_dir_port_create(handle, &cfg,  dlb2->poll_mode);
1588         if (ret < 0) {
1589                 DLB2_LOG_ERR("dlb2: dlb2_dir_port_create error, ret=%d (driver status: %s)\n",
1590                              ret, dlb2_error_strings[cfg.response.status]);
1591                 goto error_exit;
1592         }
1593
1594         qm_port_id = cfg.response.id;
1595
1596         DLB2_LOG_DBG("dlb2: ev_port %d uses qm DIR port %d <<<<<\n",
1597                      ev_port->id, qm_port_id);
1598
1599         qm_port = &ev_port->qm_port;
1600         qm_port->ev_port = ev_port; /* back ptr */
1601         qm_port->dlb2 = dlb2;  /* back ptr */
1602
1603         /*
1604          * Init local qe struct(s).
1605          * Note: MOVDIR64 requires the enqueue QE to be aligned
1606          */
1607
1608         snprintf(mz_name, sizeof(mz_name), "dlb2_dir_port%d",
1609                  ev_port->id);
1610
1611         ret = dlb2_init_qe_mem(qm_port, mz_name);
1612
1613         if (ret < 0) {
1614                 DLB2_LOG_ERR("dlb2: init_qe_mem failed, ret=%d\n", ret);
1615                 goto error_exit;
1616         }
1617
1618         qm_port->id = qm_port_id;
1619
1620         if (dlb2->version == DLB2_HW_V2) {
1621                 qm_port->cached_ldb_credits = 0;
1622                 qm_port->cached_dir_credits = 0;
1623         } else
1624                 qm_port->cached_credits = 0;
1625
1626         /* CQs with depth < 8 use an 8-entry queue, but withhold credits so
1627          * the effective depth is smaller.
1628          */
1629         qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
1630         qm_port->cq_idx = 0;
1631         qm_port->cq_idx_unmasked = 0;
1632
1633         if (dlb2->poll_mode == DLB2_CQ_POLL_MODE_SPARSE)
1634                 qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
1635         else
1636                 qm_port->cq_depth_mask = cfg.cq_depth - 1;
1637
1638         qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
1639         /* starting value of gen bit - it toggles at wrap time */
1640         qm_port->gen_bit = 1;
1641         dlb2_hw_cq_bitmask_init(qm_port, qm_port->cq_depth);
1642
1643         qm_port->int_armed = false;
1644
1645         /* Save off for later use in info and lookup APIs. */
1646         qm_port->qid_mappings = &dlb2->qm_dir_to_ev_queue_id[0];
1647
1648         qm_port->dequeue_depth = dequeue_depth;
1649
1650         /* Directed ports are auto-pop, by default. */
1651         qm_port->token_pop_mode = AUTO_POP;
1652         qm_port->owed_tokens = 0;
1653         qm_port->issued_releases = 0;
1654
1655         /* Save config message too. */
1656         rte_memcpy(&qm_port->cfg.dir, &cfg, sizeof(qm_port->cfg.dir));
1657
1658         /* update state */
1659         qm_port->state = PORT_STARTED; /* enabled at create time */
1660         qm_port->config_state = DLB2_CONFIGURED;
1661
1662         if (dlb2->version == DLB2_HW_V2) {
1663                 qm_port->dir_credits = dir_credit_high_watermark;
1664                 qm_port->ldb_credits = ldb_credit_high_watermark;
1665                 qm_port->credit_pool[DLB2_DIR_QUEUE] = &dlb2->dir_credit_pool;
1666                 qm_port->credit_pool[DLB2_LDB_QUEUE] = &dlb2->ldb_credit_pool;
1667
1668                 DLB2_LOG_DBG("dlb2: created dir port %d, depth = %d cr=%d,%d\n",
1669                              qm_port_id,
1670                              dequeue_depth,
1671                              dir_credit_high_watermark,
1672                              ldb_credit_high_watermark);
1673         } else {
1674                 qm_port->credits = credit_high_watermark;
1675                 qm_port->credit_pool[DLB2_COMBINED_POOL] = &dlb2->credit_pool;
1676
1677                 DLB2_LOG_DBG("dlb2: created dir port %d, depth = %d cr=%d\n",
1678                              qm_port_id,
1679                              dequeue_depth,
1680                              credit_high_watermark);
1681         }
1682
1683 #if (!defined RTE_ARCH_X86_64)
1684         qm_port->use_scalar = true;
1685 #else
1686         if ((qm_port->cq_depth > 64) ||
1687             (!rte_is_power_of_2(qm_port->cq_depth)) ||
1688             (dlb2->vector_opts_enabled == false))
1689                 qm_port->use_scalar = true;
1690 #endif
1691
1692         rte_spinlock_unlock(&handle->resource_lock);
1693
1694         return 0;
1695
1696 error_exit:
1697
1698         if (qm_port)
1699                 dlb2_free_qe_mem(qm_port);
1700
1701         rte_spinlock_unlock(&handle->resource_lock);
1702
1703         DLB2_LOG_ERR("dlb2: create dir port failed!\n");
1704
1705         return ret;
1706 }
1707
1708 static int
1709 dlb2_eventdev_port_setup(struct rte_eventdev *dev,
1710                          uint8_t ev_port_id,
1711                          const struct rte_event_port_conf *port_conf)
1712 {
1713         struct dlb2_eventdev *dlb2;
1714         struct dlb2_eventdev_port *ev_port;
1715         int ret;
1716         uint32_t hw_credit_quanta, sw_credit_quanta;
1717
1718         if (dev == NULL || port_conf == NULL) {
1719                 DLB2_LOG_ERR("Null parameter\n");
1720                 return -EINVAL;
1721         }
1722
1723         dlb2 = dlb2_pmd_priv(dev);
1724
1725         if (ev_port_id >= DLB2_MAX_NUM_PORTS(dlb2->version))
1726                 return -EINVAL;
1727
1728         if (port_conf->dequeue_depth >
1729                 evdev_dlb2_default_info.max_event_port_dequeue_depth ||
1730             port_conf->enqueue_depth >
1731                 evdev_dlb2_default_info.max_event_port_enqueue_depth)
1732                 return -EINVAL;
1733
1734         ev_port = &dlb2->ev_ports[ev_port_id];
1735         /* configured? */
1736         if (ev_port->setup_done) {
1737                 DLB2_LOG_ERR("evport %d is already configured\n", ev_port_id);
1738                 return -EINVAL;
1739         }
1740
1741         ev_port->qm_port.is_directed = port_conf->event_port_cfg &
1742                 RTE_EVENT_PORT_CFG_SINGLE_LINK;
1743
1744         if (!ev_port->qm_port.is_directed) {
1745                 ret = dlb2_hw_create_ldb_port(dlb2,
1746                                               ev_port,
1747                                               port_conf->dequeue_depth,
1748                                               port_conf->enqueue_depth);
1749                 if (ret < 0) {
1750                         DLB2_LOG_ERR("Failed to create the lB port ve portId=%d\n",
1751                                      ev_port_id);
1752
1753                         return ret;
1754                 }
1755         } else {
1756                 ret = dlb2_hw_create_dir_port(dlb2,
1757                                               ev_port,
1758                                               port_conf->dequeue_depth,
1759                                               port_conf->enqueue_depth);
1760                 if (ret < 0) {
1761                         DLB2_LOG_ERR("Failed to create the DIR port\n");
1762                         return ret;
1763                 }
1764         }
1765
1766         /* Save off port config for reconfig */
1767         ev_port->conf = *port_conf;
1768
1769         ev_port->id = ev_port_id;
1770         ev_port->enq_configured = true;
1771         ev_port->setup_done = true;
1772         ev_port->inflight_max = port_conf->new_event_threshold;
1773         ev_port->implicit_release = !(port_conf->event_port_cfg &
1774                   RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
1775         ev_port->outstanding_releases = 0;
1776         ev_port->inflight_credits = 0;
1777         ev_port->dlb2 = dlb2; /* reverse link */
1778
1779         /* Default for worker ports */
1780         sw_credit_quanta = dlb2->sw_credit_quanta;
1781         hw_credit_quanta = dlb2->hw_credit_quanta;
1782
1783         if (port_conf->event_port_cfg & RTE_EVENT_PORT_CFG_HINT_PRODUCER) {
1784                 /* Producer type ports. Mostly enqueue */
1785                 sw_credit_quanta = DLB2_SW_CREDIT_P_QUANTA_DEFAULT;
1786                 hw_credit_quanta = DLB2_SW_CREDIT_P_BATCH_SZ;
1787         }
1788         if (port_conf->event_port_cfg & RTE_EVENT_PORT_CFG_HINT_CONSUMER) {
1789                 /* Consumer type ports. Mostly dequeue */
1790                 sw_credit_quanta = DLB2_SW_CREDIT_C_QUANTA_DEFAULT;
1791                 hw_credit_quanta = DLB2_SW_CREDIT_C_BATCH_SZ;
1792         }
1793         ev_port->credit_update_quanta = sw_credit_quanta;
1794         ev_port->qm_port.hw_credit_quanta = hw_credit_quanta;
1795
1796         /* Tear down pre-existing port->queue links */
1797         if (dlb2->run_state == DLB2_RUN_STATE_STOPPED)
1798                 dlb2_port_link_teardown(dlb2, &dlb2->ev_ports[ev_port_id]);
1799
1800         dev->data->ports[ev_port_id] = &dlb2->ev_ports[ev_port_id];
1801
1802         return 0;
1803 }
1804
1805 static int16_t
1806 dlb2_hw_map_ldb_qid_to_port(struct dlb2_hw_dev *handle,
1807                             uint32_t qm_port_id,
1808                             uint16_t qm_qid,
1809                             uint8_t priority)
1810 {
1811         struct dlb2_map_qid_args cfg;
1812         int32_t ret;
1813
1814         if (handle == NULL)
1815                 return -EINVAL;
1816
1817         /* Build message */
1818         cfg.port_id = qm_port_id;
1819         cfg.qid = qm_qid;
1820         cfg.priority = EV_TO_DLB2_PRIO(priority);
1821
1822         ret = dlb2_iface_map_qid(handle, &cfg);
1823         if (ret < 0) {
1824                 DLB2_LOG_ERR("dlb2: map qid error, ret=%d (driver status: %s)\n",
1825                              ret, dlb2_error_strings[cfg.response.status]);
1826                 DLB2_LOG_ERR("dlb2: grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
1827                              handle->domain_id, cfg.port_id,
1828                              cfg.qid,
1829                              cfg.priority);
1830         } else {
1831                 DLB2_LOG_DBG("dlb2: mapped queue %d to qm_port %d\n",
1832                              qm_qid, qm_port_id);
1833         }
1834
1835         return ret;
1836 }
1837
1838 static int
1839 dlb2_event_queue_join_ldb(struct dlb2_eventdev *dlb2,
1840                           struct dlb2_eventdev_port *ev_port,
1841                           struct dlb2_eventdev_queue *ev_queue,
1842                           uint8_t priority)
1843 {
1844         int first_avail = -1;
1845         int ret, i;
1846
1847         for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
1848                 if (ev_port->link[i].valid) {
1849                         if (ev_port->link[i].queue_id == ev_queue->id &&
1850                             ev_port->link[i].priority == priority) {
1851                                 if (ev_port->link[i].mapped)
1852                                         return 0; /* already mapped */
1853                                 first_avail = i;
1854                         }
1855                 } else if (first_avail == -1)
1856                         first_avail = i;
1857         }
1858         if (first_avail == -1) {
1859                 DLB2_LOG_ERR("dlb2: qm_port %d has no available QID slots.\n",
1860                              ev_port->qm_port.id);
1861                 return -EINVAL;
1862         }
1863
1864         ret = dlb2_hw_map_ldb_qid_to_port(&dlb2->qm_instance,
1865                                           ev_port->qm_port.id,
1866                                           ev_queue->qm_queue.id,
1867                                           priority);
1868
1869         if (!ret)
1870                 ev_port->link[first_avail].mapped = true;
1871
1872         return ret;
1873 }
1874
1875 static int32_t
1876 dlb2_hw_create_dir_queue(struct dlb2_eventdev *dlb2,
1877                          struct dlb2_eventdev_queue *ev_queue,
1878                          int32_t qm_port_id)
1879 {
1880         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
1881         struct dlb2_create_dir_queue_args cfg;
1882         int32_t ret;
1883
1884         /* The directed port is always configured before its queue */
1885         cfg.port_id = qm_port_id;
1886
1887         if (ev_queue->depth_threshold == 0) {
1888                 cfg.depth_threshold = dlb2->default_depth_thresh;
1889                 ev_queue->depth_threshold =
1890                         dlb2->default_depth_thresh;
1891         } else
1892                 cfg.depth_threshold = ev_queue->depth_threshold;
1893
1894         ret = dlb2_iface_dir_queue_create(handle, &cfg);
1895         if (ret < 0) {
1896                 DLB2_LOG_ERR("dlb2: create DIR event queue error, ret=%d (driver status: %s)\n",
1897                              ret, dlb2_error_strings[cfg.response.status]);
1898                 return -EINVAL;
1899         }
1900
1901         return cfg.response.id;
1902 }
1903
1904 static int
1905 dlb2_eventdev_dir_queue_setup(struct dlb2_eventdev *dlb2,
1906                               struct dlb2_eventdev_queue *ev_queue,
1907                               struct dlb2_eventdev_port *ev_port)
1908 {
1909         int32_t qm_qid;
1910
1911         qm_qid = dlb2_hw_create_dir_queue(dlb2, ev_queue, ev_port->qm_port.id);
1912
1913         if (qm_qid < 0) {
1914                 DLB2_LOG_ERR("Failed to create the DIR queue\n");
1915                 return qm_qid;
1916         }
1917
1918         dlb2->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
1919
1920         ev_queue->qm_queue.id = qm_qid;
1921
1922         return 0;
1923 }
1924
1925 static int
1926 dlb2_do_port_link(struct rte_eventdev *dev,
1927                   struct dlb2_eventdev_queue *ev_queue,
1928                   struct dlb2_eventdev_port *ev_port,
1929                   uint8_t prio)
1930 {
1931         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
1932         int err;
1933
1934         /* Don't link until start time. */
1935         if (dlb2->run_state == DLB2_RUN_STATE_STOPPED)
1936                 return 0;
1937
1938         if (ev_queue->qm_queue.is_directed)
1939                 err = dlb2_eventdev_dir_queue_setup(dlb2, ev_queue, ev_port);
1940         else
1941                 err = dlb2_event_queue_join_ldb(dlb2, ev_port, ev_queue, prio);
1942
1943         if (err) {
1944                 DLB2_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
1945                              ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
1946                              ev_queue->id, ev_port->id);
1947
1948                 rte_errno = err;
1949                 return -1;
1950         }
1951
1952         return 0;
1953 }
1954
1955 static int
1956 dlb2_validate_port_link(struct dlb2_eventdev_port *ev_port,
1957                         uint8_t queue_id,
1958                         bool link_exists,
1959                         int index)
1960 {
1961         struct dlb2_eventdev *dlb2 = ev_port->dlb2;
1962         struct dlb2_eventdev_queue *ev_queue;
1963         bool port_is_dir, queue_is_dir;
1964
1965         if (queue_id > dlb2->num_queues) {
1966                 rte_errno = -EINVAL;
1967                 return -1;
1968         }
1969
1970         ev_queue = &dlb2->ev_queues[queue_id];
1971
1972         if (!ev_queue->setup_done &&
1973             ev_queue->qm_queue.config_state != DLB2_PREV_CONFIGURED) {
1974                 rte_errno = -EINVAL;
1975                 return -1;
1976         }
1977
1978         port_is_dir = ev_port->qm_port.is_directed;
1979         queue_is_dir = ev_queue->qm_queue.is_directed;
1980
1981         if (port_is_dir != queue_is_dir) {
1982                 DLB2_LOG_ERR("%s queue %u can't link to %s port %u\n",
1983                              queue_is_dir ? "DIR" : "LDB", ev_queue->id,
1984                              port_is_dir ? "DIR" : "LDB", ev_port->id);
1985
1986                 rte_errno = -EINVAL;
1987                 return -1;
1988         }
1989
1990         /* Check if there is space for the requested link */
1991         if (!link_exists && index == -1) {
1992                 DLB2_LOG_ERR("no space for new link\n");
1993                 rte_errno = -ENOSPC;
1994                 return -1;
1995         }
1996
1997         /* Check if the directed port is already linked */
1998         if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
1999             !link_exists) {
2000                 DLB2_LOG_ERR("Can't link DIR port %d to >1 queues\n",
2001                              ev_port->id);
2002                 rte_errno = -EINVAL;
2003                 return -1;
2004         }
2005
2006         /* Check if the directed queue is already linked */
2007         if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
2008             !link_exists) {
2009                 DLB2_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
2010                              ev_queue->id);
2011                 rte_errno = -EINVAL;
2012                 return -1;
2013         }
2014
2015         return 0;
2016 }
2017
2018 static int
2019 dlb2_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
2020                         const uint8_t queues[], const uint8_t priorities[],
2021                         uint16_t nb_links)
2022
2023 {
2024         struct dlb2_eventdev_port *ev_port = event_port;
2025         struct dlb2_eventdev *dlb2;
2026         int i, j;
2027
2028         RTE_SET_USED(dev);
2029
2030         if (ev_port == NULL) {
2031                 DLB2_LOG_ERR("dlb2: evport not setup\n");
2032                 rte_errno = -EINVAL;
2033                 return 0;
2034         }
2035
2036         if (!ev_port->setup_done &&
2037             ev_port->qm_port.config_state != DLB2_PREV_CONFIGURED) {
2038                 DLB2_LOG_ERR("dlb2: evport not setup\n");
2039                 rte_errno = -EINVAL;
2040                 return 0;
2041         }
2042
2043         /* Note: rte_event_port_link() ensures the PMD won't receive a NULL
2044          * queues pointer.
2045          */
2046         if (nb_links == 0) {
2047                 DLB2_LOG_DBG("dlb2: nb_links is 0\n");
2048                 return 0; /* Ignore and return success */
2049         }
2050
2051         dlb2 = ev_port->dlb2;
2052
2053         DLB2_LOG_DBG("Linking %u queues to %s port %d\n",
2054                      nb_links,
2055                      ev_port->qm_port.is_directed ? "DIR" : "LDB",
2056                      ev_port->id);
2057
2058         for (i = 0; i < nb_links; i++) {
2059                 struct dlb2_eventdev_queue *ev_queue;
2060                 uint8_t queue_id, prio;
2061                 bool found = false;
2062                 int index = -1;
2063
2064                 queue_id = queues[i];
2065                 prio = priorities[i];
2066
2067                 /* Check if the link already exists. */
2068                 for (j = 0; j < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; j++)
2069                         if (ev_port->link[j].valid) {
2070                                 if (ev_port->link[j].queue_id == queue_id) {
2071                                         found = true;
2072                                         index = j;
2073                                         break;
2074                                 }
2075                         } else if (index == -1) {
2076                                 index = j;
2077                         }
2078
2079                 /* could not link */
2080                 if (index == -1)
2081                         break;
2082
2083                 /* Check if already linked at the requested priority */
2084                 if (found && ev_port->link[j].priority == prio)
2085                         continue;
2086
2087                 if (dlb2_validate_port_link(ev_port, queue_id, found, index))
2088                         break; /* return index of offending queue */
2089
2090                 ev_queue = &dlb2->ev_queues[queue_id];
2091
2092                 if (dlb2_do_port_link(dev, ev_queue, ev_port, prio))
2093                         break; /* return index of offending queue */
2094
2095                 ev_queue->num_links++;
2096
2097                 ev_port->link[index].queue_id = queue_id;
2098                 ev_port->link[index].priority = prio;
2099                 ev_port->link[index].valid = true;
2100                 /* Entry already exists?  If so, then must be prio change */
2101                 if (!found)
2102                         ev_port->num_links++;
2103         }
2104         return i;
2105 }
2106
2107 static int16_t
2108 dlb2_hw_unmap_ldb_qid_from_port(struct dlb2_hw_dev *handle,
2109                                 uint32_t qm_port_id,
2110                                 uint16_t qm_qid)
2111 {
2112         struct dlb2_unmap_qid_args cfg;
2113         int32_t ret;
2114
2115         if (handle == NULL)
2116                 return -EINVAL;
2117
2118         cfg.port_id = qm_port_id;
2119         cfg.qid = qm_qid;
2120
2121         ret = dlb2_iface_unmap_qid(handle, &cfg);
2122         if (ret < 0)
2123                 DLB2_LOG_ERR("dlb2: unmap qid error, ret=%d (driver status: %s)\n",
2124                              ret, dlb2_error_strings[cfg.response.status]);
2125
2126         return ret;
2127 }
2128
2129 static int
2130 dlb2_event_queue_detach_ldb(struct dlb2_eventdev *dlb2,
2131                             struct dlb2_eventdev_port *ev_port,
2132                             struct dlb2_eventdev_queue *ev_queue)
2133 {
2134         int ret, i;
2135
2136         /* Don't unlink until start time. */
2137         if (dlb2->run_state == DLB2_RUN_STATE_STOPPED)
2138                 return 0;
2139
2140         for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
2141                 if (ev_port->link[i].valid &&
2142                     ev_port->link[i].queue_id == ev_queue->id)
2143                         break; /* found */
2144         }
2145
2146         /* This is expected with eventdev API!
2147          * It blindly attemmpts to unmap all queues.
2148          */
2149         if (i == DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2150                 DLB2_LOG_DBG("dlb2: ignoring LB QID %d not mapped for qm_port %d.\n",
2151                              ev_queue->qm_queue.id,
2152                              ev_port->qm_port.id);
2153                 return 0;
2154         }
2155
2156         ret = dlb2_hw_unmap_ldb_qid_from_port(&dlb2->qm_instance,
2157                                               ev_port->qm_port.id,
2158                                               ev_queue->qm_queue.id);
2159         if (!ret)
2160                 ev_port->link[i].mapped = false;
2161
2162         return ret;
2163 }
2164
2165 static int
2166 dlb2_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
2167                           uint8_t queues[], uint16_t nb_unlinks)
2168 {
2169         struct dlb2_eventdev_port *ev_port = event_port;
2170         struct dlb2_eventdev *dlb2;
2171         int i;
2172
2173         RTE_SET_USED(dev);
2174
2175         if (!ev_port->setup_done) {
2176                 DLB2_LOG_ERR("dlb2: evport %d is not configured\n",
2177                              ev_port->id);
2178                 rte_errno = -EINVAL;
2179                 return 0;
2180         }
2181
2182         if (queues == NULL || nb_unlinks == 0) {
2183                 DLB2_LOG_DBG("dlb2: queues is NULL or nb_unlinks is 0\n");
2184                 return 0; /* Ignore and return success */
2185         }
2186
2187         if (ev_port->qm_port.is_directed) {
2188                 DLB2_LOG_DBG("dlb2: ignore unlink from dir port %d\n",
2189                              ev_port->id);
2190                 rte_errno = 0;
2191                 return nb_unlinks; /* as if success */
2192         }
2193
2194         dlb2 = ev_port->dlb2;
2195
2196         for (i = 0; i < nb_unlinks; i++) {
2197                 struct dlb2_eventdev_queue *ev_queue;
2198                 int ret, j;
2199
2200                 if (queues[i] >= dlb2->num_queues) {
2201                         DLB2_LOG_ERR("dlb2: invalid queue id %d\n", queues[i]);
2202                         rte_errno = -EINVAL;
2203                         return i; /* return index of offending queue */
2204                 }
2205
2206                 ev_queue = &dlb2->ev_queues[queues[i]];
2207
2208                 /* Does a link exist? */
2209                 for (j = 0; j < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; j++)
2210                         if (ev_port->link[j].queue_id == queues[i] &&
2211                             ev_port->link[j].valid)
2212                                 break;
2213
2214                 if (j == DLB2_MAX_NUM_QIDS_PER_LDB_CQ)
2215                         continue;
2216
2217                 ret = dlb2_event_queue_detach_ldb(dlb2, ev_port, ev_queue);
2218                 if (ret) {
2219                         DLB2_LOG_ERR("unlink err=%d for port %d queue %d\n",
2220                                      ret, ev_port->id, queues[i]);
2221                         rte_errno = -ENOENT;
2222                         return i; /* return index of offending queue */
2223                 }
2224
2225                 ev_port->link[j].valid = false;
2226                 ev_port->num_links--;
2227                 ev_queue->num_links--;
2228         }
2229
2230         return nb_unlinks;
2231 }
2232
2233 static int
2234 dlb2_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
2235                                        void *event_port)
2236 {
2237         struct dlb2_eventdev_port *ev_port = event_port;
2238         struct dlb2_eventdev *dlb2;
2239         struct dlb2_hw_dev *handle;
2240         struct dlb2_pending_port_unmaps_args cfg;
2241         int ret;
2242
2243         RTE_SET_USED(dev);
2244
2245         if (!ev_port->setup_done) {
2246                 DLB2_LOG_ERR("dlb2: evport %d is not configured\n",
2247                              ev_port->id);
2248                 rte_errno = -EINVAL;
2249                 return 0;
2250         }
2251
2252         cfg.port_id = ev_port->qm_port.id;
2253         dlb2 = ev_port->dlb2;
2254         handle = &dlb2->qm_instance;
2255         ret = dlb2_iface_pending_port_unmaps(handle, &cfg);
2256
2257         if (ret < 0) {
2258                 DLB2_LOG_ERR("dlb2: num_unlinks_in_progress ret=%d (driver status: %s)\n",
2259                              ret, dlb2_error_strings[cfg.response.status]);
2260                 return ret;
2261         }
2262
2263         return cfg.response.id;
2264 }
2265
2266 static int
2267 dlb2_eventdev_reapply_configuration(struct rte_eventdev *dev)
2268 {
2269         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
2270         int ret, i;
2271
2272         /* If an event queue or port was previously configured, but hasn't been
2273          * reconfigured, reapply its original configuration.
2274          */
2275         for (i = 0; i < dlb2->num_queues; i++) {
2276                 struct dlb2_eventdev_queue *ev_queue;
2277
2278                 ev_queue = &dlb2->ev_queues[i];
2279
2280                 if (ev_queue->qm_queue.config_state != DLB2_PREV_CONFIGURED)
2281                         continue;
2282
2283                 ret = dlb2_eventdev_queue_setup(dev, i, &ev_queue->conf);
2284                 if (ret < 0) {
2285                         DLB2_LOG_ERR("dlb2: failed to reconfigure queue %d", i);
2286                         return ret;
2287                 }
2288         }
2289
2290         for (i = 0; i < dlb2->num_ports; i++) {
2291                 struct dlb2_eventdev_port *ev_port = &dlb2->ev_ports[i];
2292
2293                 if (ev_port->qm_port.config_state != DLB2_PREV_CONFIGURED)
2294                         continue;
2295
2296                 ret = dlb2_eventdev_port_setup(dev, i, &ev_port->conf);
2297                 if (ret < 0) {
2298                         DLB2_LOG_ERR("dlb2: failed to reconfigure ev_port %d",
2299                                      i);
2300                         return ret;
2301                 }
2302         }
2303
2304         return 0;
2305 }
2306
2307 static int
2308 dlb2_eventdev_apply_port_links(struct rte_eventdev *dev)
2309 {
2310         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
2311         int i;
2312
2313         /* Perform requested port->queue links */
2314         for (i = 0; i < dlb2->num_ports; i++) {
2315                 struct dlb2_eventdev_port *ev_port = &dlb2->ev_ports[i];
2316                 int j;
2317
2318                 for (j = 0; j < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
2319                         struct dlb2_eventdev_queue *ev_queue;
2320                         uint8_t prio, queue_id;
2321
2322                         if (!ev_port->link[j].valid)
2323                                 continue;
2324
2325                         prio = ev_port->link[j].priority;
2326                         queue_id = ev_port->link[j].queue_id;
2327
2328                         if (dlb2_validate_port_link(ev_port, queue_id, true, j))
2329                                 return -EINVAL;
2330
2331                         ev_queue = &dlb2->ev_queues[queue_id];
2332
2333                         if (dlb2_do_port_link(dev, ev_queue, ev_port, prio))
2334                                 return -EINVAL;
2335                 }
2336         }
2337
2338         return 0;
2339 }
2340
2341 static int
2342 dlb2_eventdev_start(struct rte_eventdev *dev)
2343 {
2344         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
2345         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
2346         struct dlb2_start_domain_args cfg;
2347         int ret, i;
2348
2349         rte_spinlock_lock(&dlb2->qm_instance.resource_lock);
2350         if (dlb2->run_state != DLB2_RUN_STATE_STOPPED) {
2351                 DLB2_LOG_ERR("bad state %d for dev_start\n",
2352                              (int)dlb2->run_state);
2353                 rte_spinlock_unlock(&dlb2->qm_instance.resource_lock);
2354                 return -EINVAL;
2355         }
2356         dlb2->run_state = DLB2_RUN_STATE_STARTING;
2357         rte_spinlock_unlock(&dlb2->qm_instance.resource_lock);
2358
2359         /* If the device was configured more than once, some event ports and/or
2360          * queues may need to be reconfigured.
2361          */
2362         ret = dlb2_eventdev_reapply_configuration(dev);
2363         if (ret)
2364                 return ret;
2365
2366         /* The DLB PMD delays port links until the device is started. */
2367         ret = dlb2_eventdev_apply_port_links(dev);
2368         if (ret)
2369                 return ret;
2370
2371         for (i = 0; i < dlb2->num_ports; i++) {
2372                 if (!dlb2->ev_ports[i].setup_done) {
2373                         DLB2_LOG_ERR("dlb2: port %d not setup", i);
2374                         return -ESTALE;
2375                 }
2376         }
2377
2378         for (i = 0; i < dlb2->num_queues; i++) {
2379                 if (dlb2->ev_queues[i].num_links == 0) {
2380                         DLB2_LOG_ERR("dlb2: queue %d is not linked", i);
2381                         return -ENOLINK;
2382                 }
2383         }
2384
2385         ret = dlb2_iface_sched_domain_start(handle, &cfg);
2386         if (ret < 0) {
2387                 DLB2_LOG_ERR("dlb2: sched_domain_start ret=%d (driver status: %s)\n",
2388                              ret, dlb2_error_strings[cfg.response.status]);
2389                 return ret;
2390         }
2391
2392         dlb2->run_state = DLB2_RUN_STATE_STARTED;
2393         DLB2_LOG_DBG("dlb2: sched_domain_start completed OK\n");
2394
2395         return 0;
2396 }
2397
2398 static uint8_t cmd_byte_map[DLB2_NUM_PORT_TYPES][DLB2_NUM_HW_SCHED_TYPES] = {
2399         {
2400                 /* Load-balanced cmd bytes */
2401                 [RTE_EVENT_OP_NEW] = DLB2_NEW_CMD_BYTE,
2402                 [RTE_EVENT_OP_FORWARD] = DLB2_FWD_CMD_BYTE,
2403                 [RTE_EVENT_OP_RELEASE] = DLB2_COMP_CMD_BYTE,
2404         },
2405         {
2406                 /* Directed cmd bytes */
2407                 [RTE_EVENT_OP_NEW] = DLB2_NEW_CMD_BYTE,
2408                 [RTE_EVENT_OP_FORWARD] = DLB2_NEW_CMD_BYTE,
2409                 [RTE_EVENT_OP_RELEASE] = DLB2_NOOP_CMD_BYTE,
2410         },
2411 };
2412
2413 static inline uint32_t
2414 dlb2_port_credits_get(struct dlb2_port *qm_port,
2415                       enum dlb2_hw_queue_types type)
2416 {
2417         uint32_t credits = *qm_port->credit_pool[type];
2418         /* By default hw_credit_quanta is DLB2_SW_CREDIT_BATCH_SZ */
2419         uint32_t batch_size = qm_port->hw_credit_quanta;
2420
2421         if (unlikely(credits < batch_size))
2422                 batch_size = credits;
2423
2424         if (likely(credits &&
2425                    __atomic_compare_exchange_n(
2426                         qm_port->credit_pool[type],
2427                         &credits, credits - batch_size, false,
2428                         __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)))
2429                 return batch_size;
2430         else
2431                 return 0;
2432 }
2433
2434 static inline void
2435 dlb2_replenish_sw_credits(struct dlb2_eventdev *dlb2,
2436                           struct dlb2_eventdev_port *ev_port)
2437 {
2438         uint16_t quanta = ev_port->credit_update_quanta;
2439
2440         if (ev_port->inflight_credits >= quanta * 2) {
2441                 /* Replenish credits, saving one quanta for enqueues */
2442                 uint16_t val = ev_port->inflight_credits - quanta;
2443
2444                 __atomic_fetch_sub(&dlb2->inflights, val, __ATOMIC_SEQ_CST);
2445                 ev_port->inflight_credits -= val;
2446         }
2447 }
2448
2449 static inline int
2450 dlb2_check_enqueue_sw_credits(struct dlb2_eventdev *dlb2,
2451                               struct dlb2_eventdev_port *ev_port)
2452 {
2453         uint32_t sw_inflights = __atomic_load_n(&dlb2->inflights,
2454                                                 __ATOMIC_SEQ_CST);
2455         const int num = 1;
2456
2457         if (unlikely(ev_port->inflight_max < sw_inflights)) {
2458                 DLB2_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
2459                 rte_errno = -ENOSPC;
2460                 return 1;
2461         }
2462
2463         if (ev_port->inflight_credits < num) {
2464                 /* check if event enqueue brings ev_port over max threshold */
2465                 uint32_t credit_update_quanta = ev_port->credit_update_quanta;
2466
2467                 if (sw_inflights + credit_update_quanta >
2468                                 dlb2->new_event_limit) {
2469                         DLB2_INC_STAT(
2470                         ev_port->stats.traffic.tx_nospc_new_event_limit,
2471                         1);
2472                         rte_errno = -ENOSPC;
2473                         return 1;
2474                 }
2475
2476                 __atomic_fetch_add(&dlb2->inflights, credit_update_quanta,
2477                                    __ATOMIC_SEQ_CST);
2478                 ev_port->inflight_credits += (credit_update_quanta);
2479
2480                 if (ev_port->inflight_credits < num) {
2481                         DLB2_INC_STAT(
2482                         ev_port->stats.traffic.tx_nospc_inflight_credits,
2483                         1);
2484                         rte_errno = -ENOSPC;
2485                         return 1;
2486                 }
2487         }
2488
2489         return 0;
2490 }
2491
2492 static inline int
2493 dlb2_check_enqueue_hw_ldb_credits(struct dlb2_port *qm_port)
2494 {
2495         if (unlikely(qm_port->cached_ldb_credits == 0)) {
2496                 qm_port->cached_ldb_credits =
2497                         dlb2_port_credits_get(qm_port,
2498                                               DLB2_LDB_QUEUE);
2499                 if (unlikely(qm_port->cached_ldb_credits == 0)) {
2500                         DLB2_INC_STAT(
2501                         qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
2502                         1);
2503                         DLB2_LOG_DBG("ldb credits exhausted\n");
2504                         return 1; /* credits exhausted */
2505                 }
2506         }
2507
2508         return 0;
2509 }
2510
2511 static inline int
2512 dlb2_check_enqueue_hw_dir_credits(struct dlb2_port *qm_port)
2513 {
2514         if (unlikely(qm_port->cached_dir_credits == 0)) {
2515                 qm_port->cached_dir_credits =
2516                         dlb2_port_credits_get(qm_port,
2517                                               DLB2_DIR_QUEUE);
2518                 if (unlikely(qm_port->cached_dir_credits == 0)) {
2519                         DLB2_INC_STAT(
2520                         qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
2521                         1);
2522                         DLB2_LOG_DBG("dir credits exhausted\n");
2523                         return 1; /* credits exhausted */
2524                 }
2525         }
2526
2527         return 0;
2528 }
2529
2530 static inline int
2531 dlb2_check_enqueue_hw_credits(struct dlb2_port *qm_port)
2532 {
2533         if (unlikely(qm_port->cached_credits == 0)) {
2534                 qm_port->cached_credits =
2535                         dlb2_port_credits_get(qm_port,
2536                                               DLB2_COMBINED_POOL);
2537                 if (unlikely(qm_port->cached_credits == 0)) {
2538                         DLB2_INC_STAT(
2539                         qm_port->ev_port->stats.traffic.tx_nospc_hw_credits, 1);
2540                         DLB2_LOG_DBG("credits exhausted\n");
2541                         return 1; /* credits exhausted */
2542                 }
2543         }
2544
2545         return 0;
2546 }
2547
2548 static __rte_always_inline void
2549 dlb2_pp_write(struct dlb2_enqueue_qe *qe4,
2550               struct process_local_port_data *port_data)
2551 {
2552         dlb2_movdir64b(port_data->pp_addr, qe4);
2553 }
2554
2555 static inline int
2556 dlb2_consume_qe_immediate(struct dlb2_port *qm_port, int num)
2557 {
2558         struct process_local_port_data *port_data;
2559         struct dlb2_cq_pop_qe *qe;
2560
2561         RTE_ASSERT(qm_port->config_state == DLB2_CONFIGURED);
2562
2563         qe = qm_port->consume_qe;
2564
2565         qe->tokens = num - 1;
2566
2567         /* No store fence needed since no pointer is being sent, and CQ token
2568          * pops can be safely reordered with other HCWs.
2569          */
2570         port_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];
2571
2572         dlb2_movntdq_single(port_data->pp_addr, qe);
2573
2574         DLB2_LOG_DBG("dlb2: consume immediate - %d QEs\n", num);
2575
2576         qm_port->owed_tokens = 0;
2577
2578         return 0;
2579 }
2580
2581 static inline void
2582 dlb2_hw_do_enqueue(struct dlb2_port *qm_port,
2583                    bool do_sfence,
2584                    struct process_local_port_data *port_data)
2585 {
2586         /* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
2587          * application writes complete before enqueueing the QE.
2588          */
2589         if (do_sfence)
2590                 rte_wmb();
2591
2592         dlb2_pp_write(qm_port->qe4, port_data);
2593 }
2594
2595 static inline void
2596 dlb2_construct_token_pop_qe(struct dlb2_port *qm_port, int idx)
2597 {
2598         struct dlb2_cq_pop_qe *qe = (void *)qm_port->qe4;
2599         int num = qm_port->owed_tokens;
2600
2601         qe[idx].cmd_byte = DLB2_POP_CMD_BYTE;
2602         qe[idx].tokens = num - 1;
2603
2604         qm_port->owed_tokens = 0;
2605 }
2606
2607 static inline void
2608 dlb2_event_build_hcws(struct dlb2_port *qm_port,
2609                       const struct rte_event ev[],
2610                       int num,
2611                       uint8_t *sched_type,
2612                       uint8_t *queue_id)
2613 {
2614         struct dlb2_enqueue_qe *qe;
2615         uint16_t sched_word[4];
2616         __m128i sse_qe[2];
2617         int i;
2618
2619         qe = qm_port->qe4;
2620
2621         sse_qe[0] = _mm_setzero_si128();
2622         sse_qe[1] = _mm_setzero_si128();
2623
2624         switch (num) {
2625         case 4:
2626                 /* Construct the metadata portion of two HCWs in one 128b SSE
2627                  * register. HCW metadata is constructed in the SSE registers
2628                  * like so:
2629                  * sse_qe[0][63:0]:   qe[0]'s metadata
2630                  * sse_qe[0][127:64]: qe[1]'s metadata
2631                  * sse_qe[1][63:0]:   qe[2]'s metadata
2632                  * sse_qe[1][127:64]: qe[3]'s metadata
2633                  */
2634
2635                 /* Convert the event operation into a command byte and store it
2636                  * in the metadata:
2637                  * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
2638                  * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
2639                  * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
2640                  * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
2641                  */
2642 #define DLB2_QE_CMD_BYTE 7
2643                 sse_qe[0] = _mm_insert_epi8(sse_qe[0],
2644                                 cmd_byte_map[qm_port->is_directed][ev[0].op],
2645                                 DLB2_QE_CMD_BYTE);
2646                 sse_qe[0] = _mm_insert_epi8(sse_qe[0],
2647                                 cmd_byte_map[qm_port->is_directed][ev[1].op],
2648                                 DLB2_QE_CMD_BYTE + 8);
2649                 sse_qe[1] = _mm_insert_epi8(sse_qe[1],
2650                                 cmd_byte_map[qm_port->is_directed][ev[2].op],
2651                                 DLB2_QE_CMD_BYTE);
2652                 sse_qe[1] = _mm_insert_epi8(sse_qe[1],
2653                                 cmd_byte_map[qm_port->is_directed][ev[3].op],
2654                                 DLB2_QE_CMD_BYTE + 8);
2655
2656                 /* Store priority, scheduling type, and queue ID in the sched
2657                  * word array because these values are re-used when the
2658                  * destination is a directed queue.
2659                  */
2660                 sched_word[0] = EV_TO_DLB2_PRIO(ev[0].priority) << 10 |
2661                                 sched_type[0] << 8 |
2662                                 queue_id[0];
2663                 sched_word[1] = EV_TO_DLB2_PRIO(ev[1].priority) << 10 |
2664                                 sched_type[1] << 8 |
2665                                 queue_id[1];
2666                 sched_word[2] = EV_TO_DLB2_PRIO(ev[2].priority) << 10 |
2667                                 sched_type[2] << 8 |
2668                                 queue_id[2];
2669                 sched_word[3] = EV_TO_DLB2_PRIO(ev[3].priority) << 10 |
2670                                 sched_type[3] << 8 |
2671                                 queue_id[3];
2672
2673                 /* Store the event priority, scheduling type, and queue ID in
2674                  * the metadata:
2675                  * sse_qe[0][31:16] = sched_word[0]
2676                  * sse_qe[0][95:80] = sched_word[1]
2677                  * sse_qe[1][31:16] = sched_word[2]
2678                  * sse_qe[1][95:80] = sched_word[3]
2679                  */
2680 #define DLB2_QE_QID_SCHED_WORD 1
2681                 sse_qe[0] = _mm_insert_epi16(sse_qe[0],
2682                                              sched_word[0],
2683                                              DLB2_QE_QID_SCHED_WORD);
2684                 sse_qe[0] = _mm_insert_epi16(sse_qe[0],
2685                                              sched_word[1],
2686                                              DLB2_QE_QID_SCHED_WORD + 4);
2687                 sse_qe[1] = _mm_insert_epi16(sse_qe[1],
2688                                              sched_word[2],
2689                                              DLB2_QE_QID_SCHED_WORD);
2690                 sse_qe[1] = _mm_insert_epi16(sse_qe[1],
2691                                              sched_word[3],
2692                                              DLB2_QE_QID_SCHED_WORD + 4);
2693
2694                 /* If the destination is a load-balanced queue, store the lock
2695                  * ID. If it is a directed queue, DLB places this field in
2696                  * bytes 10-11 of the received QE, so we format it accordingly:
2697                  * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
2698                  * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
2699                  * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
2700                  * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
2701                  */
2702 #define DLB2_QE_LOCK_ID_WORD 2
2703                 sse_qe[0] = _mm_insert_epi16(sse_qe[0],
2704                                 (sched_type[0] == DLB2_SCHED_DIRECTED) ?
2705                                         sched_word[0] : ev[0].flow_id,
2706                                 DLB2_QE_LOCK_ID_WORD);
2707                 sse_qe[0] = _mm_insert_epi16(sse_qe[0],
2708                                 (sched_type[1] == DLB2_SCHED_DIRECTED) ?
2709                                         sched_word[1] : ev[1].flow_id,
2710                                 DLB2_QE_LOCK_ID_WORD + 4);
2711                 sse_qe[1] = _mm_insert_epi16(sse_qe[1],
2712                                 (sched_type[2] == DLB2_SCHED_DIRECTED) ?
2713                                         sched_word[2] : ev[2].flow_id,
2714                                 DLB2_QE_LOCK_ID_WORD);
2715                 sse_qe[1] = _mm_insert_epi16(sse_qe[1],
2716                                 (sched_type[3] == DLB2_SCHED_DIRECTED) ?
2717                                         sched_word[3] : ev[3].flow_id,
2718                                 DLB2_QE_LOCK_ID_WORD + 4);
2719
2720                 /* Store the event type and sub event type in the metadata:
2721                  * sse_qe[0][15:0]  = flow_id[0]
2722                  * sse_qe[0][79:64] = flow_id[1]
2723                  * sse_qe[1][15:0]  = flow_id[2]
2724                  * sse_qe[1][79:64] = flow_id[3]
2725                  */
2726 #define DLB2_QE_EV_TYPE_WORD 0
2727                 sse_qe[0] = _mm_insert_epi16(sse_qe[0],
2728                                              ev[0].sub_event_type << 8 |
2729                                                 ev[0].event_type,
2730                                              DLB2_QE_EV_TYPE_WORD);
2731                 sse_qe[0] = _mm_insert_epi16(sse_qe[0],
2732                                              ev[1].sub_event_type << 8 |
2733                                                 ev[1].event_type,
2734                                              DLB2_QE_EV_TYPE_WORD + 4);
2735                 sse_qe[1] = _mm_insert_epi16(sse_qe[1],
2736                                              ev[2].sub_event_type << 8 |
2737                                                 ev[2].event_type,
2738                                              DLB2_QE_EV_TYPE_WORD);
2739                 sse_qe[1] = _mm_insert_epi16(sse_qe[1],
2740                                              ev[3].sub_event_type << 8 |
2741                                                 ev[3].event_type,
2742                                              DLB2_QE_EV_TYPE_WORD + 4);
2743
2744                 /* Store the metadata to memory (use the double-precision
2745                  * _mm_storeh_pd because there is no integer function for
2746                  * storing the upper 64b):
2747                  * qe[0] metadata = sse_qe[0][63:0]
2748                  * qe[1] metadata = sse_qe[0][127:64]
2749                  * qe[2] metadata = sse_qe[1][63:0]
2750                  * qe[3] metadata = sse_qe[1][127:64]
2751                  */
2752                 _mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
2753                 _mm_storeh_pd((double *)&qe[1].u.opaque_data,
2754                               (__m128d)sse_qe[0]);
2755                 _mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
2756                 _mm_storeh_pd((double *)&qe[3].u.opaque_data,
2757                               (__m128d)sse_qe[1]);
2758
2759                 qe[0].data = ev[0].u64;
2760                 qe[1].data = ev[1].u64;
2761                 qe[2].data = ev[2].u64;
2762                 qe[3].data = ev[3].u64;
2763
2764                 break;
2765         case 3:
2766         case 2:
2767         case 1:
2768                 for (i = 0; i < num; i++) {
2769                         qe[i].cmd_byte =
2770                                 cmd_byte_map[qm_port->is_directed][ev[i].op];
2771                         qe[i].sched_type = sched_type[i];
2772                         qe[i].data = ev[i].u64;
2773                         qe[i].qid = queue_id[i];
2774                         qe[i].priority = EV_TO_DLB2_PRIO(ev[i].priority);
2775                         qe[i].lock_id = ev[i].flow_id;
2776                         if (sched_type[i] == DLB2_SCHED_DIRECTED) {
2777                                 struct dlb2_msg_info *info =
2778                                         (struct dlb2_msg_info *)&qe[i].lock_id;
2779
2780                                 info->qid = queue_id[i];
2781                                 info->sched_type = DLB2_SCHED_DIRECTED;
2782                                 info->priority = qe[i].priority;
2783                         }
2784                         qe[i].u.event_type.major = ev[i].event_type;
2785                         qe[i].u.event_type.sub = ev[i].sub_event_type;
2786                 }
2787                 break;
2788         case 0:
2789                 break;
2790         }
2791 }
2792
2793 static inline int
2794 dlb2_event_enqueue_prep(struct dlb2_eventdev_port *ev_port,
2795                         struct dlb2_port *qm_port,
2796                         const struct rte_event ev[],
2797                         uint8_t *sched_type,
2798                         uint8_t *queue_id)
2799 {
2800         struct dlb2_eventdev *dlb2 = ev_port->dlb2;
2801         struct dlb2_eventdev_queue *ev_queue;
2802         uint16_t *cached_credits = NULL;
2803         struct dlb2_queue *qm_queue;
2804
2805         ev_queue = &dlb2->ev_queues[ev->queue_id];
2806         qm_queue = &ev_queue->qm_queue;
2807         *queue_id = qm_queue->id;
2808
2809         /* Ignore sched_type and hardware credits on release events */
2810         if (ev->op == RTE_EVENT_OP_RELEASE)
2811                 goto op_check;
2812
2813         if (!qm_queue->is_directed) {
2814                 /* Load balanced destination queue */
2815
2816                 if (dlb2->version == DLB2_HW_V2) {
2817                         if (dlb2_check_enqueue_hw_ldb_credits(qm_port)) {
2818                                 rte_errno = -ENOSPC;
2819                                 return 1;
2820                         }
2821                         cached_credits = &qm_port->cached_ldb_credits;
2822                 } else {
2823                         if (dlb2_check_enqueue_hw_credits(qm_port)) {
2824                                 rte_errno = -ENOSPC;
2825                                 return 1;
2826                         }
2827                         cached_credits = &qm_port->cached_credits;
2828                 }
2829                 switch (ev->sched_type) {
2830                 case RTE_SCHED_TYPE_ORDERED:
2831                         DLB2_LOG_DBG("dlb2: put_qe: RTE_SCHED_TYPE_ORDERED\n");
2832                         if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
2833                                 DLB2_LOG_ERR("dlb2: tried to send ordered event to unordered queue %d\n",
2834                                              *queue_id);
2835                                 rte_errno = -EINVAL;
2836                                 return 1;
2837                         }
2838                         *sched_type = DLB2_SCHED_ORDERED;
2839                         break;
2840                 case RTE_SCHED_TYPE_ATOMIC:
2841                         DLB2_LOG_DBG("dlb2: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
2842                         *sched_type = DLB2_SCHED_ATOMIC;
2843                         break;
2844                 case RTE_SCHED_TYPE_PARALLEL:
2845                         DLB2_LOG_DBG("dlb2: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
2846                         if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
2847                                 *sched_type = DLB2_SCHED_ORDERED;
2848                         else
2849                                 *sched_type = DLB2_SCHED_UNORDERED;
2850                         break;
2851                 default:
2852                         DLB2_LOG_ERR("Unsupported LDB sched type in put_qe\n");
2853                         DLB2_INC_STAT(ev_port->stats.tx_invalid, 1);
2854                         rte_errno = -EINVAL;
2855                         return 1;
2856                 }
2857         } else {
2858                 /* Directed destination queue */
2859
2860                 if (dlb2->version == DLB2_HW_V2) {
2861                         if (dlb2_check_enqueue_hw_dir_credits(qm_port)) {
2862                                 rte_errno = -ENOSPC;
2863                                 return 1;
2864                         }
2865                         cached_credits = &qm_port->cached_dir_credits;
2866                 } else {
2867                         if (dlb2_check_enqueue_hw_credits(qm_port)) {
2868                                 rte_errno = -ENOSPC;
2869                                 return 1;
2870                         }
2871                         cached_credits = &qm_port->cached_credits;
2872                 }
2873                 DLB2_LOG_DBG("dlb2: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
2874
2875                 *sched_type = DLB2_SCHED_DIRECTED;
2876         }
2877
2878 op_check:
2879         switch (ev->op) {
2880         case RTE_EVENT_OP_NEW:
2881                 /* Check that a sw credit is available */
2882                 if (dlb2_check_enqueue_sw_credits(dlb2, ev_port)) {
2883                         rte_errno = -ENOSPC;
2884                         return 1;
2885                 }
2886                 ev_port->inflight_credits--;
2887                 (*cached_credits)--;
2888                 break;
2889         case RTE_EVENT_OP_FORWARD:
2890                 /* Check for outstanding_releases underflow. If this occurs,
2891                  * the application is not using the EVENT_OPs correctly; for
2892                  * example, forwarding or releasing events that were not
2893                  * dequeued.
2894                  */
2895                 RTE_ASSERT(ev_port->outstanding_releases > 0);
2896                 ev_port->outstanding_releases--;
2897                 qm_port->issued_releases++;
2898                 (*cached_credits)--;
2899                 break;
2900         case RTE_EVENT_OP_RELEASE:
2901                 ev_port->inflight_credits++;
2902                 /* Check for outstanding_releases underflow. If this occurs,
2903                  * the application is not using the EVENT_OPs correctly; for
2904                  * example, forwarding or releasing events that were not
2905                  * dequeued.
2906                  */
2907                 RTE_ASSERT(ev_port->outstanding_releases > 0);
2908                 ev_port->outstanding_releases--;
2909                 qm_port->issued_releases++;
2910
2911                 /* Replenish s/w credits if enough are cached */
2912                 dlb2_replenish_sw_credits(dlb2, ev_port);
2913                 break;
2914         }
2915
2916         DLB2_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
2917         DLB2_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
2918
2919 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
2920         if (ev->op != RTE_EVENT_OP_RELEASE) {
2921                 DLB2_INC_STAT(ev_port->stats.queue[ev->queue_id].enq_ok, 1);
2922                 DLB2_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
2923         }
2924 #endif
2925
2926         return 0;
2927 }
2928
2929 static inline uint16_t
2930 __dlb2_event_enqueue_burst(void *event_port,
2931                            const struct rte_event events[],
2932                            uint16_t num,
2933                            bool use_delayed)
2934 {
2935         struct dlb2_eventdev_port *ev_port = event_port;
2936         struct dlb2_port *qm_port = &ev_port->qm_port;
2937         struct process_local_port_data *port_data;
2938         int i;
2939
2940         RTE_ASSERT(ev_port->enq_configured);
2941         RTE_ASSERT(events != NULL);
2942
2943         i = 0;
2944
2945         port_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];
2946
2947         while (i < num) {
2948                 uint8_t sched_types[DLB2_NUM_QES_PER_CACHE_LINE];
2949                 uint8_t queue_ids[DLB2_NUM_QES_PER_CACHE_LINE];
2950                 int pop_offs = 0;
2951                 int j = 0;
2952
2953                 memset(qm_port->qe4,
2954                        0,
2955                        DLB2_NUM_QES_PER_CACHE_LINE *
2956                        sizeof(struct dlb2_enqueue_qe));
2957
2958                 for (; j < DLB2_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
2959                         const struct rte_event *ev = &events[i + j];
2960                         int16_t thresh = qm_port->token_pop_thresh;
2961
2962                         if (use_delayed &&
2963                             qm_port->token_pop_mode == DELAYED_POP &&
2964                             (ev->op == RTE_EVENT_OP_FORWARD ||
2965                              ev->op == RTE_EVENT_OP_RELEASE) &&
2966                             qm_port->issued_releases >= thresh - 1) {
2967                                 /* Insert the token pop QE and break out. This
2968                                  * may result in a partial HCW, but that is
2969                                  * simpler than supporting arbitrary QE
2970                                  * insertion.
2971                                  */
2972                                 dlb2_construct_token_pop_qe(qm_port, j);
2973
2974                                 /* Reset the releases for the next QE batch */
2975                                 qm_port->issued_releases -= thresh;
2976
2977                                 pop_offs = 1;
2978                                 j++;
2979                                 break;
2980                         }
2981
2982                         if (dlb2_event_enqueue_prep(ev_port, qm_port, ev,
2983                                                     &sched_types[j],
2984                                                     &queue_ids[j]))
2985                                 break;
2986                 }
2987
2988                 if (j == 0)
2989                         break;
2990
2991                 dlb2_event_build_hcws(qm_port, &events[i], j - pop_offs,
2992                                       sched_types, queue_ids);
2993
2994                 dlb2_hw_do_enqueue(qm_port, i == 0, port_data);
2995
2996                 /* Don't include the token pop QE in the enqueue count */
2997                 i += j - pop_offs;
2998
2999                 /* Don't interpret j < DLB2_NUM_... as out-of-credits if
3000                  * pop_offs != 0
3001                  */
3002                 if (j < DLB2_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
3003                         break;
3004         }
3005
3006         return i;
3007 }
3008
3009 static uint16_t
3010 dlb2_event_enqueue_burst(void *event_port,
3011                              const struct rte_event events[],
3012                              uint16_t num)
3013 {
3014         return __dlb2_event_enqueue_burst(event_port, events, num, false);
3015 }
3016
3017 static uint16_t
3018 dlb2_event_enqueue_burst_delayed(void *event_port,
3019                                      const struct rte_event events[],
3020                                      uint16_t num)
3021 {
3022         return __dlb2_event_enqueue_burst(event_port, events, num, true);
3023 }
3024
3025 static inline uint16_t
3026 dlb2_event_enqueue(void *event_port,
3027                    const struct rte_event events[])
3028 {
3029         return __dlb2_event_enqueue_burst(event_port, events, 1, false);
3030 }
3031
3032 static inline uint16_t
3033 dlb2_event_enqueue_delayed(void *event_port,
3034                            const struct rte_event events[])
3035 {
3036         return __dlb2_event_enqueue_burst(event_port, events, 1, true);
3037 }
3038
3039 static uint16_t
3040 dlb2_event_enqueue_new_burst(void *event_port,
3041                              const struct rte_event events[],
3042                              uint16_t num)
3043 {
3044         return __dlb2_event_enqueue_burst(event_port, events, num, false);
3045 }
3046
3047 static uint16_t
3048 dlb2_event_enqueue_new_burst_delayed(void *event_port,
3049                                      const struct rte_event events[],
3050                                      uint16_t num)
3051 {
3052         return __dlb2_event_enqueue_burst(event_port, events, num, true);
3053 }
3054
3055 static uint16_t
3056 dlb2_event_enqueue_forward_burst(void *event_port,
3057                                  const struct rte_event events[],
3058                                  uint16_t num)
3059 {
3060         return __dlb2_event_enqueue_burst(event_port, events, num, false);
3061 }
3062
3063 static uint16_t
3064 dlb2_event_enqueue_forward_burst_delayed(void *event_port,
3065                                          const struct rte_event events[],
3066                                          uint16_t num)
3067 {
3068         return __dlb2_event_enqueue_burst(event_port, events, num, true);
3069 }
3070
3071 static void
3072 dlb2_event_release(struct dlb2_eventdev *dlb2,
3073                    uint8_t port_id,
3074                    int n)
3075 {
3076         struct process_local_port_data *port_data;
3077         struct dlb2_eventdev_port *ev_port;
3078         struct dlb2_port *qm_port;
3079         int i;
3080
3081         if (port_id > dlb2->num_ports) {
3082                 DLB2_LOG_ERR("Invalid port id %d in dlb2-event_release\n",
3083                              port_id);
3084                 rte_errno = -EINVAL;
3085                 return;
3086         }
3087
3088         ev_port = &dlb2->ev_ports[port_id];
3089         qm_port = &ev_port->qm_port;
3090         port_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];
3091
3092         i = 0;
3093
3094         if (qm_port->is_directed) {
3095                 i = n;
3096                 goto sw_credit_update;
3097         }
3098
3099         while (i < n) {
3100                 int pop_offs = 0;
3101                 int j = 0;
3102
3103                 /* Zero-out QEs */
3104                 _mm_storeu_si128((void *)&qm_port->qe4[0], _mm_setzero_si128());
3105                 _mm_storeu_si128((void *)&qm_port->qe4[1], _mm_setzero_si128());
3106                 _mm_storeu_si128((void *)&qm_port->qe4[2], _mm_setzero_si128());
3107                 _mm_storeu_si128((void *)&qm_port->qe4[3], _mm_setzero_si128());
3108
3109
3110                 for (; j < DLB2_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
3111                         int16_t thresh = qm_port->token_pop_thresh;
3112
3113                         if (qm_port->token_pop_mode == DELAYED_POP &&
3114                             qm_port->issued_releases >= thresh - 1) {
3115                                 /* Insert the token pop QE */
3116                                 dlb2_construct_token_pop_qe(qm_port, j);
3117
3118                                 /* Reset the releases for the next QE batch */
3119                                 qm_port->issued_releases -= thresh;
3120
3121                                 pop_offs = 1;
3122                                 j++;
3123                                 break;
3124                         }
3125
3126                         qm_port->qe4[j].cmd_byte = DLB2_COMP_CMD_BYTE;
3127                         qm_port->issued_releases++;
3128                 }
3129
3130                 dlb2_hw_do_enqueue(qm_port, i == 0, port_data);
3131
3132                 /* Don't include the token pop QE in the release count */
3133                 i += j - pop_offs;
3134         }
3135
3136 sw_credit_update:
3137         /* each release returns one credit */
3138         if (unlikely(!ev_port->outstanding_releases)) {
3139                 DLB2_LOG_ERR("%s: Outstanding releases underflowed.\n",
3140                              __func__);
3141                 return;
3142         }
3143         ev_port->outstanding_releases -= i;
3144         ev_port->inflight_credits += i;
3145
3146         /* Replenish s/w credits if enough releases are performed */
3147         dlb2_replenish_sw_credits(dlb2, ev_port);
3148 }
3149
3150 static inline void
3151 dlb2_port_credits_inc(struct dlb2_port *qm_port, int num)
3152 {
3153         uint32_t batch_size = qm_port->hw_credit_quanta;
3154
3155         /* increment port credits, and return to pool if exceeds threshold */
3156         if (!qm_port->is_directed) {
3157                 if (qm_port->dlb2->version == DLB2_HW_V2) {
3158                         qm_port->cached_ldb_credits += num;
3159                         if (qm_port->cached_ldb_credits >= 2 * batch_size) {
3160                                 __atomic_fetch_add(
3161                                         qm_port->credit_pool[DLB2_LDB_QUEUE],
3162                                         batch_size, __ATOMIC_SEQ_CST);
3163                                 qm_port->cached_ldb_credits -= batch_size;
3164                         }
3165                 } else {
3166                         qm_port->cached_credits += num;
3167                         if (qm_port->cached_credits >= 2 * batch_size) {
3168                                 __atomic_fetch_add(
3169                                       qm_port->credit_pool[DLB2_COMBINED_POOL],
3170                                       batch_size, __ATOMIC_SEQ_CST);
3171                                 qm_port->cached_credits -= batch_size;
3172                         }
3173                 }
3174         } else {
3175                 if (qm_port->dlb2->version == DLB2_HW_V2) {
3176                         qm_port->cached_dir_credits += num;
3177                         if (qm_port->cached_dir_credits >= 2 * batch_size) {
3178                                 __atomic_fetch_add(
3179                                         qm_port->credit_pool[DLB2_DIR_QUEUE],
3180                                         batch_size, __ATOMIC_SEQ_CST);
3181                                 qm_port->cached_dir_credits -= batch_size;
3182                         }
3183                 } else {
3184                         qm_port->cached_credits += num;
3185                         if (qm_port->cached_credits >= 2 * batch_size) {
3186                                 __atomic_fetch_add(
3187                                       qm_port->credit_pool[DLB2_COMBINED_POOL],
3188                                       batch_size, __ATOMIC_SEQ_CST);
3189                                 qm_port->cached_credits -= batch_size;
3190                         }
3191                 }
3192         }
3193 }
3194
3195 #define CLB_MASK_IDX 0
3196 #define CLB_VAL_IDX 1
3197 static int
3198 dlb2_monitor_callback(const uint64_t val,
3199                 const uint64_t opaque[RTE_POWER_MONITOR_OPAQUE_SZ])
3200 {
3201         /* abort if the value matches */
3202         return (val & opaque[CLB_MASK_IDX]) == opaque[CLB_VAL_IDX] ? -1 : 0;
3203 }
3204
3205 static inline int
3206 dlb2_dequeue_wait(struct dlb2_eventdev *dlb2,
3207                   struct dlb2_eventdev_port *ev_port,
3208                   struct dlb2_port *qm_port,
3209                   uint64_t timeout,
3210                   uint64_t start_ticks)
3211 {
3212         struct process_local_port_data *port_data;
3213         uint64_t elapsed_ticks;
3214
3215         port_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];
3216
3217         elapsed_ticks = rte_get_timer_cycles() - start_ticks;
3218
3219         /* Wait/poll time expired */
3220         if (elapsed_ticks >= timeout) {
3221                 return 1;
3222         } else if (dlb2->umwait_allowed) {
3223                 struct rte_power_monitor_cond pmc;
3224                 volatile struct dlb2_dequeue_qe *cq_base;
3225                 union {
3226                         uint64_t raw_qe[2];
3227                         struct dlb2_dequeue_qe qe;
3228                 } qe_mask;
3229                 uint64_t expected_value;
3230                 volatile uint64_t *monitor_addr;
3231
3232                 qe_mask.qe.cq_gen = 1; /* set mask */
3233
3234                 cq_base = port_data->cq_base;
3235                 monitor_addr = (volatile uint64_t *)(volatile void *)
3236                         &cq_base[qm_port->cq_idx];
3237                 monitor_addr++; /* cq_gen bit is in second 64bit location */
3238
3239                 if (qm_port->gen_bit)
3240                         expected_value = qe_mask.raw_qe[1];
3241                 else
3242                         expected_value = 0;
3243
3244                 pmc.addr = monitor_addr;
3245                 /* store expected value and comparison mask in opaque data */
3246                 pmc.opaque[CLB_VAL_IDX] = expected_value;
3247                 pmc.opaque[CLB_MASK_IDX] = qe_mask.raw_qe[1];
3248                 /* set up callback */
3249                 pmc.fn = dlb2_monitor_callback;
3250                 pmc.size = sizeof(uint64_t);
3251
3252                 rte_power_monitor(&pmc, timeout + start_ticks);
3253
3254                 DLB2_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
3255         } else {
3256                 uint64_t poll_interval = dlb2->poll_interval;
3257                 uint64_t curr_ticks = rte_get_timer_cycles();
3258                 uint64_t init_ticks = curr_ticks;
3259
3260                 while ((curr_ticks - start_ticks < timeout) &&
3261                        (curr_ticks - init_ticks < poll_interval))
3262                         curr_ticks = rte_get_timer_cycles();
3263         }
3264
3265         return 0;
3266 }
3267
3268 static __rte_noinline int
3269 dlb2_process_dequeue_qes(struct dlb2_eventdev_port *ev_port,
3270                          struct dlb2_port *qm_port,
3271                          struct rte_event *events,
3272                          struct dlb2_dequeue_qe *qes,
3273                          int cnt)
3274 {
3275         uint8_t *qid_mappings = qm_port->qid_mappings;
3276         int i, num, evq_id;
3277
3278         for (i = 0, num = 0; i < cnt; i++) {
3279                 struct dlb2_dequeue_qe *qe = &qes[i];
3280                 int sched_type_map[DLB2_NUM_HW_SCHED_TYPES] = {
3281                         [DLB2_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
3282                         [DLB2_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
3283                         [DLB2_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
3284                         [DLB2_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
3285                 };
3286
3287                 /* Fill in event information.
3288                  * Note that flow_id must be embedded in the data by
3289                  * the app, such as the mbuf RSS hash field if the data
3290                  * buffer is a mbuf.
3291                  */
3292                 if (unlikely(qe->error)) {
3293                         DLB2_LOG_ERR("QE error bit ON\n");
3294                         DLB2_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
3295                         dlb2_consume_qe_immediate(qm_port, 1);
3296                         continue; /* Ignore */
3297                 }
3298
3299                 events[num].u64 = qe->data;
3300                 events[num].flow_id = qe->flow_id;
3301                 events[num].priority = DLB2_TO_EV_PRIO((uint8_t)qe->priority);
3302                 events[num].event_type = qe->u.event_type.major;
3303                 events[num].sub_event_type = qe->u.event_type.sub;
3304                 events[num].sched_type = sched_type_map[qe->sched_type];
3305                 events[num].impl_opaque = qe->qid_depth;
3306
3307                 /* qid not preserved for directed queues */
3308                 if (qm_port->is_directed)
3309                         evq_id = ev_port->link[0].queue_id;
3310                 else
3311                         evq_id = qid_mappings[qe->qid];
3312
3313                 events[num].queue_id = evq_id;
3314                 DLB2_INC_STAT(
3315                         ev_port->stats.queue[evq_id].qid_depth[qe->qid_depth],
3316                         1);
3317                 DLB2_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
3318                 num++;
3319         }
3320
3321         DLB2_INC_STAT(ev_port->stats.traffic.rx_ok, num);
3322
3323         return num;
3324 }
3325
3326 static inline int
3327 dlb2_process_dequeue_four_qes(struct dlb2_eventdev_port *ev_port,
3328                               struct dlb2_port *qm_port,
3329                               struct rte_event *events,
3330                               struct dlb2_dequeue_qe *qes)
3331 {
3332         int sched_type_map[] = {
3333                 [DLB2_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
3334                 [DLB2_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
3335                 [DLB2_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
3336                 [DLB2_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
3337         };
3338         const int num_events = DLB2_NUM_QES_PER_CACHE_LINE;
3339         uint8_t *qid_mappings = qm_port->qid_mappings;
3340         __m128i sse_evt[2];
3341
3342         /* In the unlikely case that any of the QE error bits are set, process
3343          * them one at a time.
3344          */
3345         if (unlikely(qes[0].error || qes[1].error ||
3346                      qes[2].error || qes[3].error))
3347                 return dlb2_process_dequeue_qes(ev_port, qm_port, events,
3348                                                  qes, num_events);
3349
3350         events[0].u64 = qes[0].data;
3351         events[1].u64 = qes[1].data;
3352         events[2].u64 = qes[2].data;
3353         events[3].u64 = qes[3].data;
3354
3355         /* Construct the metadata portion of two struct rte_events
3356          * in one 128b SSE register. Event metadata is constructed in the SSE
3357          * registers like so:
3358          * sse_evt[0][63:0]:   event[0]'s metadata
3359          * sse_evt[0][127:64]: event[1]'s metadata
3360          * sse_evt[1][63:0]:   event[2]'s metadata
3361          * sse_evt[1][127:64]: event[3]'s metadata
3362          */
3363         sse_evt[0] = _mm_setzero_si128();
3364         sse_evt[1] = _mm_setzero_si128();
3365
3366         /* Convert the hardware queue ID to an event queue ID and store it in
3367          * the metadata:
3368          * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
3369          * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
3370          * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
3371          * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
3372          */
3373 #define DLB_EVENT_QUEUE_ID_BYTE 5
3374         sse_evt[0] = _mm_insert_epi8(sse_evt[0],
3375                                      qid_mappings[qes[0].qid],
3376                                      DLB_EVENT_QUEUE_ID_BYTE);
3377         sse_evt[0] = _mm_insert_epi8(sse_evt[0],
3378                                      qid_mappings[qes[1].qid],
3379                                      DLB_EVENT_QUEUE_ID_BYTE + 8);
3380         sse_evt[1] = _mm_insert_epi8(sse_evt[1],
3381                                      qid_mappings[qes[2].qid],
3382                                      DLB_EVENT_QUEUE_ID_BYTE);
3383         sse_evt[1] = _mm_insert_epi8(sse_evt[1],
3384                                      qid_mappings[qes[3].qid],
3385                                      DLB_EVENT_QUEUE_ID_BYTE + 8);
3386
3387         /* Convert the hardware priority to an event priority and store it in
3388          * the metadata, while also returning the queue depth status
3389          * value captured by the hardware, storing it in impl_opaque, which can
3390          * be read by the application but not modified
3391          * sse_evt[0][55:48]   = DLB2_TO_EV_PRIO(qes[0].priority)
3392          * sse_evt[0][63:56]   = qes[0].qid_depth
3393          * sse_evt[0][119:112] = DLB2_TO_EV_PRIO(qes[1].priority)
3394          * sse_evt[0][127:120] = qes[1].qid_depth
3395          * sse_evt[1][55:48]   = DLB2_TO_EV_PRIO(qes[2].priority)
3396          * sse_evt[1][63:56]   = qes[2].qid_depth
3397          * sse_evt[1][119:112] = DLB2_TO_EV_PRIO(qes[3].priority)
3398          * sse_evt[1][127:120] = qes[3].qid_depth
3399          */
3400 #define DLB_EVENT_PRIO_IMPL_OPAQUE_WORD 3
3401 #define DLB_BYTE_SHIFT 8
3402         sse_evt[0] =
3403                 _mm_insert_epi16(sse_evt[0],
3404                         DLB2_TO_EV_PRIO((uint8_t)qes[0].priority) |
3405                         (qes[0].qid_depth << DLB_BYTE_SHIFT),
3406                         DLB_EVENT_PRIO_IMPL_OPAQUE_WORD);
3407         sse_evt[0] =
3408                 _mm_insert_epi16(sse_evt[0],
3409                         DLB2_TO_EV_PRIO((uint8_t)qes[1].priority) |
3410                         (qes[1].qid_depth << DLB_BYTE_SHIFT),
3411                         DLB_EVENT_PRIO_IMPL_OPAQUE_WORD + 4);
3412         sse_evt[1] =
3413                 _mm_insert_epi16(sse_evt[1],
3414                         DLB2_TO_EV_PRIO((uint8_t)qes[2].priority) |
3415                         (qes[2].qid_depth << DLB_BYTE_SHIFT),
3416                         DLB_EVENT_PRIO_IMPL_OPAQUE_WORD);
3417         sse_evt[1] =
3418                 _mm_insert_epi16(sse_evt[1],
3419                         DLB2_TO_EV_PRIO((uint8_t)qes[3].priority) |
3420                         (qes[3].qid_depth << DLB_BYTE_SHIFT),
3421                         DLB_EVENT_PRIO_IMPL_OPAQUE_WORD + 4);
3422
3423         /* Write the event type, sub event type, and flow_id to the event
3424          * metadata.
3425          * sse_evt[0][31:0]   = qes[0].flow_id |
3426          *                      qes[0].u.event_type.major << 28 |
3427          *                      qes[0].u.event_type.sub << 20;
3428          * sse_evt[0][95:64]  = qes[1].flow_id |
3429          *                      qes[1].u.event_type.major << 28 |
3430          *                      qes[1].u.event_type.sub << 20;
3431          * sse_evt[1][31:0]   = qes[2].flow_id |
3432          *                      qes[2].u.event_type.major << 28 |
3433          *                      qes[2].u.event_type.sub << 20;
3434          * sse_evt[1][95:64]  = qes[3].flow_id |
3435          *                      qes[3].u.event_type.major << 28 |
3436          *                      qes[3].u.event_type.sub << 20;
3437          */
3438 #define DLB_EVENT_EV_TYPE_DW 0
3439 #define DLB_EVENT_EV_TYPE_SHIFT 28
3440 #define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
3441         sse_evt[0] = _mm_insert_epi32(sse_evt[0],
3442                         qes[0].flow_id |
3443                         qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
3444                         qes[0].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
3445                         DLB_EVENT_EV_TYPE_DW);
3446         sse_evt[0] = _mm_insert_epi32(sse_evt[0],
3447                         qes[1].flow_id |
3448                         qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
3449                         qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
3450                         DLB_EVENT_EV_TYPE_DW + 2);
3451         sse_evt[1] = _mm_insert_epi32(sse_evt[1],
3452                         qes[2].flow_id |
3453                         qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
3454                         qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
3455                         DLB_EVENT_EV_TYPE_DW);
3456         sse_evt[1] = _mm_insert_epi32(sse_evt[1],
3457                         qes[3].flow_id |
3458                         qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
3459                         qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
3460                         DLB_EVENT_EV_TYPE_DW + 2);
3461
3462         /* Write the sched type to the event metadata. 'op' and 'rsvd' are not
3463          * set:
3464          * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
3465          * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
3466          * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
3467          * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
3468          */
3469 #define DLB_EVENT_SCHED_TYPE_BYTE 4
3470 #define DLB_EVENT_SCHED_TYPE_SHIFT 6
3471         sse_evt[0] = _mm_insert_epi8(sse_evt[0],
3472                 sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
3473                 DLB_EVENT_SCHED_TYPE_BYTE);
3474         sse_evt[0] = _mm_insert_epi8(sse_evt[0],
3475                 sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
3476                 DLB_EVENT_SCHED_TYPE_BYTE + 8);
3477         sse_evt[1] = _mm_insert_epi8(sse_evt[1],
3478                 sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
3479                 DLB_EVENT_SCHED_TYPE_BYTE);
3480         sse_evt[1] = _mm_insert_epi8(sse_evt[1],
3481                 sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
3482                 DLB_EVENT_SCHED_TYPE_BYTE + 8);
3483
3484         /* Store the metadata to the event (use the double-precision
3485          * _mm_storeh_pd because there is no integer function for storing the
3486          * upper 64b):
3487          * events[0].event = sse_evt[0][63:0]
3488          * events[1].event = sse_evt[0][127:64]
3489          * events[2].event = sse_evt[1][63:0]
3490          * events[3].event = sse_evt[1][127:64]
3491          */
3492         _mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
3493         _mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
3494         _mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
3495         _mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
3496
3497         DLB2_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
3498         DLB2_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
3499         DLB2_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
3500         DLB2_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
3501
3502         DLB2_INC_STAT(
3503                 ev_port->stats.queue[events[0].queue_id].
3504                         qid_depth[qes[0].qid_depth],
3505                 1);
3506         DLB2_INC_STAT(
3507                 ev_port->stats.queue[events[1].queue_id].
3508                         qid_depth[qes[1].qid_depth],
3509                 1);
3510         DLB2_INC_STAT(
3511                 ev_port->stats.queue[events[2].queue_id].
3512                         qid_depth[qes[2].qid_depth],
3513                 1);
3514         DLB2_INC_STAT(
3515                 ev_port->stats.queue[events[3].queue_id].
3516                         qid_depth[qes[3].qid_depth],
3517                 1);
3518
3519         DLB2_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
3520
3521         return num_events;
3522 }
3523
3524 static __rte_always_inline int
3525 dlb2_recv_qe_sparse(struct dlb2_port *qm_port, struct dlb2_dequeue_qe *qe)
3526 {
3527         volatile struct dlb2_dequeue_qe *cq_addr;
3528         uint8_t xor_mask[2] = {0x0F, 0x00};
3529         const uint8_t and_mask = 0x0F;
3530         __m128i *qes = (__m128i *)qe;
3531         uint8_t gen_bits, gen_bit;
3532         uintptr_t addr[4];
3533         uint16_t idx;
3534
3535         cq_addr = dlb2_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
3536
3537         idx = qm_port->cq_idx_unmasked & qm_port->cq_depth_mask;
3538         /* Load the next 4 QEs */
3539         addr[0] = (uintptr_t)&cq_addr[idx];
3540         addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
3541         addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
3542         addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
3543
3544         /* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
3545         rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
3546         rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
3547         rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
3548         rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
3549
3550         /* Correct the xor_mask for wrap-around QEs */
3551         gen_bit = qm_port->gen_bit;
3552         xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
3553         xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
3554         xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
3555
3556         /* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
3557          * valid, then QEs[0:N-1] are too.
3558          */
3559         qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
3560         rte_compiler_barrier();
3561         qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
3562         rte_compiler_barrier();
3563         qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
3564         rte_compiler_barrier();
3565         qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
3566
3567         /* Extract and combine the gen bits */
3568         gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
3569                    ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
3570                    ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
3571                    ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
3572
3573         /* XOR the combined bits such that a 1 represents a valid QE */
3574         gen_bits ^= xor_mask[gen_bit];
3575
3576         /* Mask off gen bits we don't care about */
3577         gen_bits &= and_mask;
3578
3579         return __builtin_popcount(gen_bits);
3580 }
3581
3582 static inline void
3583 _process_deq_qes_vec_impl(struct dlb2_port *qm_port,
3584                           struct rte_event *events,
3585                           __m128i v_qe_3,
3586                           __m128i v_qe_2,
3587                           __m128i v_qe_1,
3588                           __m128i v_qe_0,
3589                           __m128i v_qe_meta,
3590                           __m128i v_qe_status,
3591                           uint32_t valid_events)
3592 {
3593         /* Look up the event QIDs, using the hardware QIDs to index the
3594          * port's QID mapping.
3595          *
3596          * Each v_qe_[0-4] is just a 16-byte load of the whole QE. It is
3597          * passed along in registers as the QE data is required later.
3598          *
3599          * v_qe_meta is an u32 unpack of all 4x QEs. A.k.a, it contains one
3600          * 32-bit slice of each QE, so makes up a full SSE register. This
3601          * allows parallel processing of 4x QEs in a single register.
3602          */
3603
3604         __m128i v_qid_done = {0};
3605         int hw_qid0 = _mm_extract_epi8(v_qe_meta, 2);
3606         int hw_qid1 = _mm_extract_epi8(v_qe_meta, 6);
3607         int hw_qid2 = _mm_extract_epi8(v_qe_meta, 10);
3608         int hw_qid3 = _mm_extract_epi8(v_qe_meta, 14);
3609
3610         int ev_qid0 = qm_port->qid_mappings[hw_qid0];
3611         int ev_qid1 = qm_port->qid_mappings[hw_qid1];
3612         int ev_qid2 = qm_port->qid_mappings[hw_qid2];
3613         int ev_qid3 = qm_port->qid_mappings[hw_qid3];
3614
3615         int hw_sched0 = _mm_extract_epi8(v_qe_meta, 3) & 3ul;
3616         int hw_sched1 = _mm_extract_epi8(v_qe_meta, 7) & 3ul;
3617         int hw_sched2 = _mm_extract_epi8(v_qe_meta, 11) & 3ul;
3618         int hw_sched3 = _mm_extract_epi8(v_qe_meta, 15) & 3ul;
3619
3620         v_qid_done = _mm_insert_epi8(v_qid_done, ev_qid0, 2);
3621         v_qid_done = _mm_insert_epi8(v_qid_done, ev_qid1, 6);
3622         v_qid_done = _mm_insert_epi8(v_qid_done, ev_qid2, 10);
3623         v_qid_done = _mm_insert_epi8(v_qid_done, ev_qid3, 14);
3624
3625         /* Schedule field remapping using byte shuffle
3626          * - Full byte containing sched field handled here (op, rsvd are zero)
3627          * - Note sanitizing the register requires two masking ANDs:
3628          *   1) to strip prio/msg_type from byte for correct shuffle lookup
3629          *   2) to strip any non-sched-field lanes from any results to OR later
3630          * - Final byte result is >> 10 to another byte-lane inside the u32.
3631          *   This makes the final combination OR easier to make the rte_event.
3632          */
3633         __m128i v_sched_done;
3634         __m128i v_sched_bits;
3635         {
3636                 static const uint8_t sched_type_map[16] = {
3637                         [DLB2_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
3638                         [DLB2_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
3639                         [DLB2_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
3640                         [DLB2_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
3641                 };
3642                 static const uint8_t sched_and_mask[16] = {
3643                         0x00, 0x00, 0x00, 0x03,
3644                         0x00, 0x00, 0x00, 0x03,
3645                         0x00, 0x00, 0x00, 0x03,
3646                         0x00, 0x00, 0x00, 0x03,
3647                 };
3648                 const __m128i v_sched_map = _mm_loadu_si128(
3649                                              (const __m128i *)sched_type_map);
3650                 __m128i v_sched_mask = _mm_loadu_si128(
3651                                              (const __m128i *)&sched_and_mask);
3652                 v_sched_bits = _mm_and_si128(v_qe_meta, v_sched_mask);
3653                 __m128i v_sched_remapped = _mm_shuffle_epi8(v_sched_map,
3654                                                             v_sched_bits);
3655                 __m128i v_preshift = _mm_and_si128(v_sched_remapped,
3656                                                    v_sched_mask);
3657                 v_sched_done = _mm_srli_epi32(v_preshift, 10);
3658         }
3659
3660         /* Priority handling
3661          * - QE provides 3 bits of priority
3662          * - Shift << 3 to move to MSBs for byte-prio in rte_event
3663          * - Mask bits to avoid pollution, leaving only 3 prio MSBs in reg
3664          */
3665         __m128i v_prio_done;
3666         {
3667                 static const uint8_t prio_mask[16] = {
3668                         0x00, 0x00, 0x00, 0x07 << 5,
3669                         0x00, 0x00, 0x00, 0x07 << 5,
3670                         0x00, 0x00, 0x00, 0x07 << 5,
3671                         0x00, 0x00, 0x00, 0x07 << 5,
3672                 };
3673                 __m128i v_prio_mask  = _mm_loadu_si128(
3674                                                 (const __m128i *)prio_mask);
3675                 __m128i v_prio_shifted = _mm_slli_epi32(v_qe_meta, 3);
3676                 v_prio_done = _mm_and_si128(v_prio_shifted, v_prio_mask);
3677         }
3678
3679         /* Event Sub/Type handling:
3680          * we want to keep the lower 12 bits of each QE. Shift up by 20 bits
3681          * to get the sub/ev type data into rte_event location, clearing the
3682          * lower 20 bits in the process.
3683          */
3684         __m128i v_types_done;
3685         {
3686                 static const uint8_t event_mask[16] = {
3687                         0x0f, 0x00, 0x00, 0x00,
3688                         0x0f, 0x00, 0x00, 0x00,
3689                         0x0f, 0x00, 0x00, 0x00,
3690                         0x0f, 0x00, 0x00, 0x00,
3691                 };
3692                 static const uint8_t sub_event_mask[16] = {
3693                         0xff, 0x00, 0x00, 0x00,
3694                         0xff, 0x00, 0x00, 0x00,
3695                         0xff, 0x00, 0x00, 0x00,
3696                         0xff, 0x00, 0x00, 0x00,
3697                 };
3698                 static const uint8_t flow_mask[16] = {
3699                         0xff, 0xff, 0x00, 0x00,
3700                         0xff, 0xff, 0x00, 0x00,
3701                         0xff, 0xff, 0x00, 0x00,
3702                         0xff, 0xff, 0x00, 0x00,
3703                 };
3704                 __m128i v_event_mask  = _mm_loadu_si128(
3705                                         (const __m128i *)event_mask);
3706                 __m128i v_sub_event_mask  = _mm_loadu_si128(
3707                                         (const __m128i *)sub_event_mask);
3708                 __m128i v_flow_mask  = _mm_loadu_si128(
3709                                        (const __m128i *)flow_mask);
3710                 __m128i v_sub = _mm_srli_epi32(v_qe_meta, 8);
3711                 v_sub = _mm_and_si128(v_sub, v_sub_event_mask);
3712                 __m128i v_type = _mm_and_si128(v_qe_meta, v_event_mask);
3713                 v_type = _mm_slli_epi32(v_type, 8);
3714                 v_types_done = _mm_or_si128(v_type, v_sub);
3715                 v_types_done = _mm_slli_epi32(v_types_done, 20);
3716                 __m128i v_flow = _mm_and_si128(v_qe_status, v_flow_mask);
3717                 v_types_done = _mm_or_si128(v_types_done, v_flow);
3718         }
3719
3720         /* Combine QID, Sched and Prio fields, then Shift >> 8 bits to align
3721          * with the rte_event, allowing unpacks to move/blend with payload.
3722          */
3723         __m128i v_q_s_p_done;
3724         {
3725                 __m128i v_qid_sched = _mm_or_si128(v_qid_done, v_sched_done);
3726                 __m128i v_q_s_prio = _mm_or_si128(v_qid_sched, v_prio_done);
3727                 v_q_s_p_done = _mm_srli_epi32(v_q_s_prio, 8);
3728         }
3729
3730         __m128i v_unpk_ev_23, v_unpk_ev_01, v_ev_2, v_ev_3, v_ev_0, v_ev_1;
3731
3732         /* Unpack evs into u64 metadata, then indiv events */
3733         v_unpk_ev_23 = _mm_unpackhi_epi32(v_types_done, v_q_s_p_done);
3734         v_unpk_ev_01 = _mm_unpacklo_epi32(v_types_done, v_q_s_p_done);
3735
3736         switch (valid_events) {
3737         case 4:
3738                 v_ev_3 = _mm_blend_epi16(v_unpk_ev_23, v_qe_3, 0x0F);
3739                 v_ev_3 = _mm_alignr_epi8(v_ev_3, v_ev_3, 8);
3740                 _mm_storeu_si128((__m128i *)&events[3], v_ev_3);
3741                 DLB2_INC_STAT(qm_port->ev_port->stats.rx_sched_cnt[hw_sched3],
3742                               1);
3743                 /* fallthrough */
3744         case 3:
3745                 v_ev_2 = _mm_unpacklo_epi64(v_unpk_ev_23, v_qe_2);
3746                 _mm_storeu_si128((__m128i *)&events[2], v_ev_2);
3747                 DLB2_INC_STAT(qm_port->ev_port->stats.rx_sched_cnt[hw_sched2],
3748                               1);
3749                 /* fallthrough */
3750         case 2:
3751                 v_ev_1 = _mm_blend_epi16(v_unpk_ev_01, v_qe_1, 0x0F);
3752                 v_ev_1 = _mm_alignr_epi8(v_ev_1, v_ev_1, 8);
3753                 _mm_storeu_si128((__m128i *)&events[1], v_ev_1);
3754                 DLB2_INC_STAT(qm_port->ev_port->stats.rx_sched_cnt[hw_sched1],
3755                               1);
3756                 /* fallthrough */
3757         case 1:
3758                 v_ev_0 = _mm_unpacklo_epi64(v_unpk_ev_01, v_qe_0);
3759                 _mm_storeu_si128((__m128i *)&events[0], v_ev_0);
3760                 DLB2_INC_STAT(qm_port->ev_port->stats.rx_sched_cnt[hw_sched0],
3761                               1);
3762         }
3763 }
3764
3765 static __rte_always_inline int
3766 dlb2_recv_qe_sparse_vec(struct dlb2_port *qm_port, void *events,
3767                         uint32_t max_events)
3768 {
3769         /* Using unmasked idx for perf, and masking manually */
3770         uint16_t idx = qm_port->cq_idx_unmasked;
3771         volatile struct dlb2_dequeue_qe *cq_addr;
3772
3773         cq_addr = dlb2_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
3774
3775         uintptr_t qe_ptr_3 = (uintptr_t)&cq_addr[(idx + 12) &
3776                                                  qm_port->cq_depth_mask];
3777         uintptr_t qe_ptr_2 = (uintptr_t)&cq_addr[(idx +  8) &
3778                                                  qm_port->cq_depth_mask];
3779         uintptr_t qe_ptr_1 = (uintptr_t)&cq_addr[(idx +  4) &
3780                                                  qm_port->cq_depth_mask];
3781         uintptr_t qe_ptr_0 = (uintptr_t)&cq_addr[(idx +  0) &
3782                                                  qm_port->cq_depth_mask];
3783
3784         /* Load QEs from CQ: use compiler barriers to avoid load reordering */
3785         __m128i v_qe_3 = _mm_loadu_si128((const __m128i *)qe_ptr_3);
3786         rte_compiler_barrier();
3787         __m128i v_qe_2 = _mm_loadu_si128((const __m128i *)qe_ptr_2);
3788         rte_compiler_barrier();
3789         __m128i v_qe_1 = _mm_loadu_si128((const __m128i *)qe_ptr_1);
3790         rte_compiler_barrier();
3791         __m128i v_qe_0 = _mm_loadu_si128((const __m128i *)qe_ptr_0);
3792
3793         /* Generate the pkt_shuffle mask;
3794          * - Avoids load in otherwise load-heavy section of code
3795          * - Moves bytes 3,7,11,15 (gen bit bytes) to LSB bytes in XMM
3796          */
3797         const uint32_t stat_shuf_bytes = (15 << 24) | (11 << 16) | (7 << 8) | 3;
3798         __m128i v_zeros = _mm_setzero_si128();
3799         __m128i v_ffff = _mm_cmpeq_epi8(v_zeros, v_zeros);
3800         __m128i v_stat_shuf_mask = _mm_insert_epi32(v_ffff, stat_shuf_bytes, 0);
3801
3802         /* Extract u32 components required from the QE
3803          * - QE[64 to 95 ] for metadata (qid, sched, prio, event type, ...)
3804          * - QE[96 to 127] for status (cq gen bit, error)
3805          *
3806          * Note that stage 1 of the unpacking is re-used for both u32 extracts
3807          */
3808         __m128i v_qe_02 = _mm_unpackhi_epi32(v_qe_0, v_qe_2);
3809         __m128i v_qe_13 = _mm_unpackhi_epi32(v_qe_1, v_qe_3);
3810         __m128i v_qe_status = _mm_unpackhi_epi32(v_qe_02, v_qe_13);
3811         __m128i v_qe_meta   = _mm_unpacklo_epi32(v_qe_02, v_qe_13);
3812
3813         /* Status byte (gen_bit, error) handling:
3814          * - Shuffle to lanes 0,1,2,3, clear all others
3815          * - Shift right by 7 for gen bit to MSB, movemask to scalar
3816          * - Shift right by 2 for error bit to MSB, movemask to scalar
3817          */
3818         __m128i v_qe_shuffled = _mm_shuffle_epi8(v_qe_status, v_stat_shuf_mask);
3819         __m128i v_qes_shift_gen_bit = _mm_slli_epi32(v_qe_shuffled, 7);
3820         int32_t qe_gen_bits = _mm_movemask_epi8(v_qes_shift_gen_bit) & 0xf;
3821
3822         /* Expected vs Reality of QE Gen bits
3823          * - cq_rolling_mask provides expected bits
3824          * - QE loads, unpacks/shuffle and movemask provides reality
3825          * - XOR of the two gives bitmask of new packets
3826          * - POPCNT to get the number of new events
3827          */
3828         uint64_t rolling = qm_port->cq_rolling_mask & 0xF;
3829         uint64_t qe_xor_bits = (qe_gen_bits ^ rolling);
3830         uint32_t count_new = __builtin_popcount(qe_xor_bits);
3831         count_new = RTE_MIN(count_new, max_events);
3832         if (!count_new)
3833                 return 0;
3834
3835         /* emulate a 128 bit rotate using 2x 64-bit numbers and bit-shifts */
3836
3837         uint64_t m_rshift = qm_port->cq_rolling_mask >> count_new;
3838         uint64_t m_lshift = qm_port->cq_rolling_mask << (64 - count_new);
3839         uint64_t m2_rshift = qm_port->cq_rolling_mask_2 >> count_new;
3840         uint64_t m2_lshift = qm_port->cq_rolling_mask_2 << (64 - count_new);
3841
3842         /* shifted out of m2 into MSB of m */
3843         qm_port->cq_rolling_mask = (m_rshift | m2_lshift);
3844
3845         /* shifted out of m "looped back" into MSB of m2 */
3846         qm_port->cq_rolling_mask_2 = (m2_rshift | m_lshift);
3847
3848         /* Prefetch the next QEs - should run as IPC instead of cycles */
3849         rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
3850         rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
3851         rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
3852         rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
3853
3854         /* Convert QEs from XMM regs to events and store events directly */
3855         _process_deq_qes_vec_impl(qm_port, events, v_qe_3, v_qe_2, v_qe_1,
3856                                   v_qe_0, v_qe_meta, v_qe_status, count_new);
3857
3858         return count_new;
3859 }
3860
3861 static inline void
3862 dlb2_inc_cq_idx(struct dlb2_port *qm_port, int cnt)
3863 {
3864         uint16_t idx = qm_port->cq_idx_unmasked + cnt;
3865
3866         qm_port->cq_idx_unmasked = idx;
3867         qm_port->cq_idx = idx & qm_port->cq_depth_mask;
3868         qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
3869 }
3870
3871 static inline int16_t
3872 dlb2_hw_dequeue_sparse(struct dlb2_eventdev *dlb2,
3873                        struct dlb2_eventdev_port *ev_port,
3874                        struct rte_event *events,
3875                        uint16_t max_num,
3876                        uint64_t dequeue_timeout_ticks)
3877 {
3878         uint64_t start_ticks = 0ULL;
3879         struct dlb2_port *qm_port;
3880         int num = 0;
3881         bool use_scalar;
3882         uint64_t timeout;
3883
3884         qm_port = &ev_port->qm_port;
3885         use_scalar = qm_port->use_scalar;
3886
3887         if (!dlb2->global_dequeue_wait)
3888                 timeout = dequeue_timeout_ticks;
3889         else
3890                 timeout = dlb2->global_dequeue_wait_ticks;
3891
3892         start_ticks = rte_get_timer_cycles();
3893
3894         use_scalar = use_scalar || (max_num & 0x3);
3895
3896         while (num < max_num) {
3897                 struct dlb2_dequeue_qe qes[DLB2_NUM_QES_PER_CACHE_LINE];
3898                 int num_avail;
3899                 if (use_scalar) {
3900                         num_avail = dlb2_recv_qe_sparse(qm_port, qes);
3901                         num_avail = RTE_MIN(num_avail, max_num - num);
3902                         dlb2_inc_cq_idx(qm_port, num_avail << 2);
3903                         if (num_avail == DLB2_NUM_QES_PER_CACHE_LINE)
3904                                 num += dlb2_process_dequeue_four_qes(ev_port,
3905                                                                   qm_port,
3906                                                                   &events[num],
3907                                                                   &qes[0]);
3908                         else if (num_avail)
3909                                 num += dlb2_process_dequeue_qes(ev_port,
3910                                                                 qm_port,
3911                                                                 &events[num],
3912                                                                 &qes[0],
3913                                                                 num_avail);
3914                 } else { /* !use_scalar */
3915                         num_avail = dlb2_recv_qe_sparse_vec(qm_port,
3916                                                             &events[num],
3917                                                             max_num - num);
3918                         num += num_avail;
3919                         dlb2_inc_cq_idx(qm_port, num_avail << 2);
3920                         DLB2_INC_STAT(ev_port->stats.traffic.rx_ok, num_avail);
3921                 }
3922                 if (!num_avail) {
3923                         if (num > 0)
3924                                 break;
3925                         else if (dlb2_dequeue_wait(dlb2, ev_port, qm_port,
3926                                                    timeout, start_ticks))
3927                                 break;
3928                 }
3929         }
3930
3931         qm_port->owed_tokens += num;
3932
3933         if (num) {
3934                 if (qm_port->token_pop_mode == AUTO_POP)
3935                         dlb2_consume_qe_immediate(qm_port, num);
3936
3937                 ev_port->outstanding_releases += num;
3938
3939                 dlb2_port_credits_inc(qm_port, num);
3940         }
3941
3942         return num;
3943 }
3944
3945 static __rte_always_inline int
3946 dlb2_recv_qe(struct dlb2_port *qm_port, struct dlb2_dequeue_qe *qe,
3947              uint8_t *offset)
3948 {
3949         uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
3950                                    {0x00, 0x01, 0x03, 0x07} };
3951         uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
3952         volatile struct dlb2_dequeue_qe *cq_addr;
3953         __m128i *qes = (__m128i *)qe;
3954         uint64_t *cache_line_base;
3955         uint8_t gen_bits;
3956
3957         cq_addr = dlb2_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
3958         cq_addr = &cq_addr[qm_port->cq_idx];
3959
3960         cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
3961         *offset = ((uintptr_t)cq_addr & 0x30) >> 4;
3962
3963         /* Load the next CQ cache line from memory. Pack these reads as tight
3964          * as possible to reduce the chance that DLB invalidates the line while
3965          * the CPU is reading it. Read the cache line backwards to ensure that
3966          * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
3967          *
3968          * (Valid QEs start at &qe[offset])
3969          */
3970         qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
3971         qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
3972         qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
3973         qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
3974
3975         /* Evict the cache line ASAP */
3976         rte_cldemote(cache_line_base);
3977
3978         /* Extract and combine the gen bits */
3979         gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
3980                    ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
3981                    ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
3982                    ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
3983
3984         /* XOR the combined bits such that a 1 represents a valid QE */
3985         gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
3986
3987         /* Mask off gen bits we don't care about */
3988         gen_bits &= and_mask[*offset];
3989
3990         return __builtin_popcount(gen_bits);
3991 }
3992
3993 static inline int16_t
3994 dlb2_hw_dequeue(struct dlb2_eventdev *dlb2,
3995                 struct dlb2_eventdev_port *ev_port,
3996                 struct rte_event *events,
3997                 uint16_t max_num,
3998                 uint64_t dequeue_timeout_ticks)
3999 {
4000         uint64_t timeout;
4001         uint64_t start_ticks = 0ULL;
4002         struct dlb2_port *qm_port;
4003         int num = 0;
4004
4005         qm_port = &ev_port->qm_port;
4006
4007         /* We have a special implementation for waiting. Wait can be:
4008          * 1) no waiting at all
4009          * 2) busy poll only
4010          * 3) wait for interrupt. If wakeup and poll time
4011          * has expired, then return to caller
4012          * 4) umonitor/umwait repeatedly up to poll time
4013          */
4014
4015         /* If configured for per dequeue wait, then use wait value provided
4016          * to this API. Otherwise we must use the global
4017          * value from eventdev config time.
4018          */
4019         if (!dlb2->global_dequeue_wait)
4020                 timeout = dequeue_timeout_ticks;
4021         else
4022                 timeout = dlb2->global_dequeue_wait_ticks;
4023
4024         start_ticks = rte_get_timer_cycles();
4025
4026         while (num < max_num) {
4027                 struct dlb2_dequeue_qe qes[DLB2_NUM_QES_PER_CACHE_LINE];
4028                 uint8_t offset;
4029                 int num_avail;
4030
4031                 /* Copy up to 4 QEs from the current cache line into qes */
4032                 num_avail = dlb2_recv_qe(qm_port, qes, &offset);
4033
4034                 /* But don't process more than the user requested */
4035                 num_avail = RTE_MIN(num_avail, max_num - num);
4036
4037                 dlb2_inc_cq_idx(qm_port, num_avail);
4038
4039                 if (num_avail == DLB2_NUM_QES_PER_CACHE_LINE)
4040                         num += dlb2_process_dequeue_four_qes(ev_port,
4041                                                              qm_port,
4042                                                              &events[num],
4043                                                              &qes[offset]);
4044                 else if (num_avail)
4045                         num += dlb2_process_dequeue_qes(ev_port,
4046                                                         qm_port,
4047                                                         &events[num],
4048                                                         &qes[offset],
4049                                                         num_avail);
4050                 else if ((timeout == 0) || (num > 0))
4051                         /* Not waiting in any form, or 1+ events received? */
4052                         break;
4053                 else if (dlb2_dequeue_wait(dlb2, ev_port, qm_port,
4054                                            timeout, start_ticks))
4055                         break;
4056         }
4057
4058         qm_port->owed_tokens += num;
4059
4060         if (num) {
4061                 if (qm_port->token_pop_mode == AUTO_POP)
4062                         dlb2_consume_qe_immediate(qm_port, num);
4063
4064                 ev_port->outstanding_releases += num;
4065
4066                 dlb2_port_credits_inc(qm_port, num);
4067         }
4068
4069         return num;
4070 }
4071
4072 static uint16_t
4073 dlb2_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
4074                          uint64_t wait)
4075 {
4076         struct dlb2_eventdev_port *ev_port = event_port;
4077         struct dlb2_port *qm_port = &ev_port->qm_port;
4078         struct dlb2_eventdev *dlb2 = ev_port->dlb2;
4079         uint16_t cnt;
4080
4081         RTE_ASSERT(ev_port->setup_done);
4082         RTE_ASSERT(ev != NULL);
4083
4084         if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
4085                 uint16_t out_rels = ev_port->outstanding_releases;
4086
4087                 dlb2_event_release(dlb2, ev_port->id, out_rels);
4088
4089                 DLB2_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
4090         }
4091
4092         if (qm_port->token_pop_mode == DEFERRED_POP && qm_port->owed_tokens)
4093                 dlb2_consume_qe_immediate(qm_port, qm_port->owed_tokens);
4094
4095         cnt = dlb2_hw_dequeue(dlb2, ev_port, ev, num, wait);
4096
4097         DLB2_INC_STAT(ev_port->stats.traffic.total_polls, 1);
4098         DLB2_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
4099
4100         return cnt;
4101 }
4102
4103 static uint16_t
4104 dlb2_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
4105 {
4106         return dlb2_event_dequeue_burst(event_port, ev, 1, wait);
4107 }
4108
4109 static uint16_t
4110 dlb2_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
4111                                 uint16_t num, uint64_t wait)
4112 {
4113         struct dlb2_eventdev_port *ev_port = event_port;
4114         struct dlb2_port *qm_port = &ev_port->qm_port;
4115         struct dlb2_eventdev *dlb2 = ev_port->dlb2;
4116         uint16_t cnt;
4117
4118         RTE_ASSERT(ev_port->setup_done);
4119         RTE_ASSERT(ev != NULL);
4120
4121         if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
4122                 uint16_t out_rels = ev_port->outstanding_releases;
4123
4124                 dlb2_event_release(dlb2, ev_port->id, out_rels);
4125
4126                 DLB2_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
4127         }
4128
4129         if (qm_port->token_pop_mode == DEFERRED_POP && qm_port->owed_tokens)
4130                 dlb2_consume_qe_immediate(qm_port, qm_port->owed_tokens);
4131
4132         cnt = dlb2_hw_dequeue_sparse(dlb2, ev_port, ev, num, wait);
4133
4134         DLB2_INC_STAT(ev_port->stats.traffic.total_polls, 1);
4135         DLB2_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
4136         return cnt;
4137 }
4138
4139 static uint16_t
4140 dlb2_event_dequeue_sparse(void *event_port, struct rte_event *ev,
4141                           uint64_t wait)
4142 {
4143         return dlb2_event_dequeue_burst_sparse(event_port, ev, 1, wait);
4144 }
4145
4146 static void
4147 dlb2_flush_port(struct rte_eventdev *dev, int port_id)
4148 {
4149         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
4150         eventdev_stop_flush_t flush;
4151         struct rte_event ev;
4152         uint8_t dev_id;
4153         void *arg;
4154         int i;
4155
4156         flush = dev->dev_ops->dev_stop_flush;
4157         dev_id = dev->data->dev_id;
4158         arg = dev->data->dev_stop_flush_arg;
4159
4160         while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
4161                 if (flush)
4162                         flush(dev_id, ev, arg);
4163
4164                 if (dlb2->ev_ports[port_id].qm_port.is_directed)
4165                         continue;
4166
4167                 ev.op = RTE_EVENT_OP_RELEASE;
4168
4169                 rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
4170         }
4171
4172         /* Enqueue any additional outstanding releases */
4173         ev.op = RTE_EVENT_OP_RELEASE;
4174
4175         for (i = dlb2->ev_ports[port_id].outstanding_releases; i > 0; i--)
4176                 rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
4177 }
4178
4179 static uint32_t
4180 dlb2_get_ldb_queue_depth(struct dlb2_eventdev *dlb2,
4181                          struct dlb2_eventdev_queue *queue)
4182 {
4183         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
4184         struct dlb2_get_ldb_queue_depth_args cfg;
4185         int ret;
4186
4187         cfg.queue_id = queue->qm_queue.id;
4188
4189         ret = dlb2_iface_get_ldb_queue_depth(handle, &cfg);
4190         if (ret < 0) {
4191                 DLB2_LOG_ERR("dlb2: get_ldb_queue_depth ret=%d (driver status: %s)\n",
4192                              ret, dlb2_error_strings[cfg.response.status]);
4193                 return ret;
4194         }
4195
4196         return cfg.response.id;
4197 }
4198
4199 static uint32_t
4200 dlb2_get_dir_queue_depth(struct dlb2_eventdev *dlb2,
4201                          struct dlb2_eventdev_queue *queue)
4202 {
4203         struct dlb2_hw_dev *handle = &dlb2->qm_instance;
4204         struct dlb2_get_dir_queue_depth_args cfg;
4205         int ret;
4206
4207         cfg.queue_id = queue->qm_queue.id;
4208
4209         ret = dlb2_iface_get_dir_queue_depth(handle, &cfg);
4210         if (ret < 0) {
4211                 DLB2_LOG_ERR("dlb2: get_dir_queue_depth ret=%d (driver status: %s)\n",
4212                              ret, dlb2_error_strings[cfg.response.status]);
4213                 return ret;
4214         }
4215
4216         return cfg.response.id;
4217 }
4218
4219 uint32_t
4220 dlb2_get_queue_depth(struct dlb2_eventdev *dlb2,
4221                      struct dlb2_eventdev_queue *queue)
4222 {
4223         if (queue->qm_queue.is_directed)
4224                 return dlb2_get_dir_queue_depth(dlb2, queue);
4225         else
4226                 return dlb2_get_ldb_queue_depth(dlb2, queue);
4227 }
4228
4229 static bool
4230 dlb2_queue_is_empty(struct dlb2_eventdev *dlb2,
4231                     struct dlb2_eventdev_queue *queue)
4232 {
4233         return dlb2_get_queue_depth(dlb2, queue) == 0;
4234 }
4235
4236 static bool
4237 dlb2_linked_queues_empty(struct dlb2_eventdev *dlb2)
4238 {
4239         int i;
4240
4241         for (i = 0; i < dlb2->num_queues; i++) {
4242                 if (dlb2->ev_queues[i].num_links == 0)
4243                         continue;
4244                 if (!dlb2_queue_is_empty(dlb2, &dlb2->ev_queues[i]))
4245                         return false;
4246         }
4247
4248         return true;
4249 }
4250
4251 static bool
4252 dlb2_queues_empty(struct dlb2_eventdev *dlb2)
4253 {
4254         int i;
4255
4256         for (i = 0; i < dlb2->num_queues; i++) {
4257                 if (!dlb2_queue_is_empty(dlb2, &dlb2->ev_queues[i]))
4258                         return false;
4259         }
4260
4261         return true;
4262 }
4263
4264 static void
4265 dlb2_drain(struct rte_eventdev *dev)
4266 {
4267         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
4268         struct dlb2_eventdev_port *ev_port = NULL;
4269         uint8_t dev_id;
4270         int i;
4271
4272         dev_id = dev->data->dev_id;
4273
4274         while (!dlb2_linked_queues_empty(dlb2)) {
4275                 /* Flush all the ev_ports, which will drain all their connected
4276                  * queues.
4277                  */
4278                 for (i = 0; i < dlb2->num_ports; i++)
4279                         dlb2_flush_port(dev, i);
4280         }
4281
4282         /* The queues are empty, but there may be events left in the ports. */
4283         for (i = 0; i < dlb2->num_ports; i++)
4284                 dlb2_flush_port(dev, i);
4285
4286         /* If the domain's queues are empty, we're done. */
4287         if (dlb2_queues_empty(dlb2))
4288                 return;
4289
4290         /* Else, there must be at least one unlinked load-balanced queue.
4291          * Select a load-balanced port with which to drain the unlinked
4292          * queue(s).
4293          */
4294         for (i = 0; i < dlb2->num_ports; i++) {
4295                 ev_port = &dlb2->ev_ports[i];
4296
4297                 if (!ev_port->qm_port.is_directed)
4298                         break;
4299         }
4300
4301         if (i == dlb2->num_ports) {
4302                 DLB2_LOG_ERR("internal error: no LDB ev_ports\n");
4303                 return;
4304         }
4305
4306         rte_errno = 0;
4307         rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
4308
4309         if (rte_errno) {
4310                 DLB2_LOG_ERR("internal error: failed to unlink ev_port %d\n",
4311                              ev_port->id);
4312                 return;
4313         }
4314
4315         for (i = 0; i < dlb2->num_queues; i++) {
4316                 uint8_t qid, prio;
4317                 int ret;
4318
4319                 if (dlb2_queue_is_empty(dlb2, &dlb2->ev_queues[i]))
4320                         continue;
4321
4322                 qid = i;
4323                 prio = 0;
4324
4325                 /* Link the ev_port to the queue */
4326                 ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
4327                 if (ret != 1) {
4328                         DLB2_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
4329                                      ev_port->id, qid);
4330                         return;
4331                 }
4332
4333                 /* Flush the queue */
4334                 while (!dlb2_queue_is_empty(dlb2, &dlb2->ev_queues[i]))
4335                         dlb2_flush_port(dev, ev_port->id);
4336
4337                 /* Drain any extant events in the ev_port. */
4338                 dlb2_flush_port(dev, ev_port->id);
4339
4340                 /* Unlink the ev_port from the queue */
4341                 ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
4342                 if (ret != 1) {
4343                         DLB2_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
4344                                      ev_port->id, qid);
4345                         return;
4346                 }
4347         }
4348 }
4349
4350 static void
4351 dlb2_eventdev_stop(struct rte_eventdev *dev)
4352 {
4353         struct dlb2_eventdev *dlb2 = dlb2_pmd_priv(dev);
4354
4355         rte_spinlock_lock(&dlb2->qm_instance.resource_lock);
4356
4357         if (dlb2->run_state == DLB2_RUN_STATE_STOPPED) {
4358                 DLB2_LOG_DBG("Internal error: already stopped\n");
4359                 rte_spinlock_unlock(&dlb2->qm_instance.resource_lock);
4360                 return;
4361         } else if (dlb2->run_state != DLB2_RUN_STATE_STARTED) {
4362                 DLB2_LOG_ERR("Internal error: bad state %d for dev_stop\n",
4363                              (int)dlb2->run_state);
4364                 rte_spinlock_unlock(&dlb2->qm_instance.resource_lock);
4365                 return;
4366         }
4367
4368         dlb2->run_state = DLB2_RUN_STATE_STOPPING;
4369
4370         rte_spinlock_unlock(&dlb2->qm_instance.resource_lock);
4371
4372         dlb2_drain(dev);
4373
4374         dlb2->run_state = DLB2_RUN_STATE_STOPPED;
4375 }
4376
4377 static int
4378 dlb2_eventdev_close(struct rte_eventdev *dev)
4379 {
4380         dlb2_hw_reset_sched_domain(dev, false);
4381
4382         return 0;
4383 }
4384
4385 static void
4386 dlb2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
4387 {
4388         RTE_SET_USED(dev);
4389         RTE_SET_USED(id);
4390
4391         /* This function intentionally left blank. */
4392 }
4393
4394 static void
4395 dlb2_eventdev_port_release(void *port)
4396 {
4397         struct dlb2_eventdev_port *ev_port = port;
4398         struct dlb2_port *qm_port;
4399
4400         if (ev_port) {
4401                 qm_port = &ev_port->qm_port;
4402                 if (qm_port->config_state == DLB2_CONFIGURED)
4403                         dlb2_free_qe_mem(qm_port);
4404         }
4405 }
4406
4407 static int
4408 dlb2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
4409                             uint64_t *timeout_ticks)
4410 {
4411         RTE_SET_USED(dev);
4412         uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
4413
4414         *timeout_ticks = ns * cycles_per_ns;
4415
4416         return 0;
4417 }
4418
4419 static void
4420 dlb2_entry_points_init(struct rte_eventdev *dev)
4421 {
4422         struct dlb2_eventdev *dlb2;
4423
4424         /* Expose PMD's eventdev interface */
4425         static struct eventdev_ops dlb2_eventdev_entry_ops = {
4426                 .dev_infos_get    = dlb2_eventdev_info_get,
4427                 .dev_configure    = dlb2_eventdev_configure,
4428                 .dev_start        = dlb2_eventdev_start,
4429                 .dev_stop         = dlb2_eventdev_stop,
4430                 .dev_close        = dlb2_eventdev_close,
4431                 .queue_def_conf   = dlb2_eventdev_queue_default_conf_get,
4432                 .queue_setup      = dlb2_eventdev_queue_setup,
4433                 .queue_release    = dlb2_eventdev_queue_release,
4434                 .port_def_conf    = dlb2_eventdev_port_default_conf_get,
4435                 .port_setup       = dlb2_eventdev_port_setup,
4436                 .port_release     = dlb2_eventdev_port_release,
4437                 .port_link        = dlb2_eventdev_port_link,
4438                 .port_unlink      = dlb2_eventdev_port_unlink,
4439                 .port_unlinks_in_progress =
4440                                     dlb2_eventdev_port_unlinks_in_progress,
4441                 .timeout_ticks    = dlb2_eventdev_timeout_ticks,
4442                 .dump             = dlb2_eventdev_dump,
4443                 .xstats_get       = dlb2_eventdev_xstats_get,
4444                 .xstats_get_names = dlb2_eventdev_xstats_get_names,
4445                 .xstats_get_by_name = dlb2_eventdev_xstats_get_by_name,
4446                 .xstats_reset       = dlb2_eventdev_xstats_reset,
4447                 .dev_selftest     = test_dlb2_eventdev,
4448         };
4449
4450         /* Expose PMD's eventdev interface */
4451
4452         dev->dev_ops = &dlb2_eventdev_entry_ops;
4453         dev->enqueue = dlb2_event_enqueue;
4454         dev->enqueue_burst = dlb2_event_enqueue_burst;
4455         dev->enqueue_new_burst = dlb2_event_enqueue_new_burst;
4456         dev->enqueue_forward_burst = dlb2_event_enqueue_forward_burst;
4457
4458         dlb2 = dev->data->dev_private;
4459         if (dlb2->poll_mode == DLB2_CQ_POLL_MODE_SPARSE) {
4460                 dev->dequeue = dlb2_event_dequeue_sparse;
4461                 dev->dequeue_burst = dlb2_event_dequeue_burst_sparse;
4462         } else {
4463                 dev->dequeue = dlb2_event_dequeue;
4464                 dev->dequeue_burst = dlb2_event_dequeue_burst;
4465         }
4466 }
4467
4468 int
4469 dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
4470                             const char *name,
4471                             struct dlb2_devargs *dlb2_args)
4472 {
4473         struct dlb2_eventdev *dlb2;
4474         int err, i;
4475
4476         dlb2 = dev->data->dev_private;
4477
4478         dlb2->event_dev = dev; /* backlink */
4479
4480         evdev_dlb2_default_info.driver_name = name;
4481
4482         dlb2->max_num_events_override = dlb2_args->max_num_events;
4483         dlb2->num_dir_credits_override = dlb2_args->num_dir_credits_override;
4484         dlb2->qm_instance.cos_id = dlb2_args->cos_id;
4485         dlb2->poll_interval = dlb2_args->poll_interval;
4486         dlb2->sw_credit_quanta = dlb2_args->sw_credit_quanta;
4487         dlb2->hw_credit_quanta = dlb2_args->hw_credit_quanta;
4488         dlb2->default_depth_thresh = dlb2_args->default_depth_thresh;
4489         dlb2->vector_opts_enabled = dlb2_args->vector_opts_enabled;
4490
4491         err = dlb2_iface_open(&dlb2->qm_instance, name);
4492         if (err < 0) {
4493                 DLB2_LOG_ERR("could not open event hardware device, err=%d\n",
4494                              err);
4495                 return err;
4496         }
4497
4498         err = dlb2_iface_get_device_version(&dlb2->qm_instance,
4499                                             &dlb2->revision);
4500         if (err < 0) {
4501                 DLB2_LOG_ERR("dlb2: failed to get the device version, err=%d\n",
4502                              err);
4503                 return err;
4504         }
4505
4506         err = dlb2_hw_query_resources(dlb2);
4507         if (err) {
4508                 DLB2_LOG_ERR("get resources err=%d for %s\n",
4509                              err, name);
4510                 return err;
4511         }
4512
4513         dlb2_iface_hardware_init(&dlb2->qm_instance);
4514
4515         err = dlb2_iface_get_cq_poll_mode(&dlb2->qm_instance, &dlb2->poll_mode);
4516         if (err < 0) {
4517                 DLB2_LOG_ERR("dlb2: failed to get the poll mode, err=%d\n",
4518                              err);
4519                 return err;
4520         }
4521
4522         /* Complete xtstats runtime initialization */
4523         err = dlb2_xstats_init(dlb2);
4524         if (err) {
4525                 DLB2_LOG_ERR("dlb2: failed to init xstats, err=%d\n", err);
4526                 return err;
4527         }
4528
4529         /* Initialize each port's token pop mode */
4530         for (i = 0; i < DLB2_MAX_NUM_PORTS(dlb2->version); i++)
4531                 dlb2->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
4532
4533         rte_spinlock_init(&dlb2->qm_instance.resource_lock);
4534
4535         dlb2_iface_low_level_io_init();
4536
4537         dlb2_entry_points_init(dev);
4538
4539         dlb2_init_queue_depth_thresholds(dlb2,
4540                                          dlb2_args->qid_depth_thresholds.val);
4541
4542         return 0;
4543 }
4544
4545 int
4546 dlb2_secondary_eventdev_probe(struct rte_eventdev *dev,
4547                               const char *name)
4548 {
4549         struct dlb2_eventdev *dlb2;
4550         int err;
4551
4552         dlb2 = dev->data->dev_private;
4553
4554         evdev_dlb2_default_info.driver_name = name;
4555
4556         err = dlb2_iface_open(&dlb2->qm_instance, name);
4557         if (err < 0) {
4558                 DLB2_LOG_ERR("could not open event hardware device, err=%d\n",
4559                              err);
4560                 return err;
4561         }
4562
4563         err = dlb2_hw_query_resources(dlb2);
4564         if (err) {
4565                 DLB2_LOG_ERR("get resources err=%d for %s\n",
4566                              err, name);
4567                 return err;
4568         }
4569
4570         dlb2_iface_low_level_io_init();
4571
4572         dlb2_entry_points_init(dev);
4573
4574         return 0;
4575 }
4576
4577 int
4578 dlb2_parse_params(const char *params,
4579                   const char *name,
4580                   struct dlb2_devargs *dlb2_args,
4581                   uint8_t version)
4582 {
4583         int ret = 0;
4584         static const char * const args[] = { NUMA_NODE_ARG,
4585                                              DLB2_MAX_NUM_EVENTS,
4586                                              DLB2_NUM_DIR_CREDITS,
4587                                              DEV_ID_ARG,
4588                                              DLB2_QID_DEPTH_THRESH_ARG,
4589                                              DLB2_COS_ARG,
4590                                              DLB2_POLL_INTERVAL_ARG,
4591                                              DLB2_SW_CREDIT_QUANTA_ARG,
4592                                              DLB2_HW_CREDIT_QUANTA_ARG,
4593                                              DLB2_DEPTH_THRESH_ARG,
4594                                              DLB2_VECTOR_OPTS_ENAB_ARG,
4595                                              NULL };
4596
4597         if (params != NULL && params[0] != '\0') {
4598                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
4599
4600                 if (kvlist == NULL) {
4601                         RTE_LOG(INFO, PMD,
4602                                 "Ignoring unsupported parameters when creating device '%s'\n",
4603                                 name);
4604                 } else {
4605                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
4606                                                      set_numa_node,
4607                                                      &dlb2_args->socket_id);
4608                         if (ret != 0) {
4609                                 DLB2_LOG_ERR("%s: Error parsing numa node parameter",
4610                                              name);
4611                                 rte_kvargs_free(kvlist);
4612                                 return ret;
4613                         }
4614
4615                         ret = rte_kvargs_process(kvlist, DLB2_MAX_NUM_EVENTS,
4616                                                  set_max_num_events,
4617                                                  &dlb2_args->max_num_events);
4618                         if (ret != 0) {
4619                                 DLB2_LOG_ERR("%s: Error parsing max_num_events parameter",
4620                                              name);
4621                                 rte_kvargs_free(kvlist);
4622                                 return ret;
4623                         }
4624
4625                         if (version == DLB2_HW_V2) {
4626                                 ret = rte_kvargs_process(kvlist,
4627                                         DLB2_NUM_DIR_CREDITS,
4628                                         set_num_dir_credits,
4629                                         &dlb2_args->num_dir_credits_override);
4630                                 if (ret != 0) {
4631                                         DLB2_LOG_ERR("%s: Error parsing num_dir_credits parameter",
4632                                                      name);
4633                                         rte_kvargs_free(kvlist);
4634                                         return ret;
4635                                 }
4636                         }
4637                         ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
4638                                                  set_dev_id,
4639                                                  &dlb2_args->dev_id);
4640                         if (ret != 0) {
4641                                 DLB2_LOG_ERR("%s: Error parsing dev_id parameter",
4642                                              name);
4643                                 rte_kvargs_free(kvlist);
4644                                 return ret;
4645                         }
4646
4647                         if (version == DLB2_HW_V2) {
4648                                 ret = rte_kvargs_process(
4649                                         kvlist,
4650                                         DLB2_QID_DEPTH_THRESH_ARG,
4651                                         set_qid_depth_thresh,
4652                                         &dlb2_args->qid_depth_thresholds);
4653                         } else {
4654                                 ret = rte_kvargs_process(
4655                                         kvlist,
4656                                         DLB2_QID_DEPTH_THRESH_ARG,
4657                                         set_qid_depth_thresh_v2_5,
4658                                         &dlb2_args->qid_depth_thresholds);
4659                         }
4660                         if (ret != 0) {
4661                                 DLB2_LOG_ERR("%s: Error parsing qid_depth_thresh parameter",
4662                                              name);
4663                                 rte_kvargs_free(kvlist);
4664                                 return ret;
4665                         }
4666
4667                         ret = rte_kvargs_process(kvlist, DLB2_COS_ARG,
4668                                                  set_cos,
4669                                                  &dlb2_args->cos_id);
4670                         if (ret != 0) {
4671                                 DLB2_LOG_ERR("%s: Error parsing cos parameter",
4672                                              name);
4673                                 rte_kvargs_free(kvlist);
4674                                 return ret;
4675                         }
4676
4677                         ret = rte_kvargs_process(kvlist, DLB2_POLL_INTERVAL_ARG,
4678                                                  set_poll_interval,
4679                                                  &dlb2_args->poll_interval);
4680                         if (ret != 0) {
4681                                 DLB2_LOG_ERR("%s: Error parsing poll interval parameter",
4682                                              name);
4683                                 rte_kvargs_free(kvlist);
4684                                 return ret;
4685                         }
4686
4687                         ret = rte_kvargs_process(kvlist,
4688                                                  DLB2_SW_CREDIT_QUANTA_ARG,
4689                                                  set_sw_credit_quanta,
4690                                                  &dlb2_args->sw_credit_quanta);
4691                         if (ret != 0) {
4692                                 DLB2_LOG_ERR("%s: Error parsing sw credit quanta parameter",
4693                                              name);
4694                                 rte_kvargs_free(kvlist);
4695                                 return ret;
4696                         }
4697
4698                         ret = rte_kvargs_process(kvlist,
4699                                                  DLB2_HW_CREDIT_QUANTA_ARG,
4700                                                  set_hw_credit_quanta,
4701                                                  &dlb2_args->hw_credit_quanta);
4702                         if (ret != 0) {
4703                                 DLB2_LOG_ERR("%s: Error parsing hw credit quanta parameter",
4704                                              name);
4705                                 rte_kvargs_free(kvlist);
4706                                 return ret;
4707                         }
4708
4709                         ret = rte_kvargs_process(kvlist, DLB2_DEPTH_THRESH_ARG,
4710                                         set_default_depth_thresh,
4711                                         &dlb2_args->default_depth_thresh);
4712                         if (ret != 0) {
4713                                 DLB2_LOG_ERR("%s: Error parsing set depth thresh parameter",
4714                                              name);
4715                                 rte_kvargs_free(kvlist);
4716                                 return ret;
4717                         }
4718
4719                         ret = rte_kvargs_process(kvlist,
4720                                         DLB2_VECTOR_OPTS_ENAB_ARG,
4721                                         set_vector_opts_enab,
4722                                         &dlb2_args->vector_opts_enabled);
4723                         if (ret != 0) {
4724                                 DLB2_LOG_ERR("%s: Error parsing vector opts enabled",
4725                                              name);
4726                                 rte_kvargs_free(kvlist);
4727                                 return ret;
4728                         }
4729
4730                         rte_kvargs_free(kvlist);
4731                 }
4732         }
4733         return ret;
4734 }
4735 RTE_LOG_REGISTER_DEFAULT(eventdev_dlb2_log_level, NOTICE);