drivers/event/sw/sw_evdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <inttypes.h>
   6 #include <string.h>
   7
   8 #include <rte_bus_vdev.h>
   9 #include <rte_kvargs.h>
  10 #include <rte_ring.h>
  11 #include <rte_errno.h>
  12 #include <rte_event_ring.h>
  13 #include <rte_service_component.h>
  14
  15 #include "sw_evdev.h"
  16 #include "iq_ring.h"
  17
  18 #define EVENTDEV_NAME_SW_PMD event_sw
  19 #define NUMA_NODE_ARG "numa_node"
  20 #define SCHED_QUANTA_ARG "sched_quanta"
  21 #define CREDIT_QUANTA_ARG "credit_quanta"
  22
  23 static void
  24 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info);
  25
  26 static int
  27 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
  28                 const uint8_t priorities[], uint16_t num)
  29 {
  30         struct sw_port *p = port;
  31         struct sw_evdev *sw = sw_pmd_priv(dev);
  32         int i;
  33
  34         RTE_SET_USED(priorities);
  35         for (i = 0; i < num; i++) {
  36                 struct sw_qid *q = &sw->qids[queues[i]];
  37                 unsigned int j;
  38
  39                 /* check for qid map overflow */
  40                 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) {
  41                         rte_errno = -EDQUOT;
  42                         break;
  43                 }
  44
  45                 if (p->is_directed && p->num_qids_mapped > 0) {
  46                         rte_errno = -EDQUOT;
  47                         break;
  48                 }
  49
  50                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
  51                         if (q->cq_map[j] == p->id)
  52                                 break;
  53                 }
  54
  55                 /* check if port is already linked */
  56                 if (j < q->cq_num_mapped_cqs)
  57                         continue;
  58
  59                 if (q->type == SW_SCHED_TYPE_DIRECT) {
  60                         /* check directed qids only map to one port */
  61                         if (p->num_qids_mapped > 0) {
  62                                 rte_errno = -EDQUOT;
  63                                 break;
  64                         }
  65                         /* check port only takes a directed flow */
  66                         if (num > 1) {
  67                                 rte_errno = -EDQUOT;
  68                                 break;
  69                         }
  70
  71                         p->is_directed = 1;
  72                         p->num_qids_mapped = 1;
  73                 } else if (q->type == RTE_SCHED_TYPE_ORDERED) {
  74                         p->num_ordered_qids++;
  75                         p->num_qids_mapped++;
  76                 } else if (q->type == RTE_SCHED_TYPE_ATOMIC ||
  77                                 q->type == RTE_SCHED_TYPE_PARALLEL) {
  78                         p->num_qids_mapped++;
  79                 }
  80
  81                 q->cq_map[q->cq_num_mapped_cqs] = p->id;
  82                 rte_smp_wmb();
  83                 q->cq_num_mapped_cqs++;
  84         }
  85         return i;
  86 }
  87
  88 static int
  89 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
  90                 uint16_t nb_unlinks)
  91 {
  92         struct sw_port *p = port;
  93         struct sw_evdev *sw = sw_pmd_priv(dev);
  94         unsigned int i, j;
  95
  96         int unlinked = 0;
  97         for (i = 0; i < nb_unlinks; i++) {
  98                 struct sw_qid *q = &sw->qids[queues[i]];
  99                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
 100                         if (q->cq_map[j] == p->id) {
 101                                 q->cq_map[j] =
 102                                         q->cq_map[q->cq_num_mapped_cqs - 1];
 103                                 rte_smp_wmb();
 104                                 q->cq_num_mapped_cqs--;
 105                                 unlinked++;
 106
 107                                 p->num_qids_mapped--;
 108
 109                                 if (q->type == RTE_SCHED_TYPE_ORDERED)
 110                                         p->num_ordered_qids--;
 111
 112                                 continue;
 113                         }
 114                 }
 115         }
 116         return unlinked;
 117 }
 118
 119 static int
 120 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 121                 const struct rte_event_port_conf *conf)
 122 {
 123         struct sw_evdev *sw = sw_pmd_priv(dev);
 124         struct sw_port *p = &sw->ports[port_id];
 125         char buf[RTE_RING_NAMESIZE];
 126         unsigned int i;
 127
 128         struct rte_event_dev_info info;
 129         sw_info_get(dev, &info);
 130
 131         /* detect re-configuring and return credits to instance if needed */
 132         if (p->initialized) {
 133                 /* taking credits from pool is done one quanta at a time, and
 134                  * credits may be spend (counted in p->inflights) or still
 135                  * available in the port (p->inflight_credits). We must return
 136                  * the sum to no leak credits
 137                  */
 138                 int possible_inflights = p->inflight_credits + p->inflights;
 139                 rte_atomic32_sub(&sw->inflights, possible_inflights);
 140         }
 141
 142         *p = (struct sw_port){0}; /* zero entire structure */
 143         p->id = port_id;
 144         p->sw = sw;
 145
 146         /* check to see if rings exists - port_setup() can be called multiple
 147          * times legally (assuming device is stopped). If ring exists, free it
 148          * to so it gets re-created with the correct size
 149          */
 150         snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id,
 151                         port_id, "rx_worker_ring");
 152         struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf);
 153         if (existing_ring)
 154                 rte_event_ring_free(existing_ring);
 155
 156         p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH,
 157                         dev->data->socket_id,
 158                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
 159         if (p->rx_worker_ring == NULL) {
 160                 SW_LOG_ERR("Error creating RX worker ring for port %d\n",
 161                                 port_id);
 162                 return -1;
 163         }
 164
 165         p->inflight_max = conf->new_event_threshold;
 166
 167         /* check if ring exists, same as rx_worker above */
 168         snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
 169                         port_id, "cq_worker_ring");
 170         existing_ring = rte_event_ring_lookup(buf);
 171         if (existing_ring)
 172                 rte_event_ring_free(existing_ring);
 173
 174         p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth,
 175                         dev->data->socket_id,
 176                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
 177         if (p->cq_worker_ring == NULL) {
 178                 rte_event_ring_free(p->rx_worker_ring);
 179                 SW_LOG_ERR("Error creating CQ worker ring for port %d\n",
 180                                 port_id);
 181                 return -1;
 182         }
 183         sw->cq_ring_space[port_id] = conf->dequeue_depth;
 184
 185         /* set hist list contents to empty */
 186         for (i = 0; i < SW_PORT_HIST_LIST; i++) {
 187                 p->hist_list[i].fid = -1;
 188                 p->hist_list[i].qid = -1;
 189         }
 190         dev->data->ports[port_id] = p;
 191
 192         rte_smp_wmb();
 193         p->initialized = 1;
 194         return 0;
 195 }
 196
 197 static void
 198 sw_port_release(void *port)
 199 {
 200         struct sw_port *p = (void *)port;
 201         if (p == NULL)
 202                 return;
 203
 204         rte_event_ring_free(p->rx_worker_ring);
 205         rte_event_ring_free(p->cq_worker_ring);
 206         memset(p, 0, sizeof(*p));
 207 }
 208
 209 static int32_t
 210 qid_init(struct sw_evdev *sw, unsigned int idx, int type,
 211                 const struct rte_event_queue_conf *queue_conf)
 212 {
 213         unsigned int i;
 214         int dev_id = sw->data->dev_id;
 215         int socket_id = sw->data->socket_id;
 216         char buf[IQ_RING_NAMESIZE];
 217         struct sw_qid *qid = &sw->qids[idx];
 218
 219         for (i = 0; i < SW_IQS_MAX; i++) {
 220                 snprintf(buf, sizeof(buf), "q_%u_iq_%d", idx, i);
 221                 qid->iq[i] = iq_ring_create(buf, socket_id);
 222                 if (!qid->iq[i]) {
 223                         SW_LOG_DBG("ring create failed");
 224                         goto cleanup;
 225                 }
 226         }
 227
 228         /* Initialize the FID structures to no pinning (-1), and zero packets */
 229         const struct sw_fid_t fid = {.cq = -1, .pcount = 0};
 230         for (i = 0; i < RTE_DIM(qid->fids); i++)
 231                 qid->fids[i] = fid;
 232
 233         qid->id = idx;
 234         qid->type = type;
 235         qid->priority = queue_conf->priority;
 236
 237         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 238                 char ring_name[RTE_RING_NAMESIZE];
 239                 uint32_t window_size;
 240
 241                 /* rte_ring and window_size_mask require require window_size to
 242                  * be a power-of-2.
 243                  */
 244                 window_size = rte_align32pow2(
 245                                 queue_conf->nb_atomic_order_sequences);
 246
 247                 qid->window_size = window_size - 1;
 248
 249                 if (!window_size) {
 250                         SW_LOG_DBG(
 251                                 "invalid reorder_window_size for ordered queue\n"
 252                                 );
 253                         goto cleanup;
 254                 }
 255
 256                 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i);
 257                 qid->reorder_buffer = rte_zmalloc_socket(buf,
 258                                 window_size * sizeof(qid->reorder_buffer[0]),
 259                                 0, socket_id);
 260                 if (!qid->reorder_buffer) {
 261                         SW_LOG_DBG("reorder_buffer malloc failed\n");
 262                         goto cleanup;
 263                 }
 264
 265                 memset(&qid->reorder_buffer[0],
 266                        0,
 267                        window_size * sizeof(qid->reorder_buffer[0]));
 268
 269                 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist",
 270                                 dev_id, idx);
 271
 272                 /* lookup the ring, and if it already exists, free it */
 273                 struct rte_ring *cleanup = rte_ring_lookup(ring_name);
 274                 if (cleanup)
 275                         rte_ring_free(cleanup);
 276
 277                 qid->reorder_buffer_freelist = rte_ring_create(ring_name,
 278                                 window_size,
 279                                 socket_id,
 280                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 281                 if (!qid->reorder_buffer_freelist) {
 282                         SW_LOG_DBG("freelist ring create failed");
 283                         goto cleanup;
 284                 }
 285
 286                 /* Populate the freelist with reorder buffer entries. Enqueue
 287                  * 'window_size - 1' entries because the rte_ring holds only
 288                  * that many.
 289                  */
 290                 for (i = 0; i < window_size - 1; i++) {
 291                         if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist,
 292                                                 &qid->reorder_buffer[i]) < 0)
 293                                 goto cleanup;
 294                 }
 295
 296                 qid->reorder_buffer_index = 0;
 297                 qid->cq_next_tx = 0;
 298         }
 299
 300         qid->initialized = 1;
 301
 302         return 0;
 303
 304 cleanup:
 305         for (i = 0; i < SW_IQS_MAX; i++) {
 306                 if (qid->iq[i])
 307                         iq_ring_destroy(qid->iq[i]);
 308         }
 309
 310         if (qid->reorder_buffer) {
 311                 rte_free(qid->reorder_buffer);
 312                 qid->reorder_buffer = NULL;
 313         }
 314
 315         if (qid->reorder_buffer_freelist) {
 316                 rte_ring_free(qid->reorder_buffer_freelist);
 317                 qid->reorder_buffer_freelist = NULL;
 318         }
 319
 320         return -EINVAL;
 321 }
 322
 323 static void
 324 sw_queue_release(struct rte_eventdev *dev, uint8_t id)
 325 {
 326         struct sw_evdev *sw = sw_pmd_priv(dev);
 327         struct sw_qid *qid = &sw->qids[id];
 328         uint32_t i;
 329
 330         for (i = 0; i < SW_IQS_MAX; i++)
 331                 iq_ring_destroy(qid->iq[i]);
 332
 333         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 334                 rte_free(qid->reorder_buffer);
 335                 rte_ring_free(qid->reorder_buffer_freelist);
 336         }
 337         memset(qid, 0, sizeof(*qid));
 338 }
 339
 340 static int
 341 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
 342                 const struct rte_event_queue_conf *conf)
 343 {
 344         int type;
 345
 346         type = conf->schedule_type;
 347
 348         if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) {
 349                 type = SW_SCHED_TYPE_DIRECT;
 350         } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES
 351                         & conf->event_queue_cfg) {
 352                 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n");
 353                 return -ENOTSUP;
 354         }
 355
 356         struct sw_evdev *sw = sw_pmd_priv(dev);
 357
 358         if (sw->qids[queue_id].initialized)
 359                 sw_queue_release(dev, queue_id);
 360
 361         return qid_init(sw, queue_id, type, conf);
 362 }
 363
 364 static void
 365 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
 366                                  struct rte_event_queue_conf *conf)
 367 {
 368         RTE_SET_USED(dev);
 369         RTE_SET_USED(queue_id);
 370
 371         static const struct rte_event_queue_conf default_conf = {
 372                 .nb_atomic_flows = 4096,
 373                 .nb_atomic_order_sequences = 1,
 374                 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
 375                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 376         };
 377
 378         *conf = default_conf;
 379 }
 380
 381 static void
 382 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 383                  struct rte_event_port_conf *port_conf)
 384 {
 385         RTE_SET_USED(dev);
 386         RTE_SET_USED(port_id);
 387
 388         port_conf->new_event_threshold = 1024;
 389         port_conf->dequeue_depth = 16;
 390         port_conf->enqueue_depth = 16;
 391 }
 392
 393 static int
 394 sw_dev_configure(const struct rte_eventdev *dev)
 395 {
 396         struct sw_evdev *sw = sw_pmd_priv(dev);
 397         const struct rte_eventdev_data *data = dev->data;
 398         const struct rte_event_dev_config *conf = &data->dev_conf;
 399
 400         sw->qid_count = conf->nb_event_queues;
 401         sw->port_count = conf->nb_event_ports;
 402         sw->nb_events_limit = conf->nb_events_limit;
 403         rte_atomic32_set(&sw->inflights, 0);
 404
 405         if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
 406                 return -ENOTSUP;
 407
 408         return 0;
 409 }
 410
 411 struct rte_eth_dev;
 412
 413 static int
 414 sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev,
 415                         const struct rte_eth_dev *eth_dev,
 416                         uint32_t *caps)
 417 {
 418         RTE_SET_USED(dev);
 419         RTE_SET_USED(eth_dev);
 420         *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
 421         return 0;
 422 }
 423
 424 static void
 425 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info)
 426 {
 427         RTE_SET_USED(dev);
 428
 429         static const struct rte_event_dev_info evdev_sw_info = {
 430                         .driver_name = SW_PMD_NAME,
 431                         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
 432                         .max_event_queue_flows = SW_QID_NUM_FIDS,
 433                         .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX,
 434                         .max_event_priority_levels = SW_IQS_MAX,
 435                         .max_event_ports = SW_PORTS_MAX,
 436                         .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH,
 437                         .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH,
 438                         .max_num_events = SW_INFLIGHT_EVENTS_TOTAL,
 439                         .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
 440                                         RTE_EVENT_DEV_CAP_BURST_MODE |
 441                                         RTE_EVENT_DEV_CAP_EVENT_QOS),
 442         };
 443
 444         *info = evdev_sw_info;
 445 }
 446
 447 static void
 448 sw_dump(struct rte_eventdev *dev, FILE *f)
 449 {
 450         const struct sw_evdev *sw = sw_pmd_priv(dev);
 451
 452         static const char * const q_type_strings[] = {
 453                         "Ordered", "Atomic", "Parallel", "Directed"
 454         };
 455         uint32_t i;
 456         fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name",
 457                         sw->port_count, sw->qid_count);
 458
 459         fprintf(f, "\trx   %"PRIu64"\n\tdrop %"PRIu64"\n\ttx   %"PRIu64"\n",
 460                 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts);
 461         fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called);
 462         fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called);
 463         fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues);
 464         fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues);
 465         uint32_t inflights = rte_atomic32_read(&sw->inflights);
 466         uint32_t credits = sw->nb_events_limit - inflights;
 467         fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits);
 468
 469 #define COL_RED "\x1b[31m"
 470 #define COL_RESET "\x1b[0m"
 471
 472         for (i = 0; i < sw->port_count; i++) {
 473                 int max, j;
 474                 const struct sw_port *p = &sw->ports[i];
 475                 if (!p->initialized) {
 476                         fprintf(f, "  %sPort %d not initialized.%s\n",
 477                                 COL_RED, i, COL_RESET);
 478                         continue;
 479                 }
 480                 fprintf(f, "  Port %d %s\n", i,
 481                         p->is_directed ? " (SingleCons)" : "");
 482                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64
 483                         "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts,
 484                         sw->ports[i].stats.rx_dropped,
 485                         sw->ports[i].stats.tx_pkts,
 486                         (p->inflights == p->inflight_max) ?
 487                                 COL_RED : COL_RESET,
 488                         sw->ports[i].inflights, COL_RESET);
 489
 490                 fprintf(f, "\tMax New: %u"
 491                         "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n",
 492                         sw->ports[i].inflight_max,
 493                         sw->ports[i].avg_pkt_ticks,
 494                         sw->ports[i].inflight_credits);
 495                 fprintf(f, "\tReceive burst distribution:\n");
 496                 float zp_percent = p->zero_polls * 100.0 / p->total_polls;
 497                 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ",
 498                                 zp_percent);
 499                 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;)
 500                         if (p->poll_buckets[max] != 0)
 501                                 break;
 502                 for (j = 0; j <= max; j++) {
 503                         if (p->poll_buckets[j] != 0) {
 504                                 float poll_pc = p->poll_buckets[j] * 100.0 /
 505                                         p->total_polls;
 506                                 fprintf(f, "%u-%u:%.02f%% ",
 507                                         ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1),
 508                                         ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT),
 509                                         poll_pc);
 510                         }
 511                 }
 512                 fprintf(f, "\n");
 513
 514                 if (p->rx_worker_ring) {
 515                         uint64_t used = rte_event_ring_count(p->rx_worker_ring);
 516                         uint64_t space = rte_event_ring_free_count(
 517                                         p->rx_worker_ring);
 518                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 519                         fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4"
 520                                         PRIu64 COL_RESET"\n", col, used, space);
 521                 } else
 522                         fprintf(f, "\trx ring not initialized.\n");
 523
 524                 if (p->cq_worker_ring) {
 525                         uint64_t used = rte_event_ring_count(p->cq_worker_ring);
 526                         uint64_t space = rte_event_ring_free_count(
 527                                         p->cq_worker_ring);
 528                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 529                         fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4"
 530                                         PRIu64 COL_RESET"\n", col, used, space);
 531                 } else
 532                         fprintf(f, "\tcq ring not initialized.\n");
 533         }
 534
 535         for (i = 0; i < sw->qid_count; i++) {
 536                 const struct sw_qid *qid = &sw->qids[i];
 537                 if (!qid->initialized) {
 538                         fprintf(f, "  %sQueue %d not initialized.%s\n",
 539                                 COL_RED, i, COL_RESET);
 540                         continue;
 541                 }
 542                 int affinities_per_port[SW_PORTS_MAX] = {0};
 543                 uint32_t inflights = 0;
 544
 545                 fprintf(f, "  Queue %d (%s)\n", i, q_type_strings[qid->type]);
 546                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64"\n",
 547                         qid->stats.rx_pkts, qid->stats.rx_dropped,
 548                         qid->stats.tx_pkts);
 549                 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 550                         struct rte_ring *rob_buf_free =
 551                                 qid->reorder_buffer_freelist;
 552                         if (rob_buf_free)
 553                                 fprintf(f, "\tReorder entries in use: %u\n",
 554                                         rte_ring_free_count(rob_buf_free));
 555                         else
 556                                 fprintf(f,
 557                                         "\tReorder buffer not initialized\n");
 558                 }
 559
 560                 uint32_t flow;
 561                 for (flow = 0; flow < RTE_DIM(qid->fids); flow++)
 562                         if (qid->fids[flow].cq != -1) {
 563                                 affinities_per_port[qid->fids[flow].cq]++;
 564                                 inflights += qid->fids[flow].pcount;
 565                         }
 566
 567                 uint32_t port;
 568                 fprintf(f, "\tPer Port Stats:\n");
 569                 for (port = 0; port < sw->port_count; port++) {
 570                         fprintf(f, "\t  Port %d: Pkts: %"PRIu64, port,
 571                                         qid->to_port[port]);
 572                         fprintf(f, "\tFlows: %d\n", affinities_per_port[port]);
 573                 }
 574
 575                 uint32_t iq;
 576                 uint32_t iq_printed = 0;
 577                 for (iq = 0; iq < SW_IQS_MAX; iq++) {
 578                         if (!qid->iq[iq]) {
 579                                 fprintf(f, "\tiq %d is not initialized.\n", iq);
 580                                 iq_printed = 1;
 581                                 continue;
 582                         }
 583                         uint32_t used = iq_ring_count(qid->iq[iq]);
 584                         uint32_t free = iq_ring_free_count(qid->iq[iq]);
 585                         const char *col = (free == 0) ? COL_RED : COL_RESET;
 586                         if (used > 0) {
 587                                 fprintf(f, "\t%siq %d: Used %d\tFree %d"
 588                                         COL_RESET"\n", col, iq, used, free);
 589                                 iq_printed = 1;
 590                         }
 591                 }
 592                 if (iq_printed == 0)
 593                         fprintf(f, "\t-- iqs empty --\n");
 594         }
 595 }
 596
 597 static int
 598 sw_start(struct rte_eventdev *dev)
 599 {
 600         unsigned int i, j;
 601         struct sw_evdev *sw = sw_pmd_priv(dev);
 602
 603         rte_service_component_runstate_set(sw->service_id, 1);
 604
 605         /* check a service core is mapped to this service */
 606         if (!rte_service_runstate_get(sw->service_id)) {
 607                 SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
 608                                 sw->service_name);
 609                 return -ENOENT;
 610         }
 611
 612         /* check all ports are set up */
 613         for (i = 0; i < sw->port_count; i++)
 614                 if (sw->ports[i].rx_worker_ring == NULL) {
 615                         SW_LOG_ERR("Port %d not configured\n", i);
 616                         return -ESTALE;
 617                 }
 618
 619         /* check all queues are configured and mapped to ports*/
 620         for (i = 0; i < sw->qid_count; i++)
 621                 if (sw->qids[i].iq[0] == NULL ||
 622                                 sw->qids[i].cq_num_mapped_cqs == 0) {
 623                         SW_LOG_ERR("Queue %d not configured\n", i);
 624                         return -ENOLINK;
 625                 }
 626
 627         /* build up our prioritized array of qids */
 628         /* We don't use qsort here, as if all/multiple entries have the same
 629          * priority, the result is non-deterministic. From "man 3 qsort":
 630          * "If two members compare as equal, their order in the sorted
 631          * array is undefined."
 632          */
 633         uint32_t qidx = 0;
 634         for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) {
 635                 for (i = 0; i < sw->qid_count; i++) {
 636                         if (sw->qids[i].priority == j) {
 637                                 sw->qids_prioritized[qidx] = &sw->qids[i];
 638                                 qidx++;
 639                         }
 640                 }
 641         }
 642
 643         if (sw_xstats_init(sw) < 0)
 644                 return -EINVAL;
 645
 646         rte_smp_wmb();
 647         sw->started = 1;
 648
 649         return 0;
 650 }
 651
 652 static void
 653 sw_stop(struct rte_eventdev *dev)
 654 {
 655         struct sw_evdev *sw = sw_pmd_priv(dev);
 656         sw_xstats_uninit(sw);
 657         sw->started = 0;
 658         rte_smp_wmb();
 659 }
 660
 661 static int
 662 sw_close(struct rte_eventdev *dev)
 663 {
 664         struct sw_evdev *sw = sw_pmd_priv(dev);
 665         uint32_t i;
 666
 667         for (i = 0; i < sw->qid_count; i++)
 668                 sw_queue_release(dev, i);
 669         sw->qid_count = 0;
 670
 671         for (i = 0; i < sw->port_count; i++)
 672                 sw_port_release(&sw->ports[i]);
 673         sw->port_count = 0;
 674
 675         memset(&sw->stats, 0, sizeof(sw->stats));
 676         sw->sched_called = 0;
 677         sw->sched_no_iq_enqueues = 0;
 678         sw->sched_no_cq_enqueues = 0;
 679         sw->sched_cq_qid_called = 0;
 680
 681         return 0;
 682 }
 683
 684 static int
 685 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 686 {
 687         int *socket_id = opaque;
 688         *socket_id = atoi(value);
 689         if (*socket_id >= RTE_MAX_NUMA_NODES)
 690                 return -1;
 691         return 0;
 692 }
 693
 694 static int
 695 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque)
 696 {
 697         int *quanta = opaque;
 698         *quanta = atoi(value);
 699         if (*quanta < 0 || *quanta >= 4096)
 700                 return -1;
 701         return 0;
 702 }
 703
 704 static int
 705 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 706 {
 707         int *credit = opaque;
 708         *credit = atoi(value);
 709         if (*credit < 0 || *credit >= 128)
 710                 return -1;
 711         return 0;
 712 }
 713
 714
 715 static int32_t sw_sched_service_func(void *args)
 716 {
 717         struct rte_eventdev *dev = args;
 718         sw_event_schedule(dev);
 719         return 0;
 720 }
 721
 722 static int
 723 sw_probe(struct rte_vdev_device *vdev)
 724 {
 725         static const struct rte_eventdev_ops evdev_sw_ops = {
 726                         .dev_configure = sw_dev_configure,
 727                         .dev_infos_get = sw_info_get,
 728                         .dev_close = sw_close,
 729                         .dev_start = sw_start,
 730                         .dev_stop = sw_stop,
 731                         .dump = sw_dump,
 732
 733                         .queue_def_conf = sw_queue_def_conf,
 734                         .queue_setup = sw_queue_setup,
 735                         .queue_release = sw_queue_release,
 736                         .port_def_conf = sw_port_def_conf,
 737                         .port_setup = sw_port_setup,
 738                         .port_release = sw_port_release,
 739                         .port_link = sw_port_link,
 740                         .port_unlink = sw_port_unlink,
 741
 742                         .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get,
 743
 744                         .xstats_get = sw_xstats_get,
 745                         .xstats_get_names = sw_xstats_get_names,
 746                         .xstats_get_by_name = sw_xstats_get_by_name,
 747                         .xstats_reset = sw_xstats_reset,
 748         };
 749
 750         static const char *const args[] = {
 751                 NUMA_NODE_ARG,
 752                 SCHED_QUANTA_ARG,
 753                 CREDIT_QUANTA_ARG,
 754                 NULL
 755         };
 756         const char *name;
 757         const char *params;
 758         struct rte_eventdev *dev;
 759         struct sw_evdev *sw;
 760         int socket_id = rte_socket_id();
 761         int sched_quanta  = SW_DEFAULT_SCHED_QUANTA;
 762         int credit_quanta = SW_DEFAULT_CREDIT_QUANTA;
 763
 764         name = rte_vdev_device_name(vdev);
 765         params = rte_vdev_device_args(vdev);
 766         if (params != NULL && params[0] != '\0') {
 767                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
 768
 769                 if (!kvlist) {
 770                         SW_LOG_INFO(
 771                                 "Ignoring unsupported parameters when creating device '%s'\n",
 772                                 name);
 773                 } else {
 774                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
 775                                         assign_numa_node, &socket_id);
 776                         if (ret != 0) {
 777                                 SW_LOG_ERR(
 778                                         "%s: Error parsing numa node parameter",
 779                                         name);
 780                                 rte_kvargs_free(kvlist);
 781                                 return ret;
 782                         }
 783
 784                         ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG,
 785                                         set_sched_quanta, &sched_quanta);
 786                         if (ret != 0) {
 787                                 SW_LOG_ERR(
 788                                         "%s: Error parsing sched quanta parameter",
 789                                         name);
 790                                 rte_kvargs_free(kvlist);
 791                                 return ret;
 792                         }
 793
 794                         ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG,
 795                                         set_credit_quanta, &credit_quanta);
 796                         if (ret != 0) {
 797                                 SW_LOG_ERR(
 798                                         "%s: Error parsing credit quanta parameter",
 799                                         name);
 800                                 rte_kvargs_free(kvlist);
 801                                 return ret;
 802                         }
 803
 804                         rte_kvargs_free(kvlist);
 805                 }
 806         }
 807
 808         SW_LOG_INFO(
 809                         "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n",
 810                         name, socket_id, sched_quanta, credit_quanta);
 811
 812         dev = rte_event_pmd_vdev_init(name,
 813                         sizeof(struct sw_evdev), socket_id);
 814         if (dev == NULL) {
 815                 SW_LOG_ERR("eventdev vdev init() failed");
 816                 return -EFAULT;
 817         }
 818         dev->dev_ops = &evdev_sw_ops;
 819         dev->enqueue = sw_event_enqueue;
 820         dev->enqueue_burst = sw_event_enqueue_burst;
 821         dev->enqueue_new_burst = sw_event_enqueue_burst;
 822         dev->enqueue_forward_burst = sw_event_enqueue_burst;
 823         dev->dequeue = sw_event_dequeue;
 824         dev->dequeue_burst = sw_event_dequeue_burst;
 825
 826         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 827                 return 0;
 828
 829         sw = dev->data->dev_private;
 830         sw->data = dev->data;
 831
 832         /* copy values passed from vdev command line to instance */
 833         sw->credit_update_quanta = credit_quanta;
 834         sw->sched_quanta = sched_quanta;
 835
 836         /* register service with EAL */
 837         struct rte_service_spec service;
 838         memset(&service, 0, sizeof(struct rte_service_spec));
 839         snprintf(service.name, sizeof(service.name), "%s_service", name);
 840         snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
 841                         name);
 842         service.socket_id = socket_id;
 843         service.callback = sw_sched_service_func;
 844         service.callback_userdata = (void *)dev;
 845
 846         int32_t ret = rte_service_component_register(&service, &sw->service_id);
 847         if (ret) {
 848                 SW_LOG_ERR("service register() failed");
 849                 return -ENOEXEC;
 850         }
 851
 852         dev->data->service_inited = 1;
 853         dev->data->service_id = sw->service_id;
 854
 855         return 0;
 856 }
 857
 858 static int
 859 sw_remove(struct rte_vdev_device *vdev)
 860 {
 861         const char *name;
 862
 863         name = rte_vdev_device_name(vdev);
 864         if (name == NULL)
 865                 return -EINVAL;
 866
 867         SW_LOG_INFO("Closing eventdev sw device %s\n", name);
 868
 869         return rte_event_pmd_vdev_uninit(name);
 870 }
 871
 872 static struct rte_vdev_driver evdev_sw_pmd_drv = {
 873         .probe = sw_probe,
 874         .remove = sw_remove
 875 };
 876
 877 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv);
 878 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
 879                 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>");