drivers/event/sw/sw_evdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <inttypes.h>
  34 #include <string.h>
  35
  36 #include <rte_vdev.h>
  37 #include <rte_memzone.h>
  38 #include <rte_kvargs.h>
  39 #include <rte_ring.h>
  40 #include <rte_errno.h>
  41
  42 #include "sw_evdev.h"
  43 #include "iq_ring.h"
  44 #include "event_ring.h"
  45
  46 #define EVENTDEV_NAME_SW_PMD event_sw
  47 #define NUMA_NODE_ARG "numa_node"
  48 #define SCHED_QUANTA_ARG "sched_quanta"
  49 #define CREDIT_QUANTA_ARG "credit_quanta"
  50
  51 static void
  52 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info);
  53
  54 static int
  55 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
  56                 const uint8_t priorities[], uint16_t num)
  57 {
  58         struct sw_port *p = port;
  59         struct sw_evdev *sw = sw_pmd_priv(dev);
  60         int i;
  61
  62         RTE_SET_USED(priorities);
  63         for (i = 0; i < num; i++) {
  64                 struct sw_qid *q = &sw->qids[queues[i]];
  65
  66                 /* check for qid map overflow */
  67                 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) {
  68                         rte_errno = -EDQUOT;
  69                         break;
  70                 }
  71
  72                 if (p->is_directed && p->num_qids_mapped > 0) {
  73                         rte_errno = -EDQUOT;
  74                         break;
  75                 }
  76
  77                 if (q->type == SW_SCHED_TYPE_DIRECT) {
  78                         /* check directed qids only map to one port */
  79                         if (p->num_qids_mapped > 0) {
  80                                 rte_errno = -EDQUOT;
  81                                 break;
  82                         }
  83                         /* check port only takes a directed flow */
  84                         if (num > 1) {
  85                                 rte_errno = -EDQUOT;
  86                                 break;
  87                         }
  88
  89                         p->is_directed = 1;
  90                         p->num_qids_mapped = 1;
  91                 } else if (q->type == RTE_SCHED_TYPE_ORDERED) {
  92                         p->num_ordered_qids++;
  93                         p->num_qids_mapped++;
  94                 } else if (q->type == RTE_SCHED_TYPE_ATOMIC ||
  95                                 q->type == RTE_SCHED_TYPE_PARALLEL) {
  96                         p->num_qids_mapped++;
  97                 }
  98
  99                 q->cq_map[q->cq_num_mapped_cqs] = p->id;
 100                 rte_smp_wmb();
 101                 q->cq_num_mapped_cqs++;
 102         }
 103         return i;
 104 }
 105
 106 static int
 107 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
 108                 uint16_t nb_unlinks)
 109 {
 110         struct sw_port *p = port;
 111         struct sw_evdev *sw = sw_pmd_priv(dev);
 112         unsigned int i, j;
 113
 114         int unlinked = 0;
 115         for (i = 0; i < nb_unlinks; i++) {
 116                 struct sw_qid *q = &sw->qids[queues[i]];
 117                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
 118                         if (q->cq_map[j] == p->id) {
 119                                 q->cq_map[j] =
 120                                         q->cq_map[q->cq_num_mapped_cqs - 1];
 121                                 rte_smp_wmb();
 122                                 q->cq_num_mapped_cqs--;
 123                                 unlinked++;
 124
 125                                 p->num_qids_mapped--;
 126
 127                                 if (q->type == RTE_SCHED_TYPE_ORDERED)
 128                                         p->num_ordered_qids--;
 129
 130                                 continue;
 131                         }
 132                 }
 133         }
 134         return unlinked;
 135 }
 136
 137 static int
 138 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 139                 const struct rte_event_port_conf *conf)
 140 {
 141         struct sw_evdev *sw = sw_pmd_priv(dev);
 142         struct sw_port *p = &sw->ports[port_id];
 143         char buf[QE_RING_NAMESIZE];
 144         unsigned int i;
 145
 146         struct rte_event_dev_info info;
 147         sw_info_get(dev, &info);
 148
 149         /* detect re-configuring and return credits to instance if needed */
 150         if (p->initialized) {
 151                 /* taking credits from pool is done one quanta at a time, and
 152                  * credits may be spend (counted in p->inflights) or still
 153                  * available in the port (p->inflight_credits). We must return
 154                  * the sum to no leak credits
 155                  */
 156                 int possible_inflights = p->inflight_credits + p->inflights;
 157                 rte_atomic32_sub(&sw->inflights, possible_inflights);
 158         }
 159
 160         *p = (struct sw_port){0}; /* zero entire structure */
 161         p->id = port_id;
 162         p->sw = sw;
 163
 164         snprintf(buf, sizeof(buf), "sw%d_%s", dev->data->dev_id,
 165                         "rx_worker_ring");
 166         p->rx_worker_ring = qe_ring_create(buf, MAX_SW_PROD_Q_DEPTH,
 167                         dev->data->socket_id);
 168         if (p->rx_worker_ring == NULL) {
 169                 SW_LOG_ERR("Error creating RX worker ring for port %d\n",
 170                                 port_id);
 171                 return -1;
 172         }
 173
 174         p->inflight_max = conf->new_event_threshold;
 175
 176         snprintf(buf, sizeof(buf), "sw%d_%s", dev->data->dev_id,
 177                         "cq_worker_ring");
 178         p->cq_worker_ring = qe_ring_create(buf, conf->dequeue_depth,
 179                         dev->data->socket_id);
 180         if (p->cq_worker_ring == NULL) {
 181                 qe_ring_destroy(p->rx_worker_ring);
 182                 SW_LOG_ERR("Error creating CQ worker ring for port %d\n",
 183                                 port_id);
 184                 return -1;
 185         }
 186         sw->cq_ring_space[port_id] = conf->dequeue_depth;
 187
 188         /* set hist list contents to empty */
 189         for (i = 0; i < SW_PORT_HIST_LIST; i++) {
 190                 p->hist_list[i].fid = -1;
 191                 p->hist_list[i].qid = -1;
 192         }
 193         dev->data->ports[port_id] = p;
 194
 195         rte_smp_wmb();
 196         p->initialized = 1;
 197         return 0;
 198 }
 199
 200 static void
 201 sw_port_release(void *port)
 202 {
 203         struct sw_port *p = (void *)port;
 204         if (p == NULL)
 205                 return;
 206
 207         qe_ring_destroy(p->rx_worker_ring);
 208         qe_ring_destroy(p->cq_worker_ring);
 209         memset(p, 0, sizeof(*p));
 210 }
 211
 212 static int32_t
 213 qid_init(struct sw_evdev *sw, unsigned int idx, int type,
 214                 const struct rte_event_queue_conf *queue_conf)
 215 {
 216         unsigned int i;
 217         int dev_id = sw->data->dev_id;
 218         int socket_id = sw->data->socket_id;
 219         char buf[IQ_RING_NAMESIZE];
 220         struct sw_qid *qid = &sw->qids[idx];
 221
 222         for (i = 0; i < SW_IQS_MAX; i++) {
 223                 snprintf(buf, sizeof(buf), "q_%u_iq_%d", idx, i);
 224                 qid->iq[i] = iq_ring_create(buf, socket_id);
 225                 if (!qid->iq[i]) {
 226                         SW_LOG_DBG("ring create failed");
 227                         goto cleanup;
 228                 }
 229         }
 230
 231         /* Initialize the FID structures to no pinning (-1), and zero packets */
 232         const struct sw_fid_t fid = {.cq = -1, .pcount = 0};
 233         for (i = 0; i < RTE_DIM(qid->fids); i++)
 234                 qid->fids[i] = fid;
 235
 236         qid->id = idx;
 237         qid->type = type;
 238         qid->priority = queue_conf->priority;
 239
 240         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 241                 char ring_name[RTE_RING_NAMESIZE];
 242                 uint32_t window_size;
 243
 244                 /* rte_ring and window_size_mask require require window_size to
 245                  * be a power-of-2.
 246                  */
 247                 window_size = rte_align32pow2(
 248                                 queue_conf->nb_atomic_order_sequences);
 249
 250                 qid->window_size = window_size - 1;
 251
 252                 if (!window_size) {
 253                         SW_LOG_DBG(
 254                                 "invalid reorder_window_size for ordered queue\n"
 255                                 );
 256                         goto cleanup;
 257                 }
 258
 259                 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i);
 260                 qid->reorder_buffer = rte_zmalloc_socket(buf,
 261                                 window_size * sizeof(qid->reorder_buffer[0]),
 262                                 0, socket_id);
 263                 if (!qid->reorder_buffer) {
 264                         SW_LOG_DBG("reorder_buffer malloc failed\n");
 265                         goto cleanup;
 266                 }
 267
 268                 memset(&qid->reorder_buffer[0],
 269                        0,
 270                        window_size * sizeof(qid->reorder_buffer[0]));
 271
 272                 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist",
 273                                 dev_id, idx);
 274
 275                 /* lookup the ring, and if it already exists, free it */
 276                 struct rte_ring *cleanup = rte_ring_lookup(ring_name);
 277                 if (cleanup)
 278                         rte_ring_free(cleanup);
 279
 280                 qid->reorder_buffer_freelist = rte_ring_create(ring_name,
 281                                 window_size,
 282                                 socket_id,
 283                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 284                 if (!qid->reorder_buffer_freelist) {
 285                         SW_LOG_DBG("freelist ring create failed");
 286                         goto cleanup;
 287                 }
 288
 289                 /* Populate the freelist with reorder buffer entries. Enqueue
 290                  * 'window_size - 1' entries because the rte_ring holds only
 291                  * that many.
 292                  */
 293                 for (i = 0; i < window_size - 1; i++) {
 294                         if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist,
 295                                                 &qid->reorder_buffer[i]) < 0)
 296                                 goto cleanup;
 297                 }
 298
 299                 qid->reorder_buffer_index = 0;
 300                 qid->cq_next_tx = 0;
 301         }
 302
 303         qid->initialized = 1;
 304
 305         return 0;
 306
 307 cleanup:
 308         for (i = 0; i < SW_IQS_MAX; i++) {
 309                 if (qid->iq[i])
 310                         iq_ring_destroy(qid->iq[i]);
 311         }
 312
 313         if (qid->reorder_buffer) {
 314                 rte_free(qid->reorder_buffer);
 315                 qid->reorder_buffer = NULL;
 316         }
 317
 318         if (qid->reorder_buffer_freelist) {
 319                 rte_ring_free(qid->reorder_buffer_freelist);
 320                 qid->reorder_buffer_freelist = NULL;
 321         }
 322
 323         return -EINVAL;
 324 }
 325
 326 static int
 327 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
 328                 const struct rte_event_queue_conf *conf)
 329 {
 330         int type;
 331
 332         /* SINGLE_LINK can be OR-ed with other types, so handle first */
 333         if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) {
 334                 type = SW_SCHED_TYPE_DIRECT;
 335         } else {
 336                 switch (conf->event_queue_cfg) {
 337                 case RTE_EVENT_QUEUE_CFG_ATOMIC_ONLY:
 338                         type = RTE_SCHED_TYPE_ATOMIC;
 339                         break;
 340                 case RTE_EVENT_QUEUE_CFG_ORDERED_ONLY:
 341                         type = RTE_SCHED_TYPE_ORDERED;
 342                         break;
 343                 case RTE_EVENT_QUEUE_CFG_PARALLEL_ONLY:
 344                         type = RTE_SCHED_TYPE_PARALLEL;
 345                         break;
 346                 case RTE_EVENT_QUEUE_CFG_ALL_TYPES:
 347                         SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n");
 348                         return -ENOTSUP;
 349                 default:
 350                         SW_LOG_ERR("Unknown queue type %d requested\n",
 351                                    conf->event_queue_cfg);
 352                         return -EINVAL;
 353                 }
 354         }
 355
 356         struct sw_evdev *sw = sw_pmd_priv(dev);
 357         return qid_init(sw, queue_id, type, conf);
 358 }
 359
 360 static void
 361 sw_queue_release(struct rte_eventdev *dev, uint8_t id)
 362 {
 363         struct sw_evdev *sw = sw_pmd_priv(dev);
 364         struct sw_qid *qid = &sw->qids[id];
 365         uint32_t i;
 366
 367         for (i = 0; i < SW_IQS_MAX; i++)
 368                 iq_ring_destroy(qid->iq[i]);
 369
 370         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 371                 rte_free(qid->reorder_buffer);
 372                 rte_ring_free(qid->reorder_buffer_freelist);
 373         }
 374         memset(qid, 0, sizeof(*qid));
 375 }
 376
 377 static void
 378 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
 379                                  struct rte_event_queue_conf *conf)
 380 {
 381         RTE_SET_USED(dev);
 382         RTE_SET_USED(queue_id);
 383
 384         static const struct rte_event_queue_conf default_conf = {
 385                 .nb_atomic_flows = 4096,
 386                 .nb_atomic_order_sequences = 1,
 387                 .event_queue_cfg = RTE_EVENT_QUEUE_CFG_ATOMIC_ONLY,
 388                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 389         };
 390
 391         *conf = default_conf;
 392 }
 393
 394 static void
 395 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 396                  struct rte_event_port_conf *port_conf)
 397 {
 398         RTE_SET_USED(dev);
 399         RTE_SET_USED(port_id);
 400
 401         port_conf->new_event_threshold = 1024;
 402         port_conf->dequeue_depth = 16;
 403         port_conf->enqueue_depth = 16;
 404 }
 405
 406 static int
 407 sw_dev_configure(const struct rte_eventdev *dev)
 408 {
 409         struct sw_evdev *sw = sw_pmd_priv(dev);
 410         const struct rte_eventdev_data *data = dev->data;
 411         const struct rte_event_dev_config *conf = &data->dev_conf;
 412
 413         sw->qid_count = conf->nb_event_queues;
 414         sw->port_count = conf->nb_event_ports;
 415         sw->nb_events_limit = conf->nb_events_limit;
 416         rte_atomic32_set(&sw->inflights, 0);
 417
 418         if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
 419                 return -ENOTSUP;
 420
 421         return 0;
 422 }
 423
 424 static void
 425 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info)
 426 {
 427         RTE_SET_USED(dev);
 428
 429         static const struct rte_event_dev_info evdev_sw_info = {
 430                         .driver_name = SW_PMD_NAME,
 431                         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
 432                         .max_event_queue_flows = SW_QID_NUM_FIDS,
 433                         .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX,
 434                         .max_event_priority_levels = SW_IQS_MAX,
 435                         .max_event_ports = SW_PORTS_MAX,
 436                         .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH,
 437                         .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH,
 438                         .max_num_events = SW_INFLIGHT_EVENTS_TOTAL,
 439                         .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
 440                                         RTE_EVENT_DEV_CAP_BURST_MODE |
 441                                         RTE_EVENT_DEV_CAP_EVENT_QOS),
 442         };
 443
 444         *info = evdev_sw_info;
 445 }
 446
 447 static void
 448 sw_dump(struct rte_eventdev *dev, FILE *f)
 449 {
 450         const struct sw_evdev *sw = sw_pmd_priv(dev);
 451
 452         static const char * const q_type_strings[] = {
 453                         "Ordered", "Atomic", "Parallel", "Directed"
 454         };
 455         uint32_t i;
 456         fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name",
 457                         sw->port_count, sw->qid_count);
 458
 459         fprintf(f, "\trx   %"PRIu64"\n\tdrop %"PRIu64"\n\ttx   %"PRIu64"\n",
 460                 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts);
 461         fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called);
 462         fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called);
 463         fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues);
 464         fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues);
 465         uint32_t inflights = rte_atomic32_read(&sw->inflights);
 466         uint32_t credits = sw->nb_events_limit - inflights;
 467         fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits);
 468
 469 #define COL_RED "\x1b[31m"
 470 #define COL_RESET "\x1b[0m"
 471
 472         for (i = 0; i < sw->port_count; i++) {
 473                 int max, j;
 474                 const struct sw_port *p = &sw->ports[i];
 475                 if (!p->initialized) {
 476                         fprintf(f, "  %sPort %d not initialized.%s\n",
 477                                 COL_RED, i, COL_RESET);
 478                         continue;
 479                 }
 480                 fprintf(f, "  Port %d %s\n", i,
 481                         p->is_directed ? " (SingleCons)" : "");
 482                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64
 483                         "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts,
 484                         sw->ports[i].stats.rx_dropped,
 485                         sw->ports[i].stats.tx_pkts,
 486                         (p->inflights == p->inflight_max) ?
 487                                 COL_RED : COL_RESET,
 488                         sw->ports[i].inflights, COL_RESET);
 489
 490                 fprintf(f, "\tMax New: %u"
 491                         "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n",
 492                         sw->ports[i].inflight_max,
 493                         sw->ports[i].avg_pkt_ticks,
 494                         sw->ports[i].inflight_credits);
 495                 fprintf(f, "\tReceive burst distribution:\n");
 496                 float zp_percent = p->zero_polls * 100.0 / p->total_polls;
 497                 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ",
 498                                 zp_percent);
 499                 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;)
 500                         if (p->poll_buckets[max] != 0)
 501                                 break;
 502                 for (j = 0; j <= max; j++) {
 503                         if (p->poll_buckets[j] != 0) {
 504                                 float poll_pc = p->poll_buckets[j] * 100.0 /
 505                                         p->total_polls;
 506                                 fprintf(f, "%u-%u:%.02f%% ",
 507                                         ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1),
 508                                         ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT),
 509                                         poll_pc);
 510                         }
 511                 }
 512                 fprintf(f, "\n");
 513
 514                 if (p->rx_worker_ring) {
 515                         uint64_t used = qe_ring_count(p->rx_worker_ring);
 516                         uint64_t space = qe_ring_free_count(p->rx_worker_ring);
 517                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 518                         fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4"
 519                                         PRIu64 COL_RESET"\n", col, used, space);
 520                 } else
 521                         fprintf(f, "\trx ring not initialized.\n");
 522
 523                 if (p->cq_worker_ring) {
 524                         uint64_t used = qe_ring_count(p->cq_worker_ring);
 525                         uint64_t space = qe_ring_free_count(p->cq_worker_ring);
 526                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 527                         fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4"
 528                                         PRIu64 COL_RESET"\n", col, used, space);
 529                 } else
 530                         fprintf(f, "\tcq ring not initialized.\n");
 531         }
 532
 533         for (i = 0; i < sw->qid_count; i++) {
 534                 const struct sw_qid *qid = &sw->qids[i];
 535                 if (!qid->initialized) {
 536                         fprintf(f, "  %sQueue %d not initialized.%s\n",
 537                                 COL_RED, i, COL_RESET);
 538                         continue;
 539                 }
 540                 int affinities_per_port[SW_PORTS_MAX] = {0};
 541                 uint32_t inflights = 0;
 542
 543                 fprintf(f, "  Queue %d (%s)\n", i, q_type_strings[qid->type]);
 544                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64"\n",
 545                         qid->stats.rx_pkts, qid->stats.rx_dropped,
 546                         qid->stats.tx_pkts);
 547                 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 548                         struct rte_ring *rob_buf_free =
 549                                 qid->reorder_buffer_freelist;
 550                         if (rob_buf_free)
 551                                 fprintf(f, "\tReorder entries in use: %u\n",
 552                                         rte_ring_free_count(rob_buf_free));
 553                         else
 554                                 fprintf(f,
 555                                         "\tReorder buffer not initialized\n");
 556                 }
 557
 558                 uint32_t flow;
 559                 for (flow = 0; flow < RTE_DIM(qid->fids); flow++)
 560                         if (qid->fids[flow].cq != -1) {
 561                                 affinities_per_port[qid->fids[flow].cq]++;
 562                                 inflights += qid->fids[flow].pcount;
 563                         }
 564
 565                 uint32_t port;
 566                 fprintf(f, "\tPer Port Stats:\n");
 567                 for (port = 0; port < sw->port_count; port++) {
 568                         fprintf(f, "\t  Port %d: Pkts: %"PRIu64, port,
 569                                         qid->to_port[port]);
 570                         fprintf(f, "\tFlows: %d\n", affinities_per_port[port]);
 571                 }
 572
 573                 uint32_t iq;
 574                 uint32_t iq_printed = 0;
 575                 for (iq = 0; iq < SW_IQS_MAX; iq++) {
 576                         if (!qid->iq[iq]) {
 577                                 fprintf(f, "\tiq %d is not initialized.\n", iq);
 578                                 iq_printed = 1;
 579                                 continue;
 580                         }
 581                         uint32_t used = iq_ring_count(qid->iq[iq]);
 582                         uint32_t free = iq_ring_free_count(qid->iq[iq]);
 583                         const char *col = (free == 0) ? COL_RED : COL_RESET;
 584                         if (used > 0) {
 585                                 fprintf(f, "\t%siq %d: Used %d\tFree %d"
 586                                         COL_RESET"\n", col, iq, used, free);
 587                                 iq_printed = 1;
 588                         }
 589                 }
 590                 if (iq_printed == 0)
 591                         fprintf(f, "\t-- iqs empty --\n");
 592         }
 593 }
 594
 595 static int
 596 sw_start(struct rte_eventdev *dev)
 597 {
 598         unsigned int i, j;
 599         struct sw_evdev *sw = sw_pmd_priv(dev);
 600         /* check all ports are set up */
 601         for (i = 0; i < sw->port_count; i++)
 602                 if (sw->ports[i].rx_worker_ring == NULL) {
 603                         SW_LOG_ERR("Port %d not configured\n", i);
 604                         return -ESTALE;
 605                 }
 606
 607         /* check all queues are configured and mapped to ports*/
 608         for (i = 0; i < sw->qid_count; i++)
 609                 if (sw->qids[i].iq[0] == NULL ||
 610                                 sw->qids[i].cq_num_mapped_cqs == 0) {
 611                         SW_LOG_ERR("Queue %d not configured\n", i);
 612                         return -ENOLINK;
 613                 }
 614
 615         /* build up our prioritized array of qids */
 616         /* We don't use qsort here, as if all/multiple entries have the same
 617          * priority, the result is non-deterministic. From "man 3 qsort":
 618          * "If two members compare as equal, their order in the sorted
 619          * array is undefined."
 620          */
 621         uint32_t qidx = 0;
 622         for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) {
 623                 for (i = 0; i < sw->qid_count; i++) {
 624                         if (sw->qids[i].priority == j) {
 625                                 sw->qids_prioritized[qidx] = &sw->qids[i];
 626                                 qidx++;
 627                         }
 628                 }
 629         }
 630
 631         if (sw_xstats_init(sw) < 0)
 632                 return -EINVAL;
 633
 634         rte_smp_wmb();
 635         sw->started = 1;
 636
 637         return 0;
 638 }
 639
 640 static void
 641 sw_stop(struct rte_eventdev *dev)
 642 {
 643         struct sw_evdev *sw = sw_pmd_priv(dev);
 644         sw_xstats_uninit(sw);
 645         sw->started = 0;
 646         rte_smp_wmb();
 647 }
 648
 649 static int
 650 sw_close(struct rte_eventdev *dev)
 651 {
 652         struct sw_evdev *sw = sw_pmd_priv(dev);
 653         uint32_t i;
 654
 655         for (i = 0; i < sw->qid_count; i++)
 656                 sw_queue_release(dev, i);
 657         sw->qid_count = 0;
 658
 659         for (i = 0; i < sw->port_count; i++)
 660                 sw_port_release(&sw->ports[i]);
 661         sw->port_count = 0;
 662
 663         memset(&sw->stats, 0, sizeof(sw->stats));
 664         sw->sched_called = 0;
 665         sw->sched_no_iq_enqueues = 0;
 666         sw->sched_no_cq_enqueues = 0;
 667         sw->sched_cq_qid_called = 0;
 668
 669         return 0;
 670 }
 671
 672 static int
 673 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 674 {
 675         int *socket_id = opaque;
 676         *socket_id = atoi(value);
 677         if (*socket_id >= RTE_MAX_NUMA_NODES)
 678                 return -1;
 679         return 0;
 680 }
 681
 682 static int
 683 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque)
 684 {
 685         int *quanta = opaque;
 686         *quanta = atoi(value);
 687         if (*quanta < 0 || *quanta >= 4096)
 688                 return -1;
 689         return 0;
 690 }
 691
 692 static int
 693 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 694 {
 695         int *credit = opaque;
 696         *credit = atoi(value);
 697         if (*credit < 0 || *credit >= 128)
 698                 return -1;
 699         return 0;
 700 }
 701
 702 static int
 703 sw_probe(struct rte_vdev_device *vdev)
 704 {
 705         static const struct rte_eventdev_ops evdev_sw_ops = {
 706                         .dev_configure = sw_dev_configure,
 707                         .dev_infos_get = sw_info_get,
 708                         .dev_close = sw_close,
 709                         .dev_start = sw_start,
 710                         .dev_stop = sw_stop,
 711                         .dump = sw_dump,
 712
 713                         .queue_def_conf = sw_queue_def_conf,
 714                         .queue_setup = sw_queue_setup,
 715                         .queue_release = sw_queue_release,
 716                         .port_def_conf = sw_port_def_conf,
 717                         .port_setup = sw_port_setup,
 718                         .port_release = sw_port_release,
 719                         .port_link = sw_port_link,
 720                         .port_unlink = sw_port_unlink,
 721
 722                         .xstats_get = sw_xstats_get,
 723                         .xstats_get_names = sw_xstats_get_names,
 724                         .xstats_get_by_name = sw_xstats_get_by_name,
 725                         .xstats_reset = sw_xstats_reset,
 726         };
 727
 728         static const char *const args[] = {
 729                 NUMA_NODE_ARG,
 730                 SCHED_QUANTA_ARG,
 731                 CREDIT_QUANTA_ARG,
 732                 NULL
 733         };
 734         const char *name;
 735         const char *params;
 736         struct rte_eventdev *dev;
 737         struct sw_evdev *sw;
 738         int socket_id = rte_socket_id();
 739         int sched_quanta  = SW_DEFAULT_SCHED_QUANTA;
 740         int credit_quanta = SW_DEFAULT_CREDIT_QUANTA;
 741
 742         name = rte_vdev_device_name(vdev);
 743         params = rte_vdev_device_args(vdev);
 744         if (params != NULL && params[0] != '\0') {
 745                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
 746
 747                 if (!kvlist) {
 748                         SW_LOG_INFO(
 749                                 "Ignoring unsupported parameters when creating device '%s'\n",
 750                                 name);
 751                 } else {
 752                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
 753                                         assign_numa_node, &socket_id);
 754                         if (ret != 0) {
 755                                 SW_LOG_ERR(
 756                                         "%s: Error parsing numa node parameter",
 757                                         name);
 758                                 rte_kvargs_free(kvlist);
 759                                 return ret;
 760                         }
 761
 762                         ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG,
 763                                         set_sched_quanta, &sched_quanta);
 764                         if (ret != 0) {
 765                                 SW_LOG_ERR(
 766                                         "%s: Error parsing sched quanta parameter",
 767                                         name);
 768                                 rte_kvargs_free(kvlist);
 769                                 return ret;
 770                         }
 771
 772                         ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG,
 773                                         set_credit_quanta, &credit_quanta);
 774                         if (ret != 0) {
 775                                 SW_LOG_ERR(
 776                                         "%s: Error parsing credit quanta parameter",
 777                                         name);
 778                                 rte_kvargs_free(kvlist);
 779                                 return ret;
 780                         }
 781
 782                         rte_kvargs_free(kvlist);
 783                 }
 784         }
 785
 786         SW_LOG_INFO(
 787                         "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n",
 788                         name, socket_id, sched_quanta, credit_quanta);
 789
 790         dev = rte_event_pmd_vdev_init(name,
 791                         sizeof(struct sw_evdev), socket_id);
 792         if (dev == NULL) {
 793                 SW_LOG_ERR("eventdev vdev init() failed");
 794                 return -EFAULT;
 795         }
 796         dev->dev_ops = &evdev_sw_ops;
 797         dev->enqueue = sw_event_enqueue;
 798         dev->enqueue_burst = sw_event_enqueue_burst;
 799         dev->enqueue_new_burst = sw_event_enqueue_burst;
 800         dev->dequeue = sw_event_dequeue;
 801         dev->dequeue_burst = sw_event_dequeue_burst;
 802         dev->schedule = sw_event_schedule;
 803
 804         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 805                 return 0;
 806
 807         sw = dev->data->dev_private;
 808         sw->data = dev->data;
 809
 810         /* copy values passed from vdev command line to instance */
 811         sw->credit_update_quanta = credit_quanta;
 812         sw->sched_quanta = sched_quanta;
 813
 814         return 0;
 815 }
 816
 817 static int
 818 sw_remove(struct rte_vdev_device *vdev)
 819 {
 820         const char *name;
 821
 822         name = rte_vdev_device_name(vdev);
 823         if (name == NULL)
 824                 return -EINVAL;
 825
 826         SW_LOG_INFO("Closing eventdev sw device %s\n", name);
 827
 828         return rte_event_pmd_vdev_uninit(name);
 829 }
 830
 831 static struct rte_vdev_driver evdev_sw_pmd_drv = {
 832         .probe = sw_probe,
 833         .remove = sw_remove
 834 };
 835
 836 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv);
 837 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
 838                 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>");