examples/ipsec-secgw/event_helper.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2020 Marvell International Ltd.
   3  */
   4 #include <rte_bitmap.h>
   5 #include <rte_ethdev.h>
   6 #include <rte_eventdev.h>
   7 #include <rte_event_eth_rx_adapter.h>
   8 #include <rte_event_eth_tx_adapter.h>
   9 #include <rte_malloc.h>
  10 #include <stdbool.h>
  11
  12 #include "event_helper.h"
  13 #include "ipsec-secgw.h"
  14
  15 #define DEFAULT_VECTOR_SIZE  16
  16 #define DEFAULT_VECTOR_TMO   102400
  17
  18 static volatile bool eth_core_running;
  19
  20 static int
  21 eh_get_enabled_cores(struct rte_bitmap *eth_core_mask)
  22 {
  23         int i, count = 0;
  24
  25         RTE_LCORE_FOREACH(i) {
  26                 /* Check if this core is enabled in core mask*/
  27                 if (rte_bitmap_get(eth_core_mask, i)) {
  28                         /* Found enabled core */
  29                         count++;
  30                 }
  31         }
  32         return count;
  33 }
  34
  35 static inline unsigned int
  36 eh_get_next_eth_core(struct eventmode_conf *em_conf)
  37 {
  38         static unsigned int prev_core = -1;
  39         unsigned int next_core;
  40
  41         /*
  42          * Make sure we have at least one eth core running, else the following
  43          * logic would lead to an infinite loop.
  44          */
  45         if (eh_get_enabled_cores(em_conf->eth_core_mask) == 0) {
  46                 EH_LOG_ERR("No enabled eth core found");
  47                 return RTE_MAX_LCORE;
  48         }
  49
  50         /* Only some cores are marked as eth cores, skip others */
  51         do {
  52                 /* Get the next core */
  53                 next_core = rte_get_next_lcore(prev_core, 0, 1);
  54
  55                 /* Check if we have reached max lcores */
  56                 if (next_core == RTE_MAX_LCORE)
  57                         return next_core;
  58
  59                 /* Update prev_core */
  60                 prev_core = next_core;
  61         } while (!(rte_bitmap_get(em_conf->eth_core_mask, next_core)));
  62
  63         return next_core;
  64 }
  65
  66 static inline unsigned int
  67 eh_get_next_active_core(struct eventmode_conf *em_conf, unsigned int prev_core)
  68 {
  69         unsigned int next_core;
  70
  71         /* Get next active core skipping cores reserved as eth cores */
  72         do {
  73                 /* Get the next core */
  74                 next_core = rte_get_next_lcore(prev_core, 0, 0);
  75
  76                 /* Check if we have reached max lcores */
  77                 if (next_core == RTE_MAX_LCORE)
  78                         return next_core;
  79
  80                 prev_core = next_core;
  81         } while (rte_bitmap_get(em_conf->eth_core_mask, next_core));
  82
  83         return next_core;
  84 }
  85
  86 static struct eventdev_params *
  87 eh_get_eventdev_params(struct eventmode_conf *em_conf, uint8_t eventdev_id)
  88 {
  89         int i;
  90
  91         for (i = 0; i < em_conf->nb_eventdev; i++) {
  92                 if (em_conf->eventdev_config[i].eventdev_id == eventdev_id)
  93                         break;
  94         }
  95
  96         /* No match */
  97         if (i == em_conf->nb_eventdev)
  98                 return NULL;
  99
 100         return &(em_conf->eventdev_config[i]);
 101 }
 102
 103 static inline bool
 104 eh_dev_has_rx_internal_port(uint8_t eventdev_id)
 105 {
 106         bool flag = true;
 107         int j, ret;
 108
 109         RTE_ETH_FOREACH_DEV(j) {
 110                 uint32_t caps = 0;
 111
 112                 ret = rte_event_eth_rx_adapter_caps_get(eventdev_id, j, &caps);
 113                 if (ret < 0)
 114                         return false;
 115
 116                 if (!(caps & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT))
 117                         flag = false;
 118         }
 119         return flag;
 120 }
 121
 122 static inline bool
 123 eh_dev_has_tx_internal_port(uint8_t eventdev_id)
 124 {
 125         bool flag = true;
 126         int j, ret;
 127
 128         RTE_ETH_FOREACH_DEV(j) {
 129                 uint32_t caps = 0;
 130
 131                 ret = rte_event_eth_tx_adapter_caps_get(eventdev_id, j, &caps);
 132                 if (ret < 0)
 133                         return false;
 134
 135                 if (!(caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT))
 136                         flag = false;
 137         }
 138         return flag;
 139 }
 140
 141 static inline bool
 142 eh_dev_has_burst_mode(uint8_t dev_id)
 143 {
 144         struct rte_event_dev_info dev_info;
 145
 146         rte_event_dev_info_get(dev_id, &dev_info);
 147         return (dev_info.event_dev_cap & RTE_EVENT_DEV_CAP_BURST_MODE) ?
 148                         true : false;
 149 }
 150
 151 static int
 152 eh_set_default_conf_eventdev(struct eventmode_conf *em_conf)
 153 {
 154         int lcore_count, nb_eventdev, nb_eth_dev, ret;
 155         struct eventdev_params *eventdev_config;
 156         struct rte_event_dev_info dev_info;
 157
 158         /* Get the number of event devices */
 159         nb_eventdev = rte_event_dev_count();
 160         if (nb_eventdev == 0) {
 161                 EH_LOG_ERR("No event devices detected");
 162                 return -EINVAL;
 163         }
 164
 165         if (nb_eventdev != 1) {
 166                 EH_LOG_ERR("Event mode does not support multiple event devices. "
 167                            "Please provide only one event device.");
 168                 return -EINVAL;
 169         }
 170
 171         /* Get the number of eth devs */
 172         nb_eth_dev = rte_eth_dev_count_avail();
 173         if (nb_eth_dev == 0) {
 174                 EH_LOG_ERR("No eth devices detected");
 175                 return -EINVAL;
 176         }
 177
 178         /* Get the number of lcores */
 179         lcore_count = rte_lcore_count();
 180
 181         /* Read event device info */
 182         ret = rte_event_dev_info_get(0, &dev_info);
 183         if (ret < 0) {
 184                 EH_LOG_ERR("Failed to read event device info %d", ret);
 185                 return ret;
 186         }
 187
 188         /* Check if enough ports are available */
 189         if (dev_info.max_event_ports < 2) {
 190                 EH_LOG_ERR("Not enough event ports available");
 191                 return -EINVAL;
 192         }
 193
 194         /* Get the first event dev conf */
 195         eventdev_config = &(em_conf->eventdev_config[0]);
 196
 197         /* Save number of queues & ports available */
 198         eventdev_config->eventdev_id = 0;
 199         eventdev_config->nb_eventqueue = dev_info.max_event_queues;
 200         eventdev_config->nb_eventport = dev_info.max_event_ports;
 201         eventdev_config->ev_queue_mode = RTE_EVENT_QUEUE_CFG_ALL_TYPES;
 202
 203         /* Check if there are more queues than required */
 204         if (eventdev_config->nb_eventqueue > nb_eth_dev + 1) {
 205                 /* One queue is reserved for Tx */
 206                 eventdev_config->nb_eventqueue = nb_eth_dev + 1;
 207         }
 208
 209         /* Check if there are more ports than required */
 210         if (eventdev_config->nb_eventport > lcore_count) {
 211                 /* One port per lcore is enough */
 212                 eventdev_config->nb_eventport = lcore_count;
 213         }
 214
 215         /* Update the number of event devices */
 216         em_conf->nb_eventdev++;
 217
 218         return 0;
 219 }
 220
 221 static void
 222 eh_do_capability_check(struct eventmode_conf *em_conf)
 223 {
 224         struct eventdev_params *eventdev_config;
 225         int all_internal_ports = 1;
 226         uint32_t eventdev_id;
 227         int i;
 228
 229         for (i = 0; i < em_conf->nb_eventdev; i++) {
 230
 231                 /* Get the event dev conf */
 232                 eventdev_config = &(em_conf->eventdev_config[i]);
 233                 eventdev_id = eventdev_config->eventdev_id;
 234
 235                 /* Check if event device has internal port for Rx & Tx */
 236                 if (eh_dev_has_rx_internal_port(eventdev_id) &&
 237                     eh_dev_has_tx_internal_port(eventdev_id)) {
 238                         eventdev_config->all_internal_ports = 1;
 239                 } else {
 240                         all_internal_ports = 0;
 241                 }
 242         }
 243
 244         /*
 245          * If Rx & Tx internal ports are supported by all event devices then
 246          * eth cores won't be required. Override the eth core mask requested
 247          * and decrement number of event queues by one as it won't be needed
 248          * for Tx.
 249          */
 250         if (all_internal_ports) {
 251                 rte_bitmap_reset(em_conf->eth_core_mask);
 252                 for (i = 0; i < em_conf->nb_eventdev; i++)
 253                         em_conf->eventdev_config[i].nb_eventqueue--;
 254         }
 255 }
 256
 257 static int
 258 eh_set_default_conf_link(struct eventmode_conf *em_conf)
 259 {
 260         struct eventdev_params *eventdev_config;
 261         struct eh_event_link_info *link;
 262         unsigned int lcore_id = -1;
 263         int i, link_index;
 264
 265         /*
 266          * Create a 1:1 mapping from event ports to cores. If the number
 267          * of event ports is lesser than the cores, some cores won't
 268          * execute worker. If there are more event ports, then some ports
 269          * won't be used.
 270          *
 271          */
 272
 273         /*
 274          * The event queue-port mapping is done according to the link. Since
 275          * we are falling back to the default link config, enabling
 276          * "all_ev_queue_to_ev_port" mode flag. This will map all queues
 277          * to the port.
 278          */
 279         em_conf->ext_params.all_ev_queue_to_ev_port = 1;
 280
 281         /* Get first event dev conf */
 282         eventdev_config = &(em_conf->eventdev_config[0]);
 283
 284         /* Loop through the ports */
 285         for (i = 0; i < eventdev_config->nb_eventport; i++) {
 286
 287                 /* Get next active core id */
 288                 lcore_id = eh_get_next_active_core(em_conf,
 289                                 lcore_id);
 290
 291                 if (lcore_id == RTE_MAX_LCORE) {
 292                         /* Reached max cores */
 293                         return 0;
 294                 }
 295
 296                 /* Save the current combination as one link */
 297
 298                 /* Get the index */
 299                 link_index = em_conf->nb_link;
 300
 301                 /* Get the corresponding link */
 302                 link = &(em_conf->link[link_index]);
 303
 304                 /* Save link */
 305                 link->eventdev_id = eventdev_config->eventdev_id;
 306                 link->event_port_id = i;
 307                 link->lcore_id = lcore_id;
 308
 309                 /*
 310                  * Don't set eventq_id as by default all queues
 311                  * need to be mapped to the port, which is controlled
 312                  * by the operating mode.
 313                  */
 314
 315                 /* Update number of links */
 316                 em_conf->nb_link++;
 317         }
 318
 319         return 0;
 320 }
 321
 322 static int
 323 eh_set_default_conf_rx_adapter(struct eventmode_conf *em_conf)
 324 {
 325         struct rx_adapter_connection_info *conn;
 326         struct eventdev_params *eventdev_config;
 327         struct rx_adapter_conf *adapter;
 328         bool rx_internal_port = true;
 329         bool single_ev_queue = false;
 330         int nb_eventqueue;
 331         uint32_t caps = 0;
 332         int eventdev_id;
 333         int nb_eth_dev;
 334         int adapter_id;
 335         int conn_id;
 336         int ret;
 337         int i;
 338
 339         /* Create one adapter with eth queues mapped to event queue(s) */
 340
 341         if (em_conf->nb_eventdev == 0) {
 342                 EH_LOG_ERR("No event devs registered");
 343                 return -EINVAL;
 344         }
 345
 346         /* Get the number of eth devs */
 347         nb_eth_dev = rte_eth_dev_count_avail();
 348
 349         /* Use the first event dev */
 350         eventdev_config = &(em_conf->eventdev_config[0]);
 351
 352         /* Get eventdev ID */
 353         eventdev_id = eventdev_config->eventdev_id;
 354         adapter_id = 0;
 355
 356         /* Get adapter conf */
 357         adapter = &(em_conf->rx_adapter[adapter_id]);
 358
 359         /* Set adapter conf */
 360         adapter->eventdev_id = eventdev_id;
 361         adapter->adapter_id = adapter_id;
 362
 363         /*
 364          * If event device does not have internal ports for passing
 365          * packets then reserved one queue for Tx path
 366          */
 367         nb_eventqueue = eventdev_config->all_internal_ports ?
 368                         eventdev_config->nb_eventqueue :
 369                         eventdev_config->nb_eventqueue - 1;
 370
 371         /*
 372          * Map all queues of eth device (port) to an event queue. If there
 373          * are more event queues than eth ports then create 1:1 mapping.
 374          * Otherwise map all eth ports to a single event queue.
 375          */
 376         if (nb_eth_dev > nb_eventqueue)
 377                 single_ev_queue = true;
 378
 379         for (i = 0; i < nb_eth_dev; i++) {
 380
 381                 /* Use only the ports enabled */
 382                 if ((em_conf->eth_portmask & (1 << i)) == 0)
 383                         continue;
 384
 385                 /* Get the connection id */
 386                 conn_id = adapter->nb_connections;
 387
 388                 /* Get the connection */
 389                 conn = &(adapter->conn[conn_id]);
 390
 391                 /* Set mapping between eth ports & event queues*/
 392                 conn->ethdev_id = i;
 393                 conn->eventq_id = single_ev_queue ? 0 : i;
 394
 395                 /* Add all eth queues eth port to event queue */
 396                 conn->ethdev_rx_qid = -1;
 397
 398                 /* Get Rx adapter capabilities */
 399                 ret = rte_event_eth_rx_adapter_caps_get(eventdev_id, i, &caps);
 400                 if (ret < 0) {
 401                         EH_LOG_ERR("Failed to get event device %d eth rx adapter"
 402                                    " capabilities for port %d", eventdev_id, i);
 403                         return ret;
 404                 }
 405                 if (!(caps & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT))
 406                         rx_internal_port = false;
 407
 408                 /* Update no of connections */
 409                 adapter->nb_connections++;
 410
 411         }
 412
 413         if (rx_internal_port) {
 414                 /* Rx core is not required */
 415                 adapter->rx_core_id = -1;
 416         } else {
 417                 /* Rx core is required */
 418                 adapter->rx_core_id = eh_get_next_eth_core(em_conf);
 419         }
 420
 421         /* We have setup one adapter */
 422         em_conf->nb_rx_adapter = 1;
 423
 424         return 0;
 425 }
 426
 427 static int
 428 eh_set_default_conf_tx_adapter(struct eventmode_conf *em_conf)
 429 {
 430         struct tx_adapter_connection_info *conn;
 431         struct eventdev_params *eventdev_config;
 432         struct tx_adapter_conf *tx_adapter;
 433         bool tx_internal_port = true;
 434         uint32_t caps = 0;
 435         int eventdev_id;
 436         int adapter_id;
 437         int nb_eth_dev;
 438         int conn_id;
 439         int ret;
 440         int i;
 441
 442         /*
 443          * Create one Tx adapter with all eth queues mapped to event queues
 444          * 1:1.
 445          */
 446
 447         if (em_conf->nb_eventdev == 0) {
 448                 EH_LOG_ERR("No event devs registered");
 449                 return -EINVAL;
 450         }
 451
 452         /* Get the number of eth devs */
 453         nb_eth_dev = rte_eth_dev_count_avail();
 454
 455         /* Use the first event dev */
 456         eventdev_config = &(em_conf->eventdev_config[0]);
 457
 458         /* Get eventdev ID */
 459         eventdev_id = eventdev_config->eventdev_id;
 460         adapter_id = 0;
 461
 462         /* Get adapter conf */
 463         tx_adapter = &(em_conf->tx_adapter[adapter_id]);
 464
 465         /* Set adapter conf */
 466         tx_adapter->eventdev_id = eventdev_id;
 467         tx_adapter->adapter_id = adapter_id;
 468
 469         /*
 470          * Map all Tx queues of the eth device (port) to the event device.
 471          */
 472
 473         /* Set defaults for connections */
 474
 475         /*
 476          * One eth device (port) is one connection. Map all Tx queues
 477          * of the device to the Tx adapter.
 478          */
 479
 480         for (i = 0; i < nb_eth_dev; i++) {
 481
 482                 /* Use only the ports enabled */
 483                 if ((em_conf->eth_portmask & (1 << i)) == 0)
 484                         continue;
 485
 486                 /* Get the connection id */
 487                 conn_id = tx_adapter->nb_connections;
 488
 489                 /* Get the connection */
 490                 conn = &(tx_adapter->conn[conn_id]);
 491
 492                 /* Add ethdev to connections */
 493                 conn->ethdev_id = i;
 494
 495                 /* Add all eth tx queues to adapter */
 496                 conn->ethdev_tx_qid = -1;
 497
 498                 /* Get Tx adapter capabilities */
 499                 ret = rte_event_eth_tx_adapter_caps_get(eventdev_id, i, &caps);
 500                 if (ret < 0) {
 501                         EH_LOG_ERR("Failed to get event device %d eth tx adapter"
 502                                    " capabilities for port %d", eventdev_id, i);
 503                         return ret;
 504                 }
 505                 if (!(caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT))
 506                         tx_internal_port = false;
 507
 508                 /* Update no of connections */
 509                 tx_adapter->nb_connections++;
 510         }
 511
 512         if (tx_internal_port) {
 513                 /* Tx core is not required */
 514                 tx_adapter->tx_core_id = -1;
 515         } else {
 516                 /* Tx core is required */
 517                 tx_adapter->tx_core_id = eh_get_next_eth_core(em_conf);
 518
 519                 /*
 520                  * Use one event queue per adapter for submitting packets
 521                  * for Tx. Reserving the last queue available
 522                  */
 523                 /* Queue numbers start at 0 */
 524                 tx_adapter->tx_ev_queue = eventdev_config->nb_eventqueue - 1;
 525         }
 526
 527         /* We have setup one adapter */
 528         em_conf->nb_tx_adapter = 1;
 529         return 0;
 530 }
 531
 532 static int
 533 eh_validate_conf(struct eventmode_conf *em_conf)
 534 {
 535         int ret;
 536
 537         /*
 538          * Check if event devs are specified. Else probe the event devices
 539          * and initialize the config with all ports & queues available
 540          */
 541         if (em_conf->nb_eventdev == 0) {
 542                 ret = eh_set_default_conf_eventdev(em_conf);
 543                 if (ret != 0)
 544                         return ret;
 545         }
 546
 547         /* Perform capability check for the selected event devices */
 548         eh_do_capability_check(em_conf);
 549
 550         /*
 551          * Check if links are specified. Else generate a default config for
 552          * the event ports used.
 553          */
 554         if (em_conf->nb_link == 0) {
 555                 ret = eh_set_default_conf_link(em_conf);
 556                 if (ret != 0)
 557                         return ret;
 558         }
 559
 560         /*
 561          * Check if rx adapters are specified. Else generate a default config
 562          * with one rx adapter and all eth queues - event queue mapped.
 563          */
 564         if (em_conf->nb_rx_adapter == 0) {
 565                 ret = eh_set_default_conf_rx_adapter(em_conf);
 566                 if (ret != 0)
 567                         return ret;
 568         }
 569
 570         /*
 571          * Check if tx adapters are specified. Else generate a default config
 572          * with one tx adapter.
 573          */
 574         if (em_conf->nb_tx_adapter == 0) {
 575                 ret = eh_set_default_conf_tx_adapter(em_conf);
 576                 if (ret != 0)
 577                         return ret;
 578         }
 579
 580         return 0;
 581 }
 582
 583 static int
 584 eh_initialize_eventdev(struct eventmode_conf *em_conf)
 585 {
 586         struct rte_event_queue_conf eventq_conf = {0};
 587         struct rte_event_dev_info evdev_default_conf;
 588         struct rte_event_dev_config eventdev_conf;
 589         struct eventdev_params *eventdev_config;
 590         int nb_eventdev = em_conf->nb_eventdev;
 591         struct eh_event_link_info *link;
 592         uint8_t *queue = NULL;
 593         uint8_t eventdev_id;
 594         int nb_eventqueue;
 595         uint8_t i, j;
 596         int ret;
 597
 598         for (i = 0; i < nb_eventdev; i++) {
 599
 600                 /* Get eventdev config */
 601                 eventdev_config = &(em_conf->eventdev_config[i]);
 602
 603                 /* Get event dev ID */
 604                 eventdev_id = eventdev_config->eventdev_id;
 605
 606                 /* Get the number of queues */
 607                 nb_eventqueue = eventdev_config->nb_eventqueue;
 608
 609                 /* Reset the default conf */
 610                 memset(&evdev_default_conf, 0,
 611                         sizeof(struct rte_event_dev_info));
 612
 613                 /* Get default conf of eventdev */
 614                 ret = rte_event_dev_info_get(eventdev_id, &evdev_default_conf);
 615                 if (ret < 0) {
 616                         EH_LOG_ERR(
 617                                 "Error in getting event device info[devID:%d]",
 618                                 eventdev_id);
 619                         return ret;
 620                 }
 621
 622                 memset(&eventdev_conf, 0, sizeof(struct rte_event_dev_config));
 623                 eventdev_conf.nb_events_limit =
 624                                 evdev_default_conf.max_num_events;
 625                 eventdev_conf.nb_event_queues = nb_eventqueue;
 626                 eventdev_conf.nb_event_ports =
 627                                 eventdev_config->nb_eventport;
 628                 eventdev_conf.nb_event_queue_flows =
 629                                 evdev_default_conf.max_event_queue_flows;
 630                 eventdev_conf.nb_event_port_dequeue_depth =
 631                                 evdev_default_conf.max_event_port_dequeue_depth;
 632                 eventdev_conf.nb_event_port_enqueue_depth =
 633                                 evdev_default_conf.max_event_port_enqueue_depth;
 634
 635                 /* Configure event device */
 636                 ret = rte_event_dev_configure(eventdev_id, &eventdev_conf);
 637                 if (ret < 0) {
 638                         EH_LOG_ERR("Error in configuring event device");
 639                         return ret;
 640                 }
 641
 642                 /* Configure event queues */
 643                 for (j = 0; j < nb_eventqueue; j++) {
 644
 645                         memset(&eventq_conf, 0,
 646                                         sizeof(struct rte_event_queue_conf));
 647
 648                         /* Per event dev queues can be ATQ or SINGLE LINK */
 649                         eventq_conf.event_queue_cfg =
 650                                         eventdev_config->ev_queue_mode;
 651                         /*
 652                          * All queues need to be set with sched_type as
 653                          * schedule type for the application stage. One
 654                          * queue would be reserved for the final eth tx
 655                          * stage if event device does not have internal
 656                          * ports. This will be an atomic queue.
 657                          */
 658                         if (!eventdev_config->all_internal_ports &&
 659                             j == nb_eventqueue-1) {
 660                                 eventq_conf.schedule_type =
 661                                         RTE_SCHED_TYPE_ATOMIC;
 662                         } else {
 663                                 eventq_conf.schedule_type =
 664                                         em_conf->ext_params.sched_type;
 665                         }
 666
 667                         /* Set max atomic flows to 1024 */
 668                         eventq_conf.nb_atomic_flows = 1024;
 669                         eventq_conf.nb_atomic_order_sequences = 1024;
 670
 671                         /* Setup the queue */
 672                         ret = rte_event_queue_setup(eventdev_id, j,
 673                                         &eventq_conf);
 674                         if (ret < 0) {
 675                                 EH_LOG_ERR("Failed to setup event queue %d",
 676                                            ret);
 677                                 return ret;
 678                         }
 679                 }
 680
 681                 /* Configure event ports */
 682                 for (j = 0; j <  eventdev_config->nb_eventport; j++) {
 683                         ret = rte_event_port_setup(eventdev_id, j, NULL);
 684                         if (ret < 0) {
 685                                 EH_LOG_ERR("Failed to setup event port %d",
 686                                            ret);
 687                                 return ret;
 688                         }
 689                 }
 690         }
 691
 692         /* Make event queue - event port link */
 693         for (j = 0; j <  em_conf->nb_link; j++) {
 694
 695                 /* Get link info */
 696                 link = &(em_conf->link[j]);
 697
 698                 /* Get event dev ID */
 699                 eventdev_id = link->eventdev_id;
 700
 701                 /*
 702                  * If "all_ev_queue_to_ev_port" params flag is selected, all
 703                  * queues need to be mapped to the port.
 704                  */
 705                 if (em_conf->ext_params.all_ev_queue_to_ev_port)
 706                         queue = NULL;
 707                 else
 708                         queue = &(link->eventq_id);
 709
 710                 /* Link queue to port */
 711                 ret = rte_event_port_link(eventdev_id, link->event_port_id,
 712                                 queue, NULL, 1);
 713                 if (ret < 0) {
 714                         EH_LOG_ERR("Failed to link event port %d", ret);
 715                         return ret;
 716                 }
 717         }
 718
 719         return 0;
 720 }
 721
 722 static int
 723 eh_start_eventdev(struct eventmode_conf *em_conf)
 724 {
 725         struct eventdev_params *eventdev_config;
 726         int nb_eventdev = em_conf->nb_eventdev;
 727         int i, ret;
 728
 729         /* Start event devices */
 730         for (i = 0; i < nb_eventdev; i++) {
 731
 732                 /* Get eventdev config */
 733                 eventdev_config = &(em_conf->eventdev_config[i]);
 734
 735                 ret = rte_event_dev_start(eventdev_config->eventdev_id);
 736                 if (ret < 0) {
 737                         EH_LOG_ERR("Failed to start event device %d, %d",
 738                                    i, ret);
 739                         return ret;
 740                 }
 741         }
 742         return 0;
 743 }
 744
 745 static int
 746 eh_event_vector_limits_validate(struct eventmode_conf *em_conf,
 747                                 uint8_t ev_dev_id, uint8_t ethdev_id)
 748 {
 749         struct rte_event_eth_rx_adapter_vector_limits limits = {0};
 750         uint16_t vector_size = em_conf->ext_params.vector_size;
 751         int ret;
 752
 753         ret = rte_event_eth_rx_adapter_vector_limits_get(ev_dev_id, ethdev_id,
 754                                                          &limits);
 755         if (ret) {
 756                 EH_LOG_ERR("failed to get vector limits");
 757                 return ret;
 758         }
 759
 760         if (vector_size < limits.min_sz || vector_size > limits.max_sz) {
 761                 EH_LOG_ERR("Vector size [%d] not within limits min[%d] max[%d]",
 762                            vector_size, limits.min_sz, limits.max_sz);
 763                 return -EINVAL;
 764         }
 765
 766         if (limits.log2_sz && !rte_is_power_of_2(vector_size)) {
 767                 EH_LOG_ERR("Vector size [%d] not power of 2", vector_size);
 768                 return -EINVAL;
 769         }
 770
 771         if (em_conf->vector_tmo_ns > limits.max_timeout_ns ||
 772             em_conf->vector_tmo_ns < limits.min_timeout_ns) {
 773                 EH_LOG_ERR("Vector timeout [%" PRIu64
 774                            "] not within limits max[%" PRIu64
 775                            "] min[%" PRIu64 "]",
 776                            em_conf->vector_tmo_ns,
 777                            limits.max_timeout_ns,
 778                            limits.min_timeout_ns);
 779                 return -EINVAL;
 780         }
 781         return 0;
 782 }
 783
 784 static int
 785 eh_rx_adapter_configure(struct eventmode_conf *em_conf,
 786                 struct rx_adapter_conf *adapter)
 787 {
 788         struct rte_event_eth_rx_adapter_queue_conf queue_conf = {0};
 789         struct rte_event_dev_info evdev_default_conf = {0};
 790         struct rte_event_port_conf port_conf = {0};
 791         struct rx_adapter_connection_info *conn;
 792         uint32_t service_id, socket_id, nb_elem;
 793         struct rte_mempool *vector_pool = NULL;
 794         uint32_t lcore_id = rte_lcore_id();
 795         uint8_t eventdev_id;
 796         int ret;
 797         int j;
 798
 799         /* Get event dev ID */
 800         eventdev_id = adapter->eventdev_id;
 801
 802         /* Get default configuration of event dev */
 803         ret = rte_event_dev_info_get(eventdev_id, &evdev_default_conf);
 804         if (ret < 0) {
 805                 EH_LOG_ERR("Failed to get event dev info %d", ret);
 806                 return ret;
 807         }
 808
 809         if (em_conf->ext_params.event_vector) {
 810                 socket_id = rte_lcore_to_socket_id(lcore_id);
 811                 nb_elem = (nb_bufs_in_pool / em_conf->ext_params.vector_size)
 812                           + 1;
 813
 814                 vector_pool = rte_event_vector_pool_create(
 815                         "vector_pool", nb_elem, 0,
 816                         em_conf->ext_params.vector_size,
 817                         socket_id);
 818                 if (vector_pool == NULL) {
 819                         EH_LOG_ERR("failed to create event vector pool");
 820                         return -ENOMEM;
 821                 }
 822         }
 823         /* Setup port conf */
 824         port_conf.new_event_threshold = 1200;
 825         port_conf.dequeue_depth =
 826                         evdev_default_conf.max_event_port_dequeue_depth;
 827         port_conf.enqueue_depth =
 828                         evdev_default_conf.max_event_port_enqueue_depth;
 829
 830         /* Create Rx adapter */
 831         ret = rte_event_eth_rx_adapter_create(adapter->adapter_id,
 832                         adapter->eventdev_id, &port_conf);
 833         if (ret < 0) {
 834                 EH_LOG_ERR("Failed to create rx adapter %d", ret);
 835                 return ret;
 836         }
 837
 838         /* Setup various connections in the adapter */
 839         for (j = 0; j < adapter->nb_connections; j++) {
 840                 /* Get connection */
 841                 conn = &(adapter->conn[j]);
 842
 843                 /* Setup queue conf */
 844                 queue_conf.ev.queue_id = conn->eventq_id;
 845                 queue_conf.ev.sched_type = em_conf->ext_params.sched_type;
 846                 queue_conf.ev.event_type = RTE_EVENT_TYPE_ETHDEV;
 847
 848                 if (em_conf->ext_params.event_vector) {
 849                         ret = eh_event_vector_limits_validate(em_conf,
 850                                                               eventdev_id,
 851                                                               conn->ethdev_id);
 852                         if (ret)
 853                                 return ret;
 854
 855                         queue_conf.vector_sz = em_conf->ext_params.vector_size;
 856                         queue_conf.vector_timeout_ns = em_conf->vector_tmo_ns;
 857                         queue_conf.vector_mp = vector_pool;
 858                         queue_conf.rx_queue_flags =
 859                                 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR;
 860                 }
 861
 862                 /* Add queue to the adapter */
 863                 ret = rte_event_eth_rx_adapter_queue_add(adapter->adapter_id,
 864                                 conn->ethdev_id, conn->ethdev_rx_qid,
 865                                 &queue_conf);
 866                 if (ret < 0) {
 867                         EH_LOG_ERR("Failed to add eth queue to rx adapter %d",
 868                                    ret);
 869                         return ret;
 870                 }
 871         }
 872
 873         /* Get the service ID used by rx adapter */
 874         ret = rte_event_eth_rx_adapter_service_id_get(adapter->adapter_id,
 875                                                       &service_id);
 876         if (ret != -ESRCH && ret < 0) {
 877                 EH_LOG_ERR("Failed to get service id used by rx adapter %d",
 878                            ret);
 879                 return ret;
 880         }
 881
 882         rte_service_set_runstate_mapped_check(service_id, 0);
 883
 884         /* Start adapter */
 885         ret = rte_event_eth_rx_adapter_start(adapter->adapter_id);
 886         if (ret < 0) {
 887                 EH_LOG_ERR("Failed to start rx adapter %d", ret);
 888                 return ret;
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int
 895 eh_initialize_rx_adapter(struct eventmode_conf *em_conf)
 896 {
 897         struct rx_adapter_conf *adapter;
 898         int i, ret;
 899
 900         /* Configure rx adapters */
 901         for (i = 0; i < em_conf->nb_rx_adapter; i++) {
 902                 adapter = &(em_conf->rx_adapter[i]);
 903                 ret = eh_rx_adapter_configure(em_conf, adapter);
 904                 if (ret < 0) {
 905                         EH_LOG_ERR("Failed to configure rx adapter %d", ret);
 906                         return ret;
 907                 }
 908         }
 909         return 0;
 910 }
 911
 912 static int32_t
 913 eh_start_worker_eth_core(struct eventmode_conf *conf, uint32_t lcore_id)
 914 {
 915         uint32_t service_id[EVENT_MODE_MAX_ADAPTERS_PER_RX_CORE];
 916         struct rx_adapter_conf *rx_adapter;
 917         struct tx_adapter_conf *tx_adapter;
 918         int service_count = 0;
 919         int adapter_id;
 920         int32_t ret;
 921         int i;
 922
 923         EH_LOG_INFO("Entering eth_core processing on lcore %u", lcore_id);
 924
 925         /*
 926          * Parse adapter config to check which of all Rx adapters need
 927          * to be handled by this core.
 928          */
 929         for (i = 0; i < conf->nb_rx_adapter; i++) {
 930                 /* Check if we have exceeded the max allowed */
 931                 if (service_count > EVENT_MODE_MAX_ADAPTERS_PER_RX_CORE) {
 932                         EH_LOG_ERR(
 933                               "Exceeded the max allowed adapters per rx core");
 934                         break;
 935                 }
 936
 937                 rx_adapter = &(conf->rx_adapter[i]);
 938                 if (rx_adapter->rx_core_id != lcore_id)
 939                         continue;
 940
 941                 /* Adapter is handled by this core */
 942                 adapter_id = rx_adapter->adapter_id;
 943
 944                 /* Get the service ID for the adapters */
 945                 ret = rte_event_eth_rx_adapter_service_id_get(adapter_id,
 946                                 &(service_id[service_count]));
 947
 948                 if (ret != -ESRCH && ret < 0) {
 949                         EH_LOG_ERR(
 950                                 "Failed to get service id used by rx adapter");
 951                         return ret;
 952                 }
 953
 954                 /* Update service count */
 955                 service_count++;
 956         }
 957
 958         /*
 959          * Parse adapter config to see which of all Tx adapters need
 960          * to be handled by this core.
 961          */
 962         for (i = 0; i < conf->nb_tx_adapter; i++) {
 963                 /* Check if we have exceeded the max allowed */
 964                 if (service_count > EVENT_MODE_MAX_ADAPTERS_PER_TX_CORE) {
 965                         EH_LOG_ERR(
 966                                 "Exceeded the max allowed adapters per tx core");
 967                         break;
 968                 }
 969
 970                 tx_adapter = &conf->tx_adapter[i];
 971                 if (tx_adapter->tx_core_id != lcore_id)
 972                         continue;
 973
 974                 /* Adapter is handled by this core */
 975                 adapter_id = tx_adapter->adapter_id;
 976
 977                 /* Get the service ID for the adapters */
 978                 ret = rte_event_eth_tx_adapter_service_id_get(adapter_id,
 979                                 &(service_id[service_count]));
 980
 981                 if (ret != -ESRCH && ret < 0) {
 982                         EH_LOG_ERR(
 983                                 "Failed to get service id used by tx adapter");
 984                         return ret;
 985                 }
 986
 987                 /* Update service count */
 988                 service_count++;
 989         }
 990
 991         eth_core_running = true;
 992
 993         while (eth_core_running) {
 994                 for (i = 0; i < service_count; i++) {
 995                         /* Initiate adapter service */
 996                         rte_service_run_iter_on_app_lcore(service_id[i], 0);
 997                 }
 998         }
 999
1000         return 0;
1001 }
1002
1003 static int32_t
1004 eh_stop_worker_eth_core(void)
1005 {
1006         if (eth_core_running) {
1007                 EH_LOG_INFO("Stopping eth cores");
1008                 eth_core_running = false;
1009         }
1010         return 0;
1011 }
1012
1013 static struct eh_app_worker_params *
1014 eh_find_worker(uint32_t lcore_id, struct eh_conf *conf,
1015                 struct eh_app_worker_params *app_wrkrs, uint8_t nb_wrkr_param)
1016 {
1017         struct eh_app_worker_params curr_conf = { {{0} }, NULL};
1018         struct eh_event_link_info *link = NULL;
1019         struct eh_app_worker_params *tmp_wrkr;
1020         struct eventmode_conf *em_conf;
1021         uint8_t eventdev_id;
1022         int i;
1023
1024         /* Get eventmode config */
1025         em_conf = conf->mode_params;
1026
1027         /*
1028          * Use event device from the first lcore-event link.
1029          *
1030          * Assumption: All lcore-event links tied to a core are using the
1031          * same event device. In other words, one core would be polling on
1032          * queues of a single event device only.
1033          */
1034
1035         /* Get a link for this lcore */
1036         for (i = 0; i < em_conf->nb_link; i++) {
1037                 link = &(em_conf->link[i]);
1038                 if (link->lcore_id == lcore_id)
1039                         break;
1040         }
1041
1042         if (link == NULL) {
1043                 EH_LOG_ERR("No valid link found for lcore %d", lcore_id);
1044                 return NULL;
1045         }
1046
1047         /* Get event dev ID */
1048         eventdev_id = link->eventdev_id;
1049
1050         /* Populate the curr_conf with the capabilities */
1051
1052         /* Check for Tx internal port */
1053         if (eh_dev_has_tx_internal_port(eventdev_id))
1054                 curr_conf.cap.tx_internal_port = EH_TX_TYPE_INTERNAL_PORT;
1055         else
1056                 curr_conf.cap.tx_internal_port = EH_TX_TYPE_NO_INTERNAL_PORT;
1057
1058         /* Check for burst mode */
1059         if (eh_dev_has_burst_mode(eventdev_id))
1060                 curr_conf.cap.burst = EH_RX_TYPE_BURST;
1061         else
1062                 curr_conf.cap.burst = EH_RX_TYPE_NON_BURST;
1063
1064         curr_conf.cap.ipsec_mode = conf->ipsec_mode;
1065
1066         /* Parse the passed list and see if we have matching capabilities */
1067
1068         /* Initialize the pointer used to traverse the list */
1069         tmp_wrkr = app_wrkrs;
1070
1071         for (i = 0; i < nb_wrkr_param; i++, tmp_wrkr++) {
1072
1073                 /* Skip this if capabilities are not matching */
1074                 if (tmp_wrkr->cap.u64 != curr_conf.cap.u64)
1075                         continue;
1076
1077                 /* If the checks pass, we have a match */
1078                 return tmp_wrkr;
1079         }
1080
1081         return NULL;
1082 }
1083
1084 static int
1085 eh_verify_match_worker(struct eh_app_worker_params *match_wrkr)
1086 {
1087         /* Verify registered worker */
1088         if (match_wrkr->worker_thread == NULL) {
1089                 EH_LOG_ERR("No worker registered");
1090                 return 0;
1091         }
1092
1093         /* Success */
1094         return 1;
1095 }
1096
1097 static uint8_t
1098 eh_get_event_lcore_links(uint32_t lcore_id, struct eh_conf *conf,
1099                 struct eh_event_link_info **links)
1100 {
1101         struct eh_event_link_info *link_cache;
1102         struct eventmode_conf *em_conf = NULL;
1103         struct eh_event_link_info *link;
1104         uint8_t lcore_nb_link = 0;
1105         size_t single_link_size;
1106         size_t cache_size;
1107         int index = 0;
1108         int i;
1109
1110         if (conf == NULL || links == NULL) {
1111                 EH_LOG_ERR("Invalid args");
1112                 return -EINVAL;
1113         }
1114
1115         /* Get eventmode conf */
1116         em_conf = conf->mode_params;
1117
1118         if (em_conf == NULL) {
1119                 EH_LOG_ERR("Invalid event mode parameters");
1120                 return -EINVAL;
1121         }
1122
1123         /* Get the number of links registered */
1124         for (i = 0; i < em_conf->nb_link; i++) {
1125
1126                 /* Get link */
1127                 link = &(em_conf->link[i]);
1128
1129                 /* Check if we have link intended for this lcore */
1130                 if (link->lcore_id == lcore_id) {
1131
1132                         /* Update the number of links for this core */
1133                         lcore_nb_link++;
1134
1135                 }
1136         }
1137
1138         /* Compute size of one entry to be copied */
1139         single_link_size = sizeof(struct eh_event_link_info);
1140
1141         /* Compute size of the buffer required */
1142         cache_size = lcore_nb_link * sizeof(struct eh_event_link_info);
1143
1144         /* Compute size of the buffer required */
1145         link_cache = calloc(1, cache_size);
1146
1147         /* Get the number of links registered */
1148         for (i = 0; i < em_conf->nb_link; i++) {
1149
1150                 /* Get link */
1151                 link = &(em_conf->link[i]);
1152
1153                 /* Check if we have link intended for this lcore */
1154                 if (link->lcore_id == lcore_id) {
1155
1156                         /* Cache the link */
1157                         memcpy(&link_cache[index], link, single_link_size);
1158
1159                         /* Update index */
1160                         index++;
1161                 }
1162         }
1163
1164         /* Update the links for application to use the cached links */
1165         *links = link_cache;
1166
1167         /* Return the number of cached links */
1168         return lcore_nb_link;
1169 }
1170
1171 static int
1172 eh_tx_adapter_configure(struct eventmode_conf *em_conf,
1173                 struct tx_adapter_conf *adapter)
1174 {
1175         struct rte_event_dev_info evdev_default_conf = {0};
1176         struct rte_event_port_conf port_conf = {0};
1177         struct tx_adapter_connection_info *conn;
1178         struct eventdev_params *eventdev_config;
1179         uint8_t tx_port_id = 0;
1180         uint8_t eventdev_id;
1181         uint32_t service_id;
1182         int ret, j;
1183
1184         /* Get event dev ID */
1185         eventdev_id = adapter->eventdev_id;
1186
1187         /* Get event device conf */
1188         eventdev_config = eh_get_eventdev_params(em_conf, eventdev_id);
1189
1190         /* Create Tx adapter */
1191
1192         /* Get default configuration of event dev */
1193         ret = rte_event_dev_info_get(eventdev_id, &evdev_default_conf);
1194         if (ret < 0) {
1195                 EH_LOG_ERR("Failed to get event dev info %d", ret);
1196                 return ret;
1197         }
1198
1199         /* Setup port conf */
1200         port_conf.new_event_threshold =
1201                         evdev_default_conf.max_num_events;
1202         port_conf.dequeue_depth =
1203                         evdev_default_conf.max_event_port_dequeue_depth;
1204         port_conf.enqueue_depth =
1205                         evdev_default_conf.max_event_port_enqueue_depth;
1206
1207         /* Create adapter */
1208         ret = rte_event_eth_tx_adapter_create(adapter->adapter_id,
1209                         adapter->eventdev_id, &port_conf);
1210         if (ret < 0) {
1211                 EH_LOG_ERR("Failed to create tx adapter %d", ret);
1212                 return ret;
1213         }
1214
1215         /* Setup various connections in the adapter */
1216         for (j = 0; j < adapter->nb_connections; j++) {
1217
1218                 /* Get connection */
1219                 conn = &(adapter->conn[j]);
1220
1221                 /* Add queue to the adapter */
1222                 ret = rte_event_eth_tx_adapter_queue_add(adapter->adapter_id,
1223                                 conn->ethdev_id, conn->ethdev_tx_qid);
1224                 if (ret < 0) {
1225                         EH_LOG_ERR("Failed to add eth queue to tx adapter %d",
1226                                    ret);
1227                         return ret;
1228                 }
1229         }
1230
1231         /*
1232          * Check if Tx core is assigned. If Tx core is not assigned then
1233          * the adapter has internal port for submitting Tx packets and
1234          * Tx event queue & port setup is not required
1235          */
1236         if (adapter->tx_core_id == (uint32_t) (-1)) {
1237                 /* Internal port is present */
1238                 goto skip_tx_queue_port_setup;
1239         }
1240
1241         /* Setup Tx queue & port */
1242
1243         /* Get event port used by the adapter */
1244         ret = rte_event_eth_tx_adapter_event_port_get(
1245                         adapter->adapter_id, &tx_port_id);
1246         if (ret) {
1247                 EH_LOG_ERR("Failed to get tx adapter port id %d", ret);
1248                 return ret;
1249         }
1250
1251         /*
1252          * Tx event queue is reserved for Tx adapter. Unlink this queue
1253          * from all other ports
1254          *
1255          */
1256         for (j = 0; j < eventdev_config->nb_eventport; j++) {
1257                 rte_event_port_unlink(eventdev_id, j,
1258                                       &(adapter->tx_ev_queue), 1);
1259         }
1260
1261         /* Link Tx event queue to Tx port */
1262         ret = rte_event_port_link(eventdev_id, tx_port_id,
1263                         &(adapter->tx_ev_queue), NULL, 1);
1264         if (ret != 1) {
1265                 EH_LOG_ERR("Failed to link event queue to port");
1266                 return ret;
1267         }
1268
1269         /* Get the service ID used by Tx adapter */
1270         ret = rte_event_eth_tx_adapter_service_id_get(adapter->adapter_id,
1271                                                       &service_id);
1272         if (ret != -ESRCH && ret < 0) {
1273                 EH_LOG_ERR("Failed to get service id used by tx adapter %d",
1274                            ret);
1275                 return ret;
1276         }
1277
1278         rte_service_set_runstate_mapped_check(service_id, 0);
1279
1280 skip_tx_queue_port_setup:
1281         /* Start adapter */
1282         ret = rte_event_eth_tx_adapter_start(adapter->adapter_id);
1283         if (ret < 0) {
1284                 EH_LOG_ERR("Failed to start tx adapter %d", ret);
1285                 return ret;
1286         }
1287
1288         return 0;
1289 }
1290
1291 static int
1292 eh_initialize_tx_adapter(struct eventmode_conf *em_conf)
1293 {
1294         struct tx_adapter_conf *adapter;
1295         int i, ret;
1296
1297         /* Configure Tx adapters */
1298         for (i = 0; i < em_conf->nb_tx_adapter; i++) {
1299                 adapter = &(em_conf->tx_adapter[i]);
1300                 ret = eh_tx_adapter_configure(em_conf, adapter);
1301                 if (ret < 0) {
1302                         EH_LOG_ERR("Failed to configure tx adapter %d", ret);
1303                         return ret;
1304                 }
1305         }
1306         return 0;
1307 }
1308
1309 static void
1310 eh_display_operating_mode(struct eventmode_conf *em_conf)
1311 {
1312         char sched_types[][32] = {
1313                 "RTE_SCHED_TYPE_ORDERED",
1314                 "RTE_SCHED_TYPE_ATOMIC",
1315                 "RTE_SCHED_TYPE_PARALLEL",
1316         };
1317         EH_LOG_INFO("Operating mode:");
1318
1319         EH_LOG_INFO("\tScheduling type: \t%s",
1320                 sched_types[em_conf->ext_params.sched_type]);
1321
1322         EH_LOG_INFO("");
1323 }
1324
1325 static void
1326 eh_display_event_dev_conf(struct eventmode_conf *em_conf)
1327 {
1328         char queue_mode[][32] = {
1329                 "",
1330                 "ATQ (ALL TYPE QUEUE)",
1331                 "SINGLE LINK",
1332         };
1333         char print_buf[256] = { 0 };
1334         int i;
1335
1336         EH_LOG_INFO("Event Device Configuration:");
1337
1338         for (i = 0; i < em_conf->nb_eventdev; i++) {
1339                 sprintf(print_buf,
1340                         "\tDev ID: %-2d \tQueues: %-2d \tPorts: %-2d",
1341                         em_conf->eventdev_config[i].eventdev_id,
1342                         em_conf->eventdev_config[i].nb_eventqueue,
1343                         em_conf->eventdev_config[i].nb_eventport);
1344                 sprintf(print_buf + strlen(print_buf),
1345                         "\tQueue mode: %s",
1346                         queue_mode[em_conf->eventdev_config[i].ev_queue_mode]);
1347                 EH_LOG_INFO("%s", print_buf);
1348         }
1349         EH_LOG_INFO("");
1350 }
1351
1352 static void
1353 eh_display_rx_adapter_conf(struct eventmode_conf *em_conf)
1354 {
1355         int nb_rx_adapter = em_conf->nb_rx_adapter;
1356         struct rx_adapter_connection_info *conn;
1357         struct rx_adapter_conf *adapter;
1358         char print_buf[256] = { 0 };
1359         int i, j;
1360
1361         EH_LOG_INFO("Rx adapters configured: %d", nb_rx_adapter);
1362
1363         for (i = 0; i < nb_rx_adapter; i++) {
1364                 adapter = &(em_conf->rx_adapter[i]);
1365                 sprintf(print_buf,
1366                         "\tRx adapter ID: %-2d\tConnections: %-2d\tEvent dev ID: %-2d",
1367                         adapter->adapter_id,
1368                         adapter->nb_connections,
1369                         adapter->eventdev_id);
1370                 if (adapter->rx_core_id == (uint32_t)-1)
1371                         sprintf(print_buf + strlen(print_buf),
1372                                 "\tRx core: %-2s", "[INTERNAL PORT]");
1373                 else if (adapter->rx_core_id == RTE_MAX_LCORE)
1374                         sprintf(print_buf + strlen(print_buf),
1375                                 "\tRx core: %-2s", "[NONE]");
1376                 else
1377                         sprintf(print_buf + strlen(print_buf),
1378                                 "\tRx core: %-2d", adapter->rx_core_id);
1379
1380                 EH_LOG_INFO("%s", print_buf);
1381
1382                 for (j = 0; j < adapter->nb_connections; j++) {
1383                         conn = &(adapter->conn[j]);
1384
1385                         sprintf(print_buf,
1386                                 "\t\tEthdev ID: %-2d", conn->ethdev_id);
1387
1388                         if (conn->ethdev_rx_qid == -1)
1389                                 sprintf(print_buf + strlen(print_buf),
1390                                         "\tEth rx queue: %-2s", "ALL");
1391                         else
1392                                 sprintf(print_buf + strlen(print_buf),
1393                                         "\tEth rx queue: %-2d",
1394                                         conn->ethdev_rx_qid);
1395
1396                         sprintf(print_buf + strlen(print_buf),
1397                                 "\tEvent queue: %-2d", conn->eventq_id);
1398                         EH_LOG_INFO("%s", print_buf);
1399                 }
1400         }
1401         EH_LOG_INFO("");
1402 }
1403
1404 static void
1405 eh_display_tx_adapter_conf(struct eventmode_conf *em_conf)
1406 {
1407         int nb_tx_adapter = em_conf->nb_tx_adapter;
1408         struct tx_adapter_connection_info *conn;
1409         struct tx_adapter_conf *adapter;
1410         char print_buf[256] = { 0 };
1411         int i, j;
1412
1413         EH_LOG_INFO("Tx adapters configured: %d", nb_tx_adapter);
1414
1415         for (i = 0; i < nb_tx_adapter; i++) {
1416                 adapter = &(em_conf->tx_adapter[i]);
1417                 sprintf(print_buf,
1418                         "\tTx adapter ID: %-2d\tConnections: %-2d\tEvent dev ID: %-2d",
1419                         adapter->adapter_id,
1420                         adapter->nb_connections,
1421                         adapter->eventdev_id);
1422                 if (adapter->tx_core_id == (uint32_t)-1)
1423                         sprintf(print_buf + strlen(print_buf),
1424                                 "\tTx core: %-2s", "[INTERNAL PORT]");
1425                 else if (adapter->tx_core_id == RTE_MAX_LCORE)
1426                         sprintf(print_buf + strlen(print_buf),
1427                                 "\tTx core: %-2s", "[NONE]");
1428                 else
1429                         sprintf(print_buf + strlen(print_buf),
1430                                 "\tTx core: %-2d,\tInput event queue: %-2d",
1431                                 adapter->tx_core_id, adapter->tx_ev_queue);
1432
1433                 EH_LOG_INFO("%s", print_buf);
1434
1435                 for (j = 0; j < adapter->nb_connections; j++) {
1436                         conn = &(adapter->conn[j]);
1437
1438                         sprintf(print_buf,
1439                                 "\t\tEthdev ID: %-2d", conn->ethdev_id);
1440
1441                         if (conn->ethdev_tx_qid == -1)
1442                                 sprintf(print_buf + strlen(print_buf),
1443                                         "\tEth tx queue: %-2s", "ALL");
1444                         else
1445                                 sprintf(print_buf + strlen(print_buf),
1446                                         "\tEth tx queue: %-2d",
1447                                         conn->ethdev_tx_qid);
1448                         EH_LOG_INFO("%s", print_buf);
1449                 }
1450         }
1451         EH_LOG_INFO("");
1452 }
1453
1454 static void
1455 eh_display_link_conf(struct eventmode_conf *em_conf)
1456 {
1457         struct eh_event_link_info *link;
1458         char print_buf[256] = { 0 };
1459         int i;
1460
1461         EH_LOG_INFO("Links configured: %d", em_conf->nb_link);
1462
1463         for (i = 0; i < em_conf->nb_link; i++) {
1464                 link = &(em_conf->link[i]);
1465
1466                 sprintf(print_buf,
1467                         "\tEvent dev ID: %-2d\tEvent port: %-2d",
1468                         link->eventdev_id,
1469                         link->event_port_id);
1470
1471                 if (em_conf->ext_params.all_ev_queue_to_ev_port)
1472                         sprintf(print_buf + strlen(print_buf),
1473                                 "Event queue: %-2s\t", "ALL");
1474                 else
1475                         sprintf(print_buf + strlen(print_buf),
1476                                 "Event queue: %-2d\t", link->eventq_id);
1477
1478                 sprintf(print_buf + strlen(print_buf),
1479                         "Lcore: %-2d", link->lcore_id);
1480                 EH_LOG_INFO("%s", print_buf);
1481         }
1482         EH_LOG_INFO("");
1483 }
1484
1485 struct eh_conf *
1486 eh_conf_init(void)
1487 {
1488         struct eventmode_conf *em_conf = NULL;
1489         struct eh_conf *conf = NULL;
1490         unsigned int eth_core_id;
1491         void *bitmap = NULL;
1492         uint32_t nb_bytes;
1493
1494         /* Allocate memory for config */
1495         conf = calloc(1, sizeof(struct eh_conf));
1496         if (conf == NULL) {
1497                 EH_LOG_ERR("Failed to allocate memory for eventmode helper "
1498                            "config");
1499                 return NULL;
1500         }
1501
1502         /* Set default conf */
1503
1504         /* Packet transfer mode: poll */
1505         conf->mode = EH_PKT_TRANSFER_MODE_POLL;
1506         conf->ipsec_mode = EH_IPSEC_MODE_TYPE_APP;
1507
1508         /* Keep all ethernet ports enabled by default */
1509         conf->eth_portmask = -1;
1510
1511         /* Allocate memory for event mode params */
1512         conf->mode_params = calloc(1, sizeof(struct eventmode_conf));
1513         if (conf->mode_params == NULL) {
1514                 EH_LOG_ERR("Failed to allocate memory for event mode params");
1515                 goto free_conf;
1516         }
1517
1518         /* Get eventmode conf */
1519         em_conf = conf->mode_params;
1520
1521         /* Allocate and initialize bitmap for eth cores */
1522         nb_bytes = rte_bitmap_get_memory_footprint(RTE_MAX_LCORE);
1523         if (!nb_bytes) {
1524                 EH_LOG_ERR("Failed to get bitmap footprint");
1525                 goto free_em_conf;
1526         }
1527
1528         bitmap = rte_zmalloc("event-helper-ethcore-bitmap", nb_bytes,
1529                              RTE_CACHE_LINE_SIZE);
1530         if (!bitmap) {
1531                 EH_LOG_ERR("Failed to allocate memory for eth cores bitmap\n");
1532                 goto free_em_conf;
1533         }
1534
1535         em_conf->eth_core_mask = rte_bitmap_init(RTE_MAX_LCORE, bitmap,
1536                                                  nb_bytes);
1537         if (!em_conf->eth_core_mask) {
1538                 EH_LOG_ERR("Failed to initialize bitmap");
1539                 goto free_bitmap;
1540         }
1541
1542         /* Set schedule type as not set */
1543         em_conf->ext_params.sched_type = SCHED_TYPE_NOT_SET;
1544
1545         /* Set two cores as eth cores for Rx & Tx */
1546
1547         /* Use first core other than main core as Rx core */
1548         eth_core_id = rte_get_next_lcore(0,     /* curr core */
1549                                          1,     /* skip main core */
1550                                          0      /* wrap */);
1551
1552         rte_bitmap_set(em_conf->eth_core_mask, eth_core_id);
1553
1554         /* Use next core as Tx core */
1555         eth_core_id = rte_get_next_lcore(eth_core_id,   /* curr core */
1556                                          1,             /* skip main core */
1557                                          0              /* wrap */);
1558
1559         rte_bitmap_set(em_conf->eth_core_mask, eth_core_id);
1560
1561         em_conf->ext_params.vector_size = DEFAULT_VECTOR_SIZE;
1562         em_conf->vector_tmo_ns = DEFAULT_VECTOR_TMO;
1563
1564         return conf;
1565
1566 free_bitmap:
1567         rte_free(bitmap);
1568 free_em_conf:
1569         free(em_conf);
1570 free_conf:
1571         free(conf);
1572         return NULL;
1573 }
1574
1575 void
1576 eh_conf_uninit(struct eh_conf *conf)
1577 {
1578         struct eventmode_conf *em_conf = NULL;
1579
1580         if (!conf || !conf->mode_params)
1581                 return;
1582
1583         /* Get eventmode conf */
1584         em_conf = conf->mode_params;
1585
1586         /* Free evenmode configuration memory */
1587         rte_free(em_conf->eth_core_mask);
1588         free(em_conf);
1589         free(conf);
1590 }
1591
1592 void
1593 eh_display_conf(struct eh_conf *conf)
1594 {
1595         struct eventmode_conf *em_conf;
1596
1597         if (conf == NULL) {
1598                 EH_LOG_ERR("Invalid event helper configuration");
1599                 return;
1600         }
1601
1602         if (conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
1603                 return;
1604
1605         if (conf->mode_params == NULL) {
1606                 EH_LOG_ERR("Invalid event mode parameters");
1607                 return;
1608         }
1609
1610         /* Get eventmode conf */
1611         em_conf = (struct eventmode_conf *)(conf->mode_params);
1612
1613         /* Display user exposed operating modes */
1614         eh_display_operating_mode(em_conf);
1615
1616         /* Display event device conf */
1617         eh_display_event_dev_conf(em_conf);
1618
1619         /* Display Rx adapter conf */
1620         eh_display_rx_adapter_conf(em_conf);
1621
1622         /* Display Tx adapter conf */
1623         eh_display_tx_adapter_conf(em_conf);
1624
1625         /* Display event-lcore link */
1626         eh_display_link_conf(em_conf);
1627 }
1628
1629 int32_t
1630 eh_devs_init(struct eh_conf *conf)
1631 {
1632         struct eventmode_conf *em_conf;
1633         uint16_t port_id;
1634         int ret;
1635
1636         if (conf == NULL) {
1637                 EH_LOG_ERR("Invalid event helper configuration");
1638                 return -EINVAL;
1639         }
1640
1641         if (conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
1642                 return 0;
1643
1644         if (conf->mode_params == NULL) {
1645                 EH_LOG_ERR("Invalid event mode parameters");
1646                 return -EINVAL;
1647         }
1648
1649         /* Get eventmode conf */
1650         em_conf = conf->mode_params;
1651
1652         /* Eventmode conf would need eth portmask */
1653         em_conf->eth_portmask = conf->eth_portmask;
1654
1655         /* Validate the requested config */
1656         ret = eh_validate_conf(em_conf);
1657         if (ret < 0) {
1658                 EH_LOG_ERR("Failed to validate the requested config %d", ret);
1659                 return ret;
1660         }
1661
1662         /* Display the current configuration */
1663         eh_display_conf(conf);
1664
1665         /* Stop eth devices before setting up adapter */
1666         RTE_ETH_FOREACH_DEV(port_id) {
1667
1668                 /* Use only the ports enabled */
1669                 if ((conf->eth_portmask & (1 << port_id)) == 0)
1670                         continue;
1671
1672                 ret = rte_eth_dev_stop(port_id);
1673                 if (ret != 0) {
1674                         EH_LOG_ERR("Failed to stop port %u, err: %d",
1675                                         port_id, ret);
1676                         return ret;
1677                 }
1678         }
1679
1680         /* Setup eventdev */
1681         ret = eh_initialize_eventdev(em_conf);
1682         if (ret < 0) {
1683                 EH_LOG_ERR("Failed to initialize event dev %d", ret);
1684                 return ret;
1685         }
1686
1687         /* Setup Rx adapter */
1688         ret = eh_initialize_rx_adapter(em_conf);
1689         if (ret < 0) {
1690                 EH_LOG_ERR("Failed to initialize rx adapter %d", ret);
1691                 return ret;
1692         }
1693
1694         /* Setup Tx adapter */
1695         ret = eh_initialize_tx_adapter(em_conf);
1696         if (ret < 0) {
1697                 EH_LOG_ERR("Failed to initialize tx adapter %d", ret);
1698                 return ret;
1699         }
1700
1701         /* Start eventdev */
1702         ret = eh_start_eventdev(em_conf);
1703         if (ret < 0) {
1704                 EH_LOG_ERR("Failed to start event dev %d", ret);
1705                 return ret;
1706         }
1707
1708         /* Start eth devices after setting up adapter */
1709         RTE_ETH_FOREACH_DEV(port_id) {
1710
1711                 /* Use only the ports enabled */
1712                 if ((conf->eth_portmask & (1 << port_id)) == 0)
1713                         continue;
1714
1715                 ret = rte_eth_dev_start(port_id);
1716                 if (ret < 0) {
1717                         EH_LOG_ERR("Failed to start eth dev %d, %d",
1718                                    port_id, ret);
1719                         return ret;
1720                 }
1721         }
1722
1723         return 0;
1724 }
1725
1726 int32_t
1727 eh_devs_uninit(struct eh_conf *conf)
1728 {
1729         struct eventmode_conf *em_conf;
1730         int ret, i, j;
1731         uint16_t id;
1732
1733         if (conf == NULL) {
1734                 EH_LOG_ERR("Invalid event helper configuration");
1735                 return -EINVAL;
1736         }
1737
1738         if (conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
1739                 return 0;
1740
1741         if (conf->mode_params == NULL) {
1742                 EH_LOG_ERR("Invalid event mode parameters");
1743                 return -EINVAL;
1744         }
1745
1746         /* Get eventmode conf */
1747         em_conf = conf->mode_params;
1748
1749         /* Stop and release rx adapters */
1750         for (i = 0; i < em_conf->nb_rx_adapter; i++) {
1751
1752                 id = em_conf->rx_adapter[i].adapter_id;
1753                 ret = rte_event_eth_rx_adapter_stop(id);
1754                 if (ret < 0) {
1755                         EH_LOG_ERR("Failed to stop rx adapter %d", ret);
1756                         return ret;
1757                 }
1758
1759                 for (j = 0; j < em_conf->rx_adapter[i].nb_connections; j++) {
1760
1761                         ret = rte_event_eth_rx_adapter_queue_del(id,
1762                                 em_conf->rx_adapter[i].conn[j].ethdev_id, -1);
1763                         if (ret < 0) {
1764                                 EH_LOG_ERR(
1765                                        "Failed to remove rx adapter queues %d",
1766                                        ret);
1767                                 return ret;
1768                         }
1769                 }
1770
1771                 ret = rte_event_eth_rx_adapter_free(id);
1772                 if (ret < 0) {
1773                         EH_LOG_ERR("Failed to free rx adapter %d", ret);
1774                         return ret;
1775                 }
1776         }
1777
1778         /* Stop and release event devices */
1779         for (i = 0; i < em_conf->nb_eventdev; i++) {
1780
1781                 id = em_conf->eventdev_config[i].eventdev_id;
1782                 rte_event_dev_stop(id);
1783
1784                 ret = rte_event_dev_close(id);
1785                 if (ret < 0) {
1786                         EH_LOG_ERR("Failed to close event dev %d, %d", id, ret);
1787                         return ret;
1788                 }
1789         }
1790
1791         /* Stop and release tx adapters */
1792         for (i = 0; i < em_conf->nb_tx_adapter; i++) {
1793
1794                 id = em_conf->tx_adapter[i].adapter_id;
1795                 ret = rte_event_eth_tx_adapter_stop(id);
1796                 if (ret < 0) {
1797                         EH_LOG_ERR("Failed to stop tx adapter %d", ret);
1798                         return ret;
1799                 }
1800
1801                 for (j = 0; j < em_conf->tx_adapter[i].nb_connections; j++) {
1802
1803                         ret = rte_event_eth_tx_adapter_queue_del(id,
1804                                 em_conf->tx_adapter[i].conn[j].ethdev_id, -1);
1805                         if (ret < 0) {
1806                                 EH_LOG_ERR(
1807                                         "Failed to remove tx adapter queues %d",
1808                                         ret);
1809                                 return ret;
1810                         }
1811                 }
1812
1813                 ret = rte_event_eth_tx_adapter_free(id);
1814                 if (ret < 0) {
1815                         EH_LOG_ERR("Failed to free tx adapter %d", ret);
1816                         return ret;
1817                 }
1818         }
1819
1820         return 0;
1821 }
1822
1823 void
1824 eh_launch_worker(struct eh_conf *conf, struct eh_app_worker_params *app_wrkr,
1825                 uint8_t nb_wrkr_param)
1826 {
1827         struct eh_app_worker_params *match_wrkr;
1828         struct eh_event_link_info *links = NULL;
1829         struct eventmode_conf *em_conf;
1830         uint32_t lcore_id;
1831         uint8_t nb_links;
1832
1833         if (conf == NULL) {
1834                 EH_LOG_ERR("Invalid event helper configuration");
1835                 return;
1836         }
1837
1838         if (conf->mode_params == NULL) {
1839                 EH_LOG_ERR("Invalid event mode parameters");
1840                 return;
1841         }
1842
1843         /* Get eventmode conf */
1844         em_conf = conf->mode_params;
1845
1846         /* Get core ID */
1847         lcore_id = rte_lcore_id();
1848
1849         /* Check if this is eth core */
1850         if (rte_bitmap_get(em_conf->eth_core_mask, lcore_id)) {
1851                 eh_start_worker_eth_core(em_conf, lcore_id);
1852                 return;
1853         }
1854
1855         if (app_wrkr == NULL || nb_wrkr_param == 0) {
1856                 EH_LOG_ERR("Invalid args");
1857                 return;
1858         }
1859
1860         /*
1861          * This is a regular worker thread. The application registers
1862          * multiple workers with various capabilities. Run worker
1863          * based on the selected capabilities of the event
1864          * device configured.
1865          */
1866
1867         /* Get the first matching worker for the event device */
1868         match_wrkr = eh_find_worker(lcore_id, conf, app_wrkr, nb_wrkr_param);
1869         if (match_wrkr == NULL) {
1870                 EH_LOG_ERR("Failed to match worker registered for lcore %d",
1871                            lcore_id);
1872                 goto clean_and_exit;
1873         }
1874
1875         /* Verify sanity of the matched worker */
1876         if (eh_verify_match_worker(match_wrkr) != 1) {
1877                 EH_LOG_ERR("Failed to validate the matched worker");
1878                 goto clean_and_exit;
1879         }
1880
1881         /* Get worker links */
1882         nb_links = eh_get_event_lcore_links(lcore_id, conf, &links);
1883
1884         /* Launch the worker thread */
1885         match_wrkr->worker_thread(links, nb_links);
1886
1887         /* Free links info memory */
1888         free(links);
1889
1890 clean_and_exit:
1891
1892         /* Flag eth_cores to stop, if started */
1893         eh_stop_worker_eth_core();
1894 }
1895
1896 uint8_t
1897 eh_get_tx_queue(struct eh_conf *conf, uint8_t eventdev_id)
1898 {
1899         struct eventdev_params *eventdev_config;
1900         struct eventmode_conf *em_conf;
1901
1902         if (conf == NULL) {
1903                 EH_LOG_ERR("Invalid event helper configuration");
1904                 return -EINVAL;
1905         }
1906
1907         if (conf->mode_params == NULL) {
1908                 EH_LOG_ERR("Invalid event mode parameters");
1909                 return -EINVAL;
1910         }
1911
1912         /* Get eventmode conf */
1913         em_conf = conf->mode_params;
1914
1915         /* Get event device conf */
1916         eventdev_config = eh_get_eventdev_params(em_conf, eventdev_id);
1917
1918         if (eventdev_config == NULL) {
1919                 EH_LOG_ERR("Failed to read eventdev config");
1920                 return -EINVAL;
1921         }
1922
1923         /*
1924          * The last queue is reserved to be used as atomic queue for the
1925          * last stage (eth packet tx stage)
1926          */
1927         return eventdev_config->nb_eventqueue - 1;
1928 }