examples/l2fwd-event/l2fwd_event.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <stdbool.h>
   6 #include <getopt.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_ethdev.h>
  10 #include <rte_eventdev.h>
  11 #include <rte_event_eth_rx_adapter.h>
  12 #include <rte_event_eth_tx_adapter.h>
  13 #include <rte_lcore.h>
  14 #include <rte_malloc.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "l2fwd_event.h"
  18
  19 #define L2FWD_EVENT_SINGLE      0x1
  20 #define L2FWD_EVENT_BURST       0x2
  21 #define L2FWD_EVENT_TX_DIRECT   0x4
  22 #define L2FWD_EVENT_TX_ENQ      0x8
  23 #define L2FWD_EVENT_UPDT_MAC    0x10
  24
  25 static inline int
  26 l2fwd_event_service_enable(uint32_t service_id)
  27 {
  28         uint8_t min_service_count = UINT8_MAX;
  29         uint32_t slcore_array[RTE_MAX_LCORE];
  30         unsigned int slcore = 0;
  31         uint8_t service_count;
  32         int32_t slcore_count;
  33
  34         if (!rte_service_lcore_count())
  35                 return -ENOENT;
  36
  37         slcore_count = rte_service_lcore_list(slcore_array, RTE_MAX_LCORE);
  38         if (slcore_count < 0)
  39                 return -ENOENT;
  40         /* Get the core which has least number of services running. */
  41         while (slcore_count--) {
  42                 /* Reset default mapping */
  43                 if (rte_service_map_lcore_set(service_id,
  44                                         slcore_array[slcore_count], 0) != 0)
  45                         return -ENOENT;
  46                 service_count = rte_service_lcore_count_services(
  47                                 slcore_array[slcore_count]);
  48                 if (service_count < min_service_count) {
  49                         slcore = slcore_array[slcore_count];
  50                         min_service_count = service_count;
  51                 }
  52         }
  53         if (rte_service_map_lcore_set(service_id, slcore, 1) != 0)
  54                 return -ENOENT;
  55         rte_service_lcore_start(slcore);
  56
  57         return 0;
  58 }
  59
  60 void
  61 l2fwd_event_service_setup(struct l2fwd_resources *rsrc)
  62 {
  63         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
  64         struct rte_event_dev_info evdev_info;
  65         uint32_t service_id, caps;
  66         int ret, i;
  67
  68         /* Running eventdev scheduler service on service core. 8< */
  69         rte_event_dev_info_get(evt_rsrc->event_d_id, &evdev_info);
  70         if (!(evdev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)) {
  71                 ret = rte_event_dev_service_id_get(evt_rsrc->event_d_id,
  72                                 &service_id);
  73                 if (ret != -ESRCH && ret != 0)
  74                         rte_panic("Error in starting eventdev service\n");
  75                 l2fwd_event_service_enable(service_id);
  76         }
  77         /* >8 End of running eventdev scheduler service on service core. */
  78
  79         /* Gets service ID for RX/TX adapters. 8< */
  80         for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) {
  81                 ret = rte_event_eth_rx_adapter_caps_get(evt_rsrc->event_d_id,
  82                                 evt_rsrc->rx_adptr.rx_adptr[i], &caps);
  83                 if (ret < 0)
  84                         rte_panic("Failed to get Rx adapter[%d] caps\n",
  85                                   evt_rsrc->rx_adptr.rx_adptr[i]);
  86                 ret = rte_event_eth_rx_adapter_service_id_get(
  87                                 evt_rsrc->event_d_id,
  88                                 &service_id);
  89                 if (ret != -ESRCH && ret != 0)
  90                         rte_panic("Error in starting Rx adapter[%d] service\n",
  91                                   evt_rsrc->rx_adptr.rx_adptr[i]);
  92                 l2fwd_event_service_enable(service_id);
  93         }
  94
  95         for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) {
  96                 ret = rte_event_eth_tx_adapter_caps_get(evt_rsrc->event_d_id,
  97                                 evt_rsrc->tx_adptr.tx_adptr[i], &caps);
  98                 if (ret < 0)
  99                         rte_panic("Failed to get Rx adapter[%d] caps\n",
 100                                   evt_rsrc->tx_adptr.tx_adptr[i]);
 101                 ret = rte_event_eth_tx_adapter_service_id_get(
 102                                 evt_rsrc->event_d_id,
 103                                 &service_id);
 104                 if (ret != -ESRCH && ret != 0)
 105                         rte_panic("Error in starting Rx adapter[%d] service\n",
 106                                   evt_rsrc->tx_adptr.tx_adptr[i]);
 107                 l2fwd_event_service_enable(service_id);
 108         }
 109         /* >8 End of get service ID for RX/TX adapters. */
 110 }
 111
 112 static void
 113 l2fwd_event_capability_setup(struct l2fwd_event_resources *evt_rsrc)
 114 {
 115         uint32_t caps = 0;
 116         uint16_t i;
 117         int ret;
 118
 119         RTE_ETH_FOREACH_DEV(i) {
 120                 ret = rte_event_eth_tx_adapter_caps_get(0, i, &caps);
 121                 if (ret)
 122                         rte_panic("Invalid capability for Tx adptr port %d\n",
 123                                   i);
 124
 125                 evt_rsrc->tx_mode_q |= !(caps &
 126                                    RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT);
 127         }
 128
 129         if (evt_rsrc->tx_mode_q)
 130                 l2fwd_event_set_generic_ops(&evt_rsrc->ops);
 131         else
 132                 l2fwd_event_set_internal_port_ops(&evt_rsrc->ops);
 133 }
 134
 135 static __rte_noinline int
 136 l2fwd_get_free_event_port(struct l2fwd_event_resources *evt_rsrc)
 137 {
 138         static int index;
 139         int port_id;
 140
 141         rte_spinlock_lock(&evt_rsrc->evp.lock);
 142         if (index >= evt_rsrc->evp.nb_ports) {
 143                 printf("No free event port is available\n");
 144                 return -1;
 145         }
 146
 147         port_id = evt_rsrc->evp.event_p_id[index];
 148         index++;
 149         rte_spinlock_unlock(&evt_rsrc->evp.lock);
 150
 151         return port_id;
 152 }
 153
 154 static  __rte_always_inline void
 155 l2fwd_event_fwd(struct l2fwd_resources *rsrc, struct rte_event *ev,
 156                 const uint8_t tx_q_id, const uint64_t timer_period,
 157                 const uint32_t flags)
 158 {
 159         struct rte_mbuf *mbuf = ev->mbuf;
 160         uint16_t dst_port;
 161
 162         rte_prefetch0(rte_pktmbuf_mtod(mbuf, void *));
 163         dst_port = rsrc->dst_ports[mbuf->port];
 164
 165         if (timer_period > 0)
 166                 __atomic_fetch_add(&rsrc->port_stats[mbuf->port].rx,
 167                                 1, __ATOMIC_RELAXED);
 168         mbuf->port = dst_port;
 169
 170         if (flags & L2FWD_EVENT_UPDT_MAC)
 171                 l2fwd_mac_updating(mbuf, dst_port, &rsrc->eth_addr[dst_port]);
 172
 173         if (flags & L2FWD_EVENT_TX_ENQ) {
 174                 ev->queue_id = tx_q_id;
 175                 ev->op = RTE_EVENT_OP_FORWARD;
 176         }
 177
 178         if (flags & L2FWD_EVENT_TX_DIRECT)
 179                 rte_event_eth_tx_adapter_txq_set(mbuf, 0);
 180
 181         if (timer_period > 0)
 182                 __atomic_fetch_add(&rsrc->port_stats[mbuf->port].tx,
 183                                 1, __ATOMIC_RELAXED);
 184 }
 185
 186 static __rte_always_inline void
 187 l2fwd_event_loop_single(struct l2fwd_resources *rsrc,
 188                         const uint32_t flags)
 189 {
 190         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
 191         const int port_id = l2fwd_get_free_event_port(evt_rsrc);
 192         const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
 193                                         evt_rsrc->evq.nb_queues - 1];
 194         const uint64_t timer_period = rsrc->timer_period;
 195         const uint8_t event_d_id = evt_rsrc->event_d_id;
 196         struct rte_event ev;
 197
 198         if (port_id < 0)
 199                 return;
 200
 201         printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
 202                 rte_lcore_id());
 203
 204         while (!rsrc->force_quit) {
 205                 /* Read packet from eventdev */
 206                 if (!rte_event_dequeue_burst(event_d_id, port_id, &ev, 1, 0))
 207                         continue;
 208
 209                 l2fwd_event_fwd(rsrc, &ev, tx_q_id, timer_period, flags);
 210
 211                 if (flags & L2FWD_EVENT_TX_ENQ) {
 212                         while (rte_event_enqueue_burst(event_d_id, port_id,
 213                                                        &ev, 1) &&
 214                                         !rsrc->force_quit)
 215                                 ;
 216                 }
 217
 218                 if (flags & L2FWD_EVENT_TX_DIRECT) {
 219                         while (!rte_event_eth_tx_adapter_enqueue(event_d_id,
 220                                                                 port_id,
 221                                                                 &ev, 1, 0) &&
 222                                         !rsrc->force_quit)
 223                                 ;
 224                 }
 225         }
 226 }
 227
 228 static __rte_always_inline void
 229 l2fwd_event_loop_burst(struct l2fwd_resources *rsrc,
 230                        const uint32_t flags)
 231 {
 232         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
 233         const int port_id = l2fwd_get_free_event_port(evt_rsrc);
 234         const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
 235                                         evt_rsrc->evq.nb_queues - 1];
 236         const uint64_t timer_period = rsrc->timer_period;
 237         const uint8_t event_d_id = evt_rsrc->event_d_id;
 238         const uint8_t deq_len = evt_rsrc->deq_depth;
 239         struct rte_event ev[MAX_PKT_BURST];
 240         uint16_t nb_rx, nb_tx;
 241         uint8_t i;
 242
 243         if (port_id < 0)
 244                 return;
 245
 246         printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
 247                 rte_lcore_id());
 248
 249         while (!rsrc->force_quit) {
 250                 /* Read packet from eventdev. 8< */
 251                 nb_rx = rte_event_dequeue_burst(event_d_id, port_id, ev,
 252                                                 deq_len, 0);
 253                 if (nb_rx == 0)
 254                         continue;
 255
 256                 for (i = 0; i < nb_rx; i++) {
 257                         l2fwd_event_fwd(rsrc, &ev[i], tx_q_id, timer_period,
 258                                         flags);
 259                 }
 260                 /* >8 End of reading packets from eventdev. */
 261
 262                 if (flags & L2FWD_EVENT_TX_ENQ) {
 263                         /* Forwarding to destination ports. 8< */
 264                         nb_tx = rte_event_enqueue_burst(event_d_id, port_id,
 265                                                         ev, nb_rx);
 266                         while (nb_tx < nb_rx && !rsrc->force_quit)
 267                                 nb_tx += rte_event_enqueue_burst(event_d_id,
 268                                                 port_id, ev + nb_tx,
 269                                                 nb_rx - nb_tx);
 270                         /* >8 End of forwarding to destination ports. */
 271                 }
 272
 273                 if (flags & L2FWD_EVENT_TX_DIRECT) {
 274                         nb_tx = rte_event_eth_tx_adapter_enqueue(event_d_id,
 275                                                                  port_id, ev,
 276                                                                  nb_rx, 0);
 277                         while (nb_tx < nb_rx && !rsrc->force_quit)
 278                                 nb_tx += rte_event_eth_tx_adapter_enqueue(
 279                                                 event_d_id, port_id,
 280                                                 ev + nb_tx, nb_rx - nb_tx, 0);
 281                 }
 282         }
 283 }
 284
 285 static __rte_always_inline void
 286 l2fwd_event_loop(struct l2fwd_resources *rsrc,
 287                         const uint32_t flags)
 288 {
 289         if (flags & L2FWD_EVENT_SINGLE)
 290                 l2fwd_event_loop_single(rsrc, flags);
 291         if (flags & L2FWD_EVENT_BURST)
 292                 l2fwd_event_loop_burst(rsrc, flags);
 293 }
 294
 295 static void __rte_noinline
 296 l2fwd_event_main_loop_tx_d(struct l2fwd_resources *rsrc)
 297 {
 298         l2fwd_event_loop(rsrc,
 299                          L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
 300 }
 301
 302 static void __rte_noinline
 303 l2fwd_event_main_loop_tx_d_brst(struct l2fwd_resources *rsrc)
 304 {
 305         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
 306 }
 307
 308 static void __rte_noinline
 309 l2fwd_event_main_loop_tx_q(struct l2fwd_resources *rsrc)
 310 {
 311         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
 312 }
 313
 314 static void __rte_noinline
 315 l2fwd_event_main_loop_tx_q_brst(struct l2fwd_resources *rsrc)
 316 {
 317         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
 318 }
 319
 320 static void __rte_noinline
 321 l2fwd_event_main_loop_tx_d_mac(struct l2fwd_resources *rsrc)
 322 {
 323         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 324                         L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
 325 }
 326
 327 static void __rte_noinline
 328 l2fwd_event_main_loop_tx_d_brst_mac(struct l2fwd_resources *rsrc)
 329 {
 330         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 331                         L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
 332 }
 333
 334 static void __rte_noinline
 335 l2fwd_event_main_loop_tx_q_mac(struct l2fwd_resources *rsrc)
 336 {
 337         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 338                         L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
 339 }
 340
 341 static void __rte_noinline
 342 l2fwd_event_main_loop_tx_q_brst_mac(struct l2fwd_resources *rsrc)
 343 {
 344         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 345                         L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
 346 }
 347
 348 void
 349 l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
 350 {
 351         /* [MAC_UPDT][TX_MODE][BURST] */
 352         const event_loop_cb event_loop[2][2][2] = {
 353                 [0][0][0] = l2fwd_event_main_loop_tx_d,
 354                 [0][0][1] = l2fwd_event_main_loop_tx_d_brst,
 355                 [0][1][0] = l2fwd_event_main_loop_tx_q,
 356                 [0][1][1] = l2fwd_event_main_loop_tx_q_brst,
 357                 [1][0][0] = l2fwd_event_main_loop_tx_d_mac,
 358                 [1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
 359                 [1][1][0] = l2fwd_event_main_loop_tx_q_mac,
 360                 [1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
 361         };
 362         struct l2fwd_event_resources *evt_rsrc;
 363         uint32_t event_queue_cfg;
 364         int ret;
 365
 366         if (!rte_event_dev_count())
 367                 rte_panic("No Eventdev found\n");
 368
 369         evt_rsrc = rte_zmalloc("l2fwd_event",
 370                                  sizeof(struct l2fwd_event_resources), 0);
 371         if (evt_rsrc == NULL)
 372                 rte_panic("Failed to allocate memory\n");
 373
 374         rsrc->evt_rsrc = evt_rsrc;
 375
 376         /* Setup eventdev capability callbacks */
 377         l2fwd_event_capability_setup(evt_rsrc);
 378
 379         /* Event device configuration */
 380         event_queue_cfg = evt_rsrc->ops.event_device_setup(rsrc);
 381
 382         /* Event queue configuration */
 383         evt_rsrc->ops.event_queue_setup(rsrc, event_queue_cfg);
 384
 385         /* Event port configuration */
 386         evt_rsrc->ops.event_port_setup(rsrc);
 387
 388         /* Rx/Tx adapters configuration */
 389         evt_rsrc->ops.adapter_setup(rsrc);
 390
 391         /* Start event device */
 392         ret = rte_event_dev_start(evt_rsrc->event_d_id);
 393         if (ret < 0)
 394                 rte_panic("Error in starting eventdev\n");
 395
 396         evt_rsrc->ops.l2fwd_event_loop = event_loop
 397                                         [rsrc->mac_updating]
 398                                         [evt_rsrc->tx_mode_q]
 399                                         [evt_rsrc->has_burst];
 400 }