examples/l2fwd-event/l2fwd_event.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <stdbool.h>
   6 #include <getopt.h>
   7
   8 #include <rte_atomic.h>
   9 #include <rte_cycles.h>
  10 #include <rte_ethdev.h>
  11 #include <rte_eventdev.h>
  12 #include <rte_event_eth_rx_adapter.h>
  13 #include <rte_event_eth_tx_adapter.h>
  14 #include <rte_lcore.h>
  15 #include <rte_malloc.h>
  16 #include <rte_spinlock.h>
  17
  18 #include "l2fwd_event.h"
  19
  20 #define L2FWD_EVENT_SINGLE      0x1
  21 #define L2FWD_EVENT_BURST       0x2
  22 #define L2FWD_EVENT_TX_DIRECT   0x4
  23 #define L2FWD_EVENT_TX_ENQ      0x8
  24 #define L2FWD_EVENT_UPDT_MAC    0x10
  25
  26 static inline int
  27 l2fwd_event_service_enable(uint32_t service_id)
  28 {
  29         uint8_t min_service_count = UINT8_MAX;
  30         uint32_t slcore_array[RTE_MAX_LCORE];
  31         unsigned int slcore = 0;
  32         uint8_t service_count;
  33         int32_t slcore_count;
  34
  35         if (!rte_service_lcore_count())
  36                 return -ENOENT;
  37
  38         slcore_count = rte_service_lcore_list(slcore_array, RTE_MAX_LCORE);
  39         if (slcore_count < 0)
  40                 return -ENOENT;
  41         /* Get the core which has least number of services running. */
  42         while (slcore_count--) {
  43                 /* Reset default mapping */
  44                 if (rte_service_map_lcore_set(service_id,
  45                                         slcore_array[slcore_count], 0) != 0)
  46                         return -ENOENT;
  47                 service_count = rte_service_lcore_count_services(
  48                                 slcore_array[slcore_count]);
  49                 if (service_count < min_service_count) {
  50                         slcore = slcore_array[slcore_count];
  51                         min_service_count = service_count;
  52                 }
  53         }
  54         if (rte_service_map_lcore_set(service_id, slcore, 1) != 0)
  55                 return -ENOENT;
  56         rte_service_lcore_start(slcore);
  57
  58         return 0;
  59 }
  60
  61 void
  62 l2fwd_event_service_setup(struct l2fwd_resources *rsrc)
  63 {
  64         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
  65         struct rte_event_dev_info evdev_info;
  66         uint32_t service_id, caps;
  67         int ret, i;
  68
  69         /* Running eventdev scheduler service on service core. 8< */
  70         rte_event_dev_info_get(evt_rsrc->event_d_id, &evdev_info);
  71         if (!(evdev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)) {
  72                 ret = rte_event_dev_service_id_get(evt_rsrc->event_d_id,
  73                                 &service_id);
  74                 if (ret != -ESRCH && ret != 0)
  75                         rte_panic("Error in starting eventdev service\n");
  76                 l2fwd_event_service_enable(service_id);
  77         }
  78         /* >8 End of running eventdev scheduler service on service core. */
  79
  80         /* Gets service ID for RX/TX adapters. 8< */
  81         for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) {
  82                 ret = rte_event_eth_rx_adapter_caps_get(evt_rsrc->event_d_id,
  83                                 evt_rsrc->rx_adptr.rx_adptr[i], &caps);
  84                 if (ret < 0)
  85                         rte_panic("Failed to get Rx adapter[%d] caps\n",
  86                                   evt_rsrc->rx_adptr.rx_adptr[i]);
  87                 ret = rte_event_eth_rx_adapter_service_id_get(
  88                                 evt_rsrc->event_d_id,
  89                                 &service_id);
  90                 if (ret != -ESRCH && ret != 0)
  91                         rte_panic("Error in starting Rx adapter[%d] service\n",
  92                                   evt_rsrc->rx_adptr.rx_adptr[i]);
  93                 l2fwd_event_service_enable(service_id);
  94         }
  95
  96         for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) {
  97                 ret = rte_event_eth_tx_adapter_caps_get(evt_rsrc->event_d_id,
  98                                 evt_rsrc->tx_adptr.tx_adptr[i], &caps);
  99                 if (ret < 0)
 100                         rte_panic("Failed to get Rx adapter[%d] caps\n",
 101                                   evt_rsrc->tx_adptr.tx_adptr[i]);
 102                 ret = rte_event_eth_tx_adapter_service_id_get(
 103                                 evt_rsrc->event_d_id,
 104                                 &service_id);
 105                 if (ret != -ESRCH && ret != 0)
 106                         rte_panic("Error in starting Rx adapter[%d] service\n",
 107                                   evt_rsrc->tx_adptr.tx_adptr[i]);
 108                 l2fwd_event_service_enable(service_id);
 109         }
 110         /* >8 End of get service ID for RX/TX adapters. */
 111 }
 112
 113 static void
 114 l2fwd_event_capability_setup(struct l2fwd_event_resources *evt_rsrc)
 115 {
 116         uint32_t caps = 0;
 117         uint16_t i;
 118         int ret;
 119
 120         RTE_ETH_FOREACH_DEV(i) {
 121                 ret = rte_event_eth_tx_adapter_caps_get(0, i, &caps);
 122                 if (ret)
 123                         rte_panic("Invalid capability for Tx adptr port %d\n",
 124                                   i);
 125
 126                 evt_rsrc->tx_mode_q |= !(caps &
 127                                    RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT);
 128         }
 129
 130         if (evt_rsrc->tx_mode_q)
 131                 l2fwd_event_set_generic_ops(&evt_rsrc->ops);
 132         else
 133                 l2fwd_event_set_internal_port_ops(&evt_rsrc->ops);
 134 }
 135
 136 static __rte_noinline int
 137 l2fwd_get_free_event_port(struct l2fwd_event_resources *evt_rsrc)
 138 {
 139         static int index;
 140         int port_id;
 141
 142         rte_spinlock_lock(&evt_rsrc->evp.lock);
 143         if (index >= evt_rsrc->evp.nb_ports) {
 144                 printf("No free event port is available\n");
 145                 return -1;
 146         }
 147
 148         port_id = evt_rsrc->evp.event_p_id[index];
 149         index++;
 150         rte_spinlock_unlock(&evt_rsrc->evp.lock);
 151
 152         return port_id;
 153 }
 154
 155 static  __rte_always_inline void
 156 l2fwd_event_fwd(struct l2fwd_resources *rsrc, struct rte_event *ev,
 157                 const uint8_t tx_q_id, const uint64_t timer_period,
 158                 const uint32_t flags)
 159 {
 160         struct rte_mbuf *mbuf = ev->mbuf;
 161         uint16_t dst_port;
 162
 163         rte_prefetch0(rte_pktmbuf_mtod(mbuf, void *));
 164         dst_port = rsrc->dst_ports[mbuf->port];
 165
 166         if (timer_period > 0)
 167                 __atomic_fetch_add(&rsrc->port_stats[mbuf->port].rx,
 168                                 1, __ATOMIC_RELAXED);
 169         mbuf->port = dst_port;
 170
 171         if (flags & L2FWD_EVENT_UPDT_MAC)
 172                 l2fwd_mac_updating(mbuf, dst_port, &rsrc->eth_addr[dst_port]);
 173
 174         if (flags & L2FWD_EVENT_TX_ENQ) {
 175                 ev->queue_id = tx_q_id;
 176                 ev->op = RTE_EVENT_OP_FORWARD;
 177         }
 178
 179         if (flags & L2FWD_EVENT_TX_DIRECT)
 180                 rte_event_eth_tx_adapter_txq_set(mbuf, 0);
 181
 182         if (timer_period > 0)
 183                 __atomic_fetch_add(&rsrc->port_stats[mbuf->port].tx,
 184                                 1, __ATOMIC_RELAXED);
 185 }
 186
 187 static __rte_always_inline void
 188 l2fwd_event_loop_single(struct l2fwd_resources *rsrc,
 189                         const uint32_t flags)
 190 {
 191         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
 192         const int port_id = l2fwd_get_free_event_port(evt_rsrc);
 193         const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
 194                                         evt_rsrc->evq.nb_queues - 1];
 195         const uint64_t timer_period = rsrc->timer_period;
 196         const uint8_t event_d_id = evt_rsrc->event_d_id;
 197         struct rte_event ev;
 198
 199         if (port_id < 0)
 200                 return;
 201
 202         printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
 203                 rte_lcore_id());
 204
 205         while (!rsrc->force_quit) {
 206                 /* Read packet from eventdev */
 207                 if (!rte_event_dequeue_burst(event_d_id, port_id, &ev, 1, 0))
 208                         continue;
 209
 210                 l2fwd_event_fwd(rsrc, &ev, tx_q_id, timer_period, flags);
 211
 212                 if (flags & L2FWD_EVENT_TX_ENQ) {
 213                         while (rte_event_enqueue_burst(event_d_id, port_id,
 214                                                        &ev, 1) &&
 215                                         !rsrc->force_quit)
 216                                 ;
 217                 }
 218
 219                 if (flags & L2FWD_EVENT_TX_DIRECT) {
 220                         while (!rte_event_eth_tx_adapter_enqueue(event_d_id,
 221                                                                 port_id,
 222                                                                 &ev, 1, 0) &&
 223                                         !rsrc->force_quit)
 224                                 ;
 225                 }
 226         }
 227 }
 228
 229 static __rte_always_inline void
 230 l2fwd_event_loop_burst(struct l2fwd_resources *rsrc,
 231                        const uint32_t flags)
 232 {
 233         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
 234         const int port_id = l2fwd_get_free_event_port(evt_rsrc);
 235         const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
 236                                         evt_rsrc->evq.nb_queues - 1];
 237         const uint64_t timer_period = rsrc->timer_period;
 238         const uint8_t event_d_id = evt_rsrc->event_d_id;
 239         const uint8_t deq_len = evt_rsrc->deq_depth;
 240         struct rte_event ev[MAX_PKT_BURST];
 241         uint16_t nb_rx, nb_tx;
 242         uint8_t i;
 243
 244         if (port_id < 0)
 245                 return;
 246
 247         printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
 248                 rte_lcore_id());
 249
 250         while (!rsrc->force_quit) {
 251                 /* Read packet from eventdev. 8< */
 252                 nb_rx = rte_event_dequeue_burst(event_d_id, port_id, ev,
 253                                                 deq_len, 0);
 254                 if (nb_rx == 0)
 255                         continue;
 256
 257                 for (i = 0; i < nb_rx; i++) {
 258                         l2fwd_event_fwd(rsrc, &ev[i], tx_q_id, timer_period,
 259                                         flags);
 260                 }
 261                 /* >8 End of reading packets from eventdev. */
 262
 263                 if (flags & L2FWD_EVENT_TX_ENQ) {
 264                         /* Forwarding to destination ports. 8< */
 265                         nb_tx = rte_event_enqueue_burst(event_d_id, port_id,
 266                                                         ev, nb_rx);
 267                         while (nb_tx < nb_rx && !rsrc->force_quit)
 268                                 nb_tx += rte_event_enqueue_burst(event_d_id,
 269                                                 port_id, ev + nb_tx,
 270                                                 nb_rx - nb_tx);
 271                         /* >8 End of forwarding to destination ports. */
 272                 }
 273
 274                 if (flags & L2FWD_EVENT_TX_DIRECT) {
 275                         nb_tx = rte_event_eth_tx_adapter_enqueue(event_d_id,
 276                                                                  port_id, ev,
 277                                                                  nb_rx, 0);
 278                         while (nb_tx < nb_rx && !rsrc->force_quit)
 279                                 nb_tx += rte_event_eth_tx_adapter_enqueue(
 280                                                 event_d_id, port_id,
 281                                                 ev + nb_tx, nb_rx - nb_tx, 0);
 282                 }
 283         }
 284 }
 285
 286 static __rte_always_inline void
 287 l2fwd_event_loop(struct l2fwd_resources *rsrc,
 288                         const uint32_t flags)
 289 {
 290         if (flags & L2FWD_EVENT_SINGLE)
 291                 l2fwd_event_loop_single(rsrc, flags);
 292         if (flags & L2FWD_EVENT_BURST)
 293                 l2fwd_event_loop_burst(rsrc, flags);
 294 }
 295
 296 static void __rte_noinline
 297 l2fwd_event_main_loop_tx_d(struct l2fwd_resources *rsrc)
 298 {
 299         l2fwd_event_loop(rsrc,
 300                          L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
 301 }
 302
 303 static void __rte_noinline
 304 l2fwd_event_main_loop_tx_d_brst(struct l2fwd_resources *rsrc)
 305 {
 306         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
 307 }
 308
 309 static void __rte_noinline
 310 l2fwd_event_main_loop_tx_q(struct l2fwd_resources *rsrc)
 311 {
 312         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
 313 }
 314
 315 static void __rte_noinline
 316 l2fwd_event_main_loop_tx_q_brst(struct l2fwd_resources *rsrc)
 317 {
 318         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
 319 }
 320
 321 static void __rte_noinline
 322 l2fwd_event_main_loop_tx_d_mac(struct l2fwd_resources *rsrc)
 323 {
 324         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 325                         L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
 326 }
 327
 328 static void __rte_noinline
 329 l2fwd_event_main_loop_tx_d_brst_mac(struct l2fwd_resources *rsrc)
 330 {
 331         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 332                         L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
 333 }
 334
 335 static void __rte_noinline
 336 l2fwd_event_main_loop_tx_q_mac(struct l2fwd_resources *rsrc)
 337 {
 338         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 339                         L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
 340 }
 341
 342 static void __rte_noinline
 343 l2fwd_event_main_loop_tx_q_brst_mac(struct l2fwd_resources *rsrc)
 344 {
 345         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 346                         L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
 347 }
 348
 349 void
 350 l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
 351 {
 352         /* [MAC_UPDT][TX_MODE][BURST] */
 353         const event_loop_cb event_loop[2][2][2] = {
 354                 [0][0][0] = l2fwd_event_main_loop_tx_d,
 355                 [0][0][1] = l2fwd_event_main_loop_tx_d_brst,
 356                 [0][1][0] = l2fwd_event_main_loop_tx_q,
 357                 [0][1][1] = l2fwd_event_main_loop_tx_q_brst,
 358                 [1][0][0] = l2fwd_event_main_loop_tx_d_mac,
 359                 [1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
 360                 [1][1][0] = l2fwd_event_main_loop_tx_q_mac,
 361                 [1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
 362         };
 363         struct l2fwd_event_resources *evt_rsrc;
 364         uint32_t event_queue_cfg;
 365         int ret;
 366
 367         if (!rte_event_dev_count())
 368                 rte_panic("No Eventdev found\n");
 369
 370         evt_rsrc = rte_zmalloc("l2fwd_event",
 371                                  sizeof(struct l2fwd_event_resources), 0);
 372         if (evt_rsrc == NULL)
 373                 rte_panic("Failed to allocate memory\n");
 374
 375         rsrc->evt_rsrc = evt_rsrc;
 376
 377         /* Setup eventdev capability callbacks */
 378         l2fwd_event_capability_setup(evt_rsrc);
 379
 380         /* Event device configuration */
 381         event_queue_cfg = evt_rsrc->ops.event_device_setup(rsrc);
 382
 383         /* Event queue configuration */
 384         evt_rsrc->ops.event_queue_setup(rsrc, event_queue_cfg);
 385
 386         /* Event port configuration */
 387         evt_rsrc->ops.event_port_setup(rsrc);
 388
 389         /* Rx/Tx adapters configuration */
 390         evt_rsrc->ops.adapter_setup(rsrc);
 391
 392         /* Start event device */
 393         ret = rte_event_dev_start(evt_rsrc->event_d_id);
 394         if (ret < 0)
 395                 rte_panic("Error in starting eventdev\n");
 396
 397         evt_rsrc->ops.l2fwd_event_loop = event_loop
 398                                         [rsrc->mac_updating]
 399                                         [evt_rsrc->tx_mode_q]
 400                                         [evt_rsrc->has_burst];
 401 }