examples/l2fwd-event/l2fwd_event.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <stdbool.h>
   6 #include <getopt.h>
   7
   8 #include <rte_atomic.h>
   9 #include <rte_cycles.h>
  10 #include <rte_ethdev.h>
  11 #include <rte_eventdev.h>
  12 #include <rte_event_eth_rx_adapter.h>
  13 #include <rte_event_eth_tx_adapter.h>
  14 #include <rte_lcore.h>
  15 #include <rte_malloc.h>
  16 #include <rte_spinlock.h>
  17
  18 #include "l2fwd_event.h"
  19
  20 #define L2FWD_EVENT_SINGLE      0x1
  21 #define L2FWD_EVENT_BURST       0x2
  22 #define L2FWD_EVENT_TX_DIRECT   0x4
  23 #define L2FWD_EVENT_TX_ENQ      0x8
  24 #define L2FWD_EVENT_UPDT_MAC    0x10
  25
  26 static inline int
  27 l2fwd_event_service_enable(uint32_t service_id)
  28 {
  29         uint8_t min_service_count = UINT8_MAX;
  30         uint32_t slcore_array[RTE_MAX_LCORE];
  31         unsigned int slcore = 0;
  32         uint8_t service_count;
  33         int32_t slcore_count;
  34
  35         if (!rte_service_lcore_count())
  36                 return -ENOENT;
  37
  38         slcore_count = rte_service_lcore_list(slcore_array, RTE_MAX_LCORE);
  39         if (slcore_count < 0)
  40                 return -ENOENT;
  41         /* Get the core which has least number of services running. */
  42         while (slcore_count--) {
  43                 /* Reset default mapping */
  44                 rte_service_map_lcore_set(service_id,
  45                                 slcore_array[slcore_count], 0);
  46                 service_count = rte_service_lcore_count_services(
  47                                 slcore_array[slcore_count]);
  48                 if (service_count < min_service_count) {
  49                         slcore = slcore_array[slcore_count];
  50                         min_service_count = service_count;
  51                 }
  52         }
  53         if (rte_service_map_lcore_set(service_id, slcore, 1))
  54                 return -ENOENT;
  55         rte_service_lcore_start(slcore);
  56
  57         return 0;
  58 }
  59
  60 void
  61 l2fwd_event_service_setup(struct l2fwd_resources *rsrc)
  62 {
  63         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
  64         struct rte_event_dev_info evdev_info;
  65         uint32_t service_id, caps;
  66         int ret, i;
  67
  68         rte_event_dev_info_get(evt_rsrc->event_d_id, &evdev_info);
  69         if (evdev_info.event_dev_cap  & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED) {
  70                 ret = rte_event_dev_service_id_get(evt_rsrc->event_d_id,
  71                                 &service_id);
  72                 if (ret != -ESRCH && ret != 0)
  73                         rte_panic("Error in starting eventdev service\n");
  74                 l2fwd_event_service_enable(service_id);
  75         }
  76
  77         for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) {
  78                 ret = rte_event_eth_rx_adapter_caps_get(evt_rsrc->event_d_id,
  79                                 evt_rsrc->rx_adptr.rx_adptr[i], &caps);
  80                 if (ret < 0)
  81                         rte_panic("Failed to get Rx adapter[%d] caps\n",
  82                                   evt_rsrc->rx_adptr.rx_adptr[i]);
  83                 ret = rte_event_eth_rx_adapter_service_id_get(
  84                                 evt_rsrc->event_d_id,
  85                                 &service_id);
  86                 if (ret != -ESRCH && ret != 0)
  87                         rte_panic("Error in starting Rx adapter[%d] service\n",
  88                                   evt_rsrc->rx_adptr.rx_adptr[i]);
  89                 l2fwd_event_service_enable(service_id);
  90         }
  91
  92         for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) {
  93                 ret = rte_event_eth_tx_adapter_caps_get(evt_rsrc->event_d_id,
  94                                 evt_rsrc->tx_adptr.tx_adptr[i], &caps);
  95                 if (ret < 0)
  96                         rte_panic("Failed to get Rx adapter[%d] caps\n",
  97                                   evt_rsrc->tx_adptr.tx_adptr[i]);
  98                 ret = rte_event_eth_tx_adapter_service_id_get(
  99                                 evt_rsrc->event_d_id,
 100                                 &service_id);
 101                 if (ret != -ESRCH && ret != 0)
 102                         rte_panic("Error in starting Rx adapter[%d] service\n",
 103                                   evt_rsrc->tx_adptr.tx_adptr[i]);
 104                 l2fwd_event_service_enable(service_id);
 105         }
 106 }
 107
 108 static void
 109 l2fwd_event_capability_setup(struct l2fwd_event_resources *evt_rsrc)
 110 {
 111         uint32_t caps = 0;
 112         uint16_t i;
 113         int ret;
 114
 115         RTE_ETH_FOREACH_DEV(i) {
 116                 ret = rte_event_eth_tx_adapter_caps_get(0, i, &caps);
 117                 if (ret)
 118                         rte_panic("Invalid capability for Tx adptr port %d\n",
 119                                   i);
 120
 121                 evt_rsrc->tx_mode_q |= !(caps &
 122                                    RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT);
 123         }
 124
 125         if (evt_rsrc->tx_mode_q)
 126                 l2fwd_event_set_generic_ops(&evt_rsrc->ops);
 127         else
 128                 l2fwd_event_set_internal_port_ops(&evt_rsrc->ops);
 129 }
 130
 131 static __rte_noinline int
 132 l2fwd_get_free_event_port(struct l2fwd_event_resources *evt_rsrc)
 133 {
 134         static int index;
 135         int port_id;
 136
 137         rte_spinlock_lock(&evt_rsrc->evp.lock);
 138         if (index >= evt_rsrc->evp.nb_ports) {
 139                 printf("No free event port is available\n");
 140                 return -1;
 141         }
 142
 143         port_id = evt_rsrc->evp.event_p_id[index];
 144         index++;
 145         rte_spinlock_unlock(&evt_rsrc->evp.lock);
 146
 147         return port_id;
 148 }
 149
 150 static  __rte_always_inline void
 151 l2fwd_event_fwd(struct l2fwd_resources *rsrc, struct rte_event *ev,
 152                 const uint8_t tx_q_id, const uint64_t timer_period,
 153                 const uint32_t flags)
 154 {
 155         struct rte_mbuf *mbuf = ev->mbuf;
 156         uint16_t dst_port;
 157
 158         rte_prefetch0(rte_pktmbuf_mtod(mbuf, void *));
 159         dst_port = rsrc->dst_ports[mbuf->port];
 160
 161         if (timer_period > 0)
 162                 __atomic_fetch_add(&rsrc->port_stats[mbuf->port].rx,
 163                                 1, __ATOMIC_RELAXED);
 164         mbuf->port = dst_port;
 165
 166         if (flags & L2FWD_EVENT_UPDT_MAC)
 167                 l2fwd_mac_updating(mbuf, dst_port, &rsrc->eth_addr[dst_port]);
 168
 169         if (flags & L2FWD_EVENT_TX_ENQ) {
 170                 ev->queue_id = tx_q_id;
 171                 ev->op = RTE_EVENT_OP_FORWARD;
 172         }
 173
 174         if (flags & L2FWD_EVENT_TX_DIRECT)
 175                 rte_event_eth_tx_adapter_txq_set(mbuf, 0);
 176
 177         if (timer_period > 0)
 178                 __atomic_fetch_add(&rsrc->port_stats[mbuf->port].tx,
 179                                 1, __ATOMIC_RELAXED);
 180 }
 181
 182 static __rte_always_inline void
 183 l2fwd_event_loop_single(struct l2fwd_resources *rsrc,
 184                         const uint32_t flags)
 185 {
 186         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
 187         const int port_id = l2fwd_get_free_event_port(evt_rsrc);
 188         const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
 189                                         evt_rsrc->evq.nb_queues - 1];
 190         const uint64_t timer_period = rsrc->timer_period;
 191         const uint8_t event_d_id = evt_rsrc->event_d_id;
 192         struct rte_event ev;
 193
 194         if (port_id < 0)
 195                 return;
 196
 197         printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
 198                 rte_lcore_id());
 199
 200         while (!rsrc->force_quit) {
 201                 /* Read packet from eventdev */
 202                 if (!rte_event_dequeue_burst(event_d_id, port_id, &ev, 1, 0))
 203                         continue;
 204
 205                 l2fwd_event_fwd(rsrc, &ev, tx_q_id, timer_period, flags);
 206
 207                 if (flags & L2FWD_EVENT_TX_ENQ) {
 208                         while (rte_event_enqueue_burst(event_d_id, port_id,
 209                                                        &ev, 1) &&
 210                                         !rsrc->force_quit)
 211                                 ;
 212                 }
 213
 214                 if (flags & L2FWD_EVENT_TX_DIRECT) {
 215                         while (!rte_event_eth_tx_adapter_enqueue(event_d_id,
 216                                                                 port_id,
 217                                                                 &ev, 1, 0) &&
 218                                         !rsrc->force_quit)
 219                                 ;
 220                 }
 221         }
 222 }
 223
 224 static __rte_always_inline void
 225 l2fwd_event_loop_burst(struct l2fwd_resources *rsrc,
 226                        const uint32_t flags)
 227 {
 228         struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
 229         const int port_id = l2fwd_get_free_event_port(evt_rsrc);
 230         const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
 231                                         evt_rsrc->evq.nb_queues - 1];
 232         const uint64_t timer_period = rsrc->timer_period;
 233         const uint8_t event_d_id = evt_rsrc->event_d_id;
 234         const uint8_t deq_len = evt_rsrc->deq_depth;
 235         struct rte_event ev[MAX_PKT_BURST];
 236         uint16_t nb_rx, nb_tx;
 237         uint8_t i;
 238
 239         if (port_id < 0)
 240                 return;
 241
 242         printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
 243                 rte_lcore_id());
 244
 245         while (!rsrc->force_quit) {
 246                 /* Read packet from eventdev */
 247                 nb_rx = rte_event_dequeue_burst(event_d_id, port_id, ev,
 248                                                 deq_len, 0);
 249                 if (nb_rx == 0)
 250                         continue;
 251
 252                 for (i = 0; i < nb_rx; i++) {
 253                         l2fwd_event_fwd(rsrc, &ev[i], tx_q_id, timer_period,
 254                                         flags);
 255                 }
 256
 257                 if (flags & L2FWD_EVENT_TX_ENQ) {
 258                         nb_tx = rte_event_enqueue_burst(event_d_id, port_id,
 259                                                         ev, nb_rx);
 260                         while (nb_tx < nb_rx && !rsrc->force_quit)
 261                                 nb_tx += rte_event_enqueue_burst(event_d_id,
 262                                                 port_id, ev + nb_tx,
 263                                                 nb_rx - nb_tx);
 264                 }
 265
 266                 if (flags & L2FWD_EVENT_TX_DIRECT) {
 267                         nb_tx = rte_event_eth_tx_adapter_enqueue(event_d_id,
 268                                                                  port_id, ev,
 269                                                                  nb_rx, 0);
 270                         while (nb_tx < nb_rx && !rsrc->force_quit)
 271                                 nb_tx += rte_event_eth_tx_adapter_enqueue(
 272                                                 event_d_id, port_id,
 273                                                 ev + nb_tx, nb_rx - nb_tx, 0);
 274                 }
 275         }
 276 }
 277
 278 static __rte_always_inline void
 279 l2fwd_event_loop(struct l2fwd_resources *rsrc,
 280                         const uint32_t flags)
 281 {
 282         if (flags & L2FWD_EVENT_SINGLE)
 283                 l2fwd_event_loop_single(rsrc, flags);
 284         if (flags & L2FWD_EVENT_BURST)
 285                 l2fwd_event_loop_burst(rsrc, flags);
 286 }
 287
 288 static void __rte_noinline
 289 l2fwd_event_main_loop_tx_d(struct l2fwd_resources *rsrc)
 290 {
 291         l2fwd_event_loop(rsrc,
 292                          L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
 293 }
 294
 295 static void __rte_noinline
 296 l2fwd_event_main_loop_tx_d_brst(struct l2fwd_resources *rsrc)
 297 {
 298         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
 299 }
 300
 301 static void __rte_noinline
 302 l2fwd_event_main_loop_tx_q(struct l2fwd_resources *rsrc)
 303 {
 304         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
 305 }
 306
 307 static void __rte_noinline
 308 l2fwd_event_main_loop_tx_q_brst(struct l2fwd_resources *rsrc)
 309 {
 310         l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
 311 }
 312
 313 static void __rte_noinline
 314 l2fwd_event_main_loop_tx_d_mac(struct l2fwd_resources *rsrc)
 315 {
 316         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 317                         L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
 318 }
 319
 320 static void __rte_noinline
 321 l2fwd_event_main_loop_tx_d_brst_mac(struct l2fwd_resources *rsrc)
 322 {
 323         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 324                         L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
 325 }
 326
 327 static void __rte_noinline
 328 l2fwd_event_main_loop_tx_q_mac(struct l2fwd_resources *rsrc)
 329 {
 330         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 331                         L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
 332 }
 333
 334 static void __rte_noinline
 335 l2fwd_event_main_loop_tx_q_brst_mac(struct l2fwd_resources *rsrc)
 336 {
 337         l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
 338                         L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
 339 }
 340
 341 void
 342 l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
 343 {
 344         /* [MAC_UPDT][TX_MODE][BURST] */
 345         const event_loop_cb event_loop[2][2][2] = {
 346                 [0][0][0] = l2fwd_event_main_loop_tx_d,
 347                 [0][0][1] = l2fwd_event_main_loop_tx_d_brst,
 348                 [0][1][0] = l2fwd_event_main_loop_tx_q,
 349                 [0][1][1] = l2fwd_event_main_loop_tx_q_brst,
 350                 [1][0][0] = l2fwd_event_main_loop_tx_d_mac,
 351                 [1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
 352                 [1][1][0] = l2fwd_event_main_loop_tx_q_mac,
 353                 [1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
 354         };
 355         struct l2fwd_event_resources *evt_rsrc;
 356         uint32_t event_queue_cfg;
 357         int ret;
 358
 359         if (!rte_event_dev_count())
 360                 rte_panic("No Eventdev found\n");
 361
 362         evt_rsrc = rte_zmalloc("l2fwd_event",
 363                                  sizeof(struct l2fwd_event_resources), 0);
 364         if (evt_rsrc == NULL)
 365                 rte_panic("Failed to allocate memory\n");
 366
 367         rsrc->evt_rsrc = evt_rsrc;
 368
 369         /* Setup eventdev capability callbacks */
 370         l2fwd_event_capability_setup(evt_rsrc);
 371
 372         /* Event device configuration */
 373         event_queue_cfg = evt_rsrc->ops.event_device_setup(rsrc);
 374
 375         /* Event queue configuration */
 376         evt_rsrc->ops.event_queue_setup(rsrc, event_queue_cfg);
 377
 378         /* Event port configuration */
 379         evt_rsrc->ops.event_port_setup(rsrc);
 380
 381         /* Rx/Tx adapters configuration */
 382         evt_rsrc->ops.adapter_setup(rsrc);
 383
 384         /* Start event device */
 385         ret = rte_event_dev_start(evt_rsrc->event_d_id);
 386         if (ret < 0)
 387                 rte_panic("Error in starting eventdev\n");
 388
 389         evt_rsrc->ops.l2fwd_event_loop = event_loop
 390                                         [rsrc->mac_updating]
 391                                         [evt_rsrc->tx_mode_q]
 392                                         [evt_rsrc->has_burst];
 393 }