1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation.
10 #include <rte_cycles.h>
11 #include <rte_common.h>
13 #include <rte_errno.h>
14 #include <ethdev_driver.h>
16 #include <rte_malloc.h>
17 #include <rte_service_component.h>
18 #include <rte_thash.h>
19 #include <rte_interrupts.h>
20 #include <rte_mbuf_dyn.h>
21 #include <rte_telemetry.h>
23 #include "rte_eventdev.h"
24 #include "eventdev_pmd.h"
25 #include "eventdev_trace.h"
26 #include "rte_event_eth_rx_adapter.h"
29 #define BLOCK_CNT_THRESHOLD 10
30 #define ETH_EVENT_BUFFER_SIZE (6*BATCH_SIZE)
31 #define MAX_VECTOR_SIZE 1024
32 #define MIN_VECTOR_SIZE 4
33 #define MAX_VECTOR_NS 1E9
34 #define MIN_VECTOR_NS 1E5
36 #define ETH_RX_ADAPTER_SERVICE_NAME_LEN 32
37 #define ETH_RX_ADAPTER_MEM_NAME_LEN 32
39 #define RSS_KEY_SIZE 40
40 /* value written to intr thread pipe to signal thread exit */
41 #define ETH_BRIDGE_INTR_THREAD_EXIT 1
42 /* Sentinel value to detect initialized file handle */
45 #define RXA_ADAPTER_ARRAY "rte_event_eth_rx_adapter_array"
48 * Used to store port and queue ID of interrupting Rx queue
60 * There is an instance of this struct per polled Rx queue added to the
63 struct eth_rx_poll_entry {
64 /* Eth port to poll */
66 /* Eth rx queue to poll */
70 struct eth_rx_vector_data {
71 TAILQ_ENTRY(eth_rx_vector_data) next;
74 uint16_t max_vector_count;
77 uint64_t vector_timeout_ticks;
78 struct rte_mempool *vector_pool;
79 struct rte_event_vector *vector_ev;
80 } __rte_cache_aligned;
82 TAILQ_HEAD(eth_rx_vector_data_list, eth_rx_vector_data);
84 /* Instance per adapter */
85 struct eth_event_enqueue_buffer {
86 /* Count of events in this buffer */
88 /* Array of events in this buffer */
89 struct rte_event *events;
90 /* size of event buffer */
92 /* Event enqueue happens from head */
94 /* New packets from rte_eth_rx_burst is enqued from tail */
96 /* last element in the buffer before rollover */
101 struct event_eth_rx_adapter {
103 uint8_t rss_key_be[RSS_KEY_SIZE];
104 /* Event device identifier */
106 /* Event port identifier */
107 uint8_t event_port_id;
108 /* Flag indicating per rxq event buffer */
109 bool use_queue_event_buf;
110 /* Per ethernet device structure */
111 struct eth_device_info *eth_devices;
112 /* Lock to serialize config updates with service function */
113 rte_spinlock_t rx_lock;
114 /* Max mbufs processed in any service function invocation */
116 /* Receive queues that need to be polled */
117 struct eth_rx_poll_entry *eth_rx_poll;
118 /* Size of the eth_rx_poll array */
119 uint16_t num_rx_polled;
120 /* Weighted round robin schedule */
122 /* wrr_sched[] size */
124 /* Next entry in wrr[] to begin polling */
126 /* Event burst buffer */
127 struct eth_event_enqueue_buffer event_enqueue_buffer;
128 /* Vector enable flag */
130 /* Timestamp of previous vector expiry list traversal */
131 uint64_t prev_expiry_ts;
132 /* Minimum ticks to wait before traversing expiry list */
133 uint64_t vector_tmo_ticks;
135 struct eth_rx_vector_data_list vector_list;
136 /* Per adapter stats */
137 struct rte_event_eth_rx_adapter_stats stats;
138 /* Block count, counts up to BLOCK_CNT_THRESHOLD */
139 uint16_t enq_block_count;
141 uint64_t rx_enq_block_start_ts;
142 /* epoll fd used to wait for Rx interrupts */
144 /* Num of interrupt driven interrupt queues */
145 uint32_t num_rx_intr;
146 /* Used to send <dev id, queue id> of interrupting Rx queues from
147 * the interrupt thread to the Rx thread
149 struct rte_ring *intr_ring;
150 /* Rx Queue data (dev id, queue id) for the last non-empty
154 /* queue_data is valid */
156 /* Interrupt ring lock, synchronizes Rx thread
157 * and interrupt thread
159 rte_spinlock_t intr_ring_lock;
160 /* event array passed to rte_poll_wait */
161 struct rte_epoll_event *epoll_events;
162 /* Count of interrupt vectors in use */
163 uint32_t num_intr_vec;
164 /* Thread blocked on Rx interrupts */
165 pthread_t rx_intr_thread;
166 /* Configuration callback for rte_service configuration */
167 rte_event_eth_rx_adapter_conf_cb conf_cb;
168 /* Configuration callback argument */
170 /* Set if default_cb is being used */
172 /* Service initialization state */
173 uint8_t service_inited;
174 /* Total count of Rx queues in adapter */
176 /* Memory allocation name */
177 char mem_name[ETH_RX_ADAPTER_MEM_NAME_LEN];
178 /* Socket identifier cached from eventdev */
180 /* Per adapter EAL service */
182 /* Adapter started flag */
186 } __rte_cache_aligned;
189 struct eth_device_info {
190 struct rte_eth_dev *dev;
191 struct eth_rx_queue_info *rx_queue;
193 rte_event_eth_rx_adapter_cb_fn cb_fn;
194 /* Rx callback argument */
196 /* Set if ethdev->eventdev packet transfer uses a
199 uint8_t internal_event_port;
200 /* Set if the adapter is processing rx queues for
201 * this eth device and packet processing has been
202 * started, allows for the code to know if the PMD
203 * rx_adapter_stop callback needs to be invoked
205 uint8_t dev_rx_started;
206 /* Number of queues added for this device */
207 uint16_t nb_dev_queues;
208 /* Number of poll based queues
209 * If nb_rx_poll > 0, the start callback will
210 * be invoked if not already invoked
213 /* Number of interrupt based queues
214 * If nb_rx_intr > 0, the start callback will
215 * be invoked if not already invoked.
218 /* Number of queues that use the shared interrupt */
219 uint16_t nb_shared_intr;
220 /* sum(wrr(q)) for all queues within the device
221 * useful when deleting all device queues
224 /* Intr based queue index to start polling from, this is used
225 * if the number of shared interrupts is non-zero
228 /* Intr based queue indices */
229 uint16_t *intr_queue;
230 /* device generates per Rx queue interrupt for queue index
231 * for queue indices < RTE_MAX_RXTX_INTR_VEC_ID - 1
234 /* shared interrupt enabled */
235 int shared_intr_enabled;
239 struct eth_rx_queue_info {
240 int queue_enabled; /* True if added */
243 uint16_t wt; /* Polling weight */
244 uint32_t flow_id_mask; /* Set to ~0 if app provides flow id else 0 */
246 struct eth_rx_vector_data vector_data;
247 struct eth_event_enqueue_buffer *event_buf;
248 /* use adapter stats struct for queue level stats,
249 * as same stats need to be updated for adapter and queue
251 struct rte_event_eth_rx_adapter_stats *stats;
254 static struct event_eth_rx_adapter **event_eth_rx_adapter;
256 /* Enable dynamic timestamp field in mbuf */
257 static uint64_t event_eth_rx_timestamp_dynflag;
258 static int event_eth_rx_timestamp_dynfield_offset = -1;
260 static inline rte_mbuf_timestamp_t *
261 rxa_timestamp_dynfield(struct rte_mbuf *mbuf)
263 return RTE_MBUF_DYNFIELD(mbuf,
264 event_eth_rx_timestamp_dynfield_offset, rte_mbuf_timestamp_t *);
268 rxa_validate_id(uint8_t id)
270 return id < RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE;
273 static inline struct eth_event_enqueue_buffer *
274 rxa_event_buf_get(struct event_eth_rx_adapter *rx_adapter, uint16_t eth_dev_id,
275 uint16_t rx_queue_id,
276 struct rte_event_eth_rx_adapter_stats **stats)
278 if (rx_adapter->use_queue_event_buf) {
279 struct eth_device_info *dev_info =
280 &rx_adapter->eth_devices[eth_dev_id];
281 *stats = dev_info->rx_queue[rx_queue_id].stats;
282 return dev_info->rx_queue[rx_queue_id].event_buf;
284 *stats = &rx_adapter->stats;
285 return &rx_adapter->event_enqueue_buffer;
289 #define RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \
290 if (!rxa_validate_id(id)) { \
291 RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d\n", id); \
297 rxa_sw_adapter_queue_count(struct event_eth_rx_adapter *rx_adapter)
299 return rx_adapter->num_rx_polled + rx_adapter->num_rx_intr;
302 /* Greatest common divisor */
303 static uint16_t rxa_gcd_u16(uint16_t a, uint16_t b)
307 return r ? rxa_gcd_u16(b, r) : b;
310 /* Returns the next queue in the polling sequence
312 * http://kb.linuxvirtualserver.org/wiki/Weighted_Round-Robin_Scheduling
315 rxa_wrr_next(struct event_eth_rx_adapter *rx_adapter, unsigned int n, int *cw,
316 struct eth_rx_poll_entry *eth_rx_poll, uint16_t max_wt,
317 uint16_t gcd, int prev)
333 q = eth_rx_poll[i].eth_rx_qid;
334 d = eth_rx_poll[i].eth_dev_id;
335 w = rx_adapter->eth_devices[d].rx_queue[q].wt;
343 rxa_shared_intr(struct eth_device_info *dev_info,
348 if (dev_info->dev->intr_handle == NULL)
351 multi_intr_cap = rte_intr_cap_multiple(dev_info->dev->intr_handle);
352 return !multi_intr_cap ||
353 rx_queue_id >= RTE_MAX_RXTX_INTR_VEC_ID - 1;
357 rxa_intr_queue(struct eth_device_info *dev_info,
360 struct eth_rx_queue_info *queue_info;
362 queue_info = &dev_info->rx_queue[rx_queue_id];
363 return dev_info->rx_queue &&
364 !dev_info->internal_event_port &&
365 queue_info->queue_enabled && queue_info->wt == 0;
369 rxa_polled_queue(struct eth_device_info *dev_info,
372 struct eth_rx_queue_info *queue_info;
374 queue_info = &dev_info->rx_queue[rx_queue_id];
375 return !dev_info->internal_event_port &&
376 dev_info->rx_queue &&
377 queue_info->queue_enabled && queue_info->wt != 0;
380 /* Calculate change in number of vectors after Rx queue ID is add/deleted */
382 rxa_nb_intr_vect(struct eth_device_info *dev_info, int rx_queue_id, int add)
388 nbq = dev_info->dev->data->nb_rx_queues;
389 n = 0; /* non shared count */
390 s = 0; /* shared count */
392 if (rx_queue_id == -1) {
393 for (i = 0; i < nbq; i++) {
394 if (!rxa_shared_intr(dev_info, i))
395 n += add ? !rxa_intr_queue(dev_info, i) :
396 rxa_intr_queue(dev_info, i);
398 s += add ? !rxa_intr_queue(dev_info, i) :
399 rxa_intr_queue(dev_info, i);
403 if ((add && dev_info->nb_shared_intr == 0) ||
404 (!add && dev_info->nb_shared_intr))
408 if (!rxa_shared_intr(dev_info, rx_queue_id))
409 n = add ? !rxa_intr_queue(dev_info, rx_queue_id) :
410 rxa_intr_queue(dev_info, rx_queue_id);
412 n = add ? !dev_info->nb_shared_intr :
413 dev_info->nb_shared_intr == 1;
419 /* Calculate nb_rx_intr after deleting interrupt mode rx queues
422 rxa_calc_nb_post_intr_del(struct event_eth_rx_adapter *rx_adapter,
423 struct eth_device_info *dev_info, int rx_queue_id,
424 uint32_t *nb_rx_intr)
428 if (rx_queue_id == -1)
429 intr_diff = dev_info->nb_rx_intr;
431 intr_diff = rxa_intr_queue(dev_info, rx_queue_id);
433 *nb_rx_intr = rx_adapter->num_rx_intr - intr_diff;
436 /* Calculate nb_rx_* after adding interrupt mode rx queues, newly added
437 * interrupt queues could currently be poll mode Rx queues
440 rxa_calc_nb_post_add_intr(struct event_eth_rx_adapter *rx_adapter,
441 struct eth_device_info *dev_info, int rx_queue_id,
442 uint32_t *nb_rx_poll, uint32_t *nb_rx_intr,
447 uint32_t wrr_len_diff;
449 if (rx_queue_id == -1) {
450 intr_diff = dev_info->dev->data->nb_rx_queues -
451 dev_info->nb_rx_intr;
452 poll_diff = dev_info->nb_rx_poll;
453 wrr_len_diff = dev_info->wrr_len;
455 intr_diff = !rxa_intr_queue(dev_info, rx_queue_id);
456 poll_diff = rxa_polled_queue(dev_info, rx_queue_id);
457 wrr_len_diff = poll_diff ? dev_info->rx_queue[rx_queue_id].wt :
461 *nb_rx_intr = rx_adapter->num_rx_intr + intr_diff;
462 *nb_rx_poll = rx_adapter->num_rx_polled - poll_diff;
463 *nb_wrr = rx_adapter->wrr_len - wrr_len_diff;
466 /* Calculate size of the eth_rx_poll and wrr_sched arrays
467 * after deleting poll mode rx queues
470 rxa_calc_nb_post_poll_del(struct event_eth_rx_adapter *rx_adapter,
471 struct eth_device_info *dev_info, int rx_queue_id,
472 uint32_t *nb_rx_poll, uint32_t *nb_wrr)
475 uint32_t wrr_len_diff;
477 if (rx_queue_id == -1) {
478 poll_diff = dev_info->nb_rx_poll;
479 wrr_len_diff = dev_info->wrr_len;
481 poll_diff = rxa_polled_queue(dev_info, rx_queue_id);
482 wrr_len_diff = poll_diff ? dev_info->rx_queue[rx_queue_id].wt :
486 *nb_rx_poll = rx_adapter->num_rx_polled - poll_diff;
487 *nb_wrr = rx_adapter->wrr_len - wrr_len_diff;
490 /* Calculate nb_rx_* after adding poll mode rx queues
493 rxa_calc_nb_post_add_poll(struct event_eth_rx_adapter *rx_adapter,
494 struct eth_device_info *dev_info, int rx_queue_id,
495 uint16_t wt, uint32_t *nb_rx_poll,
496 uint32_t *nb_rx_intr, uint32_t *nb_wrr)
500 uint32_t wrr_len_diff;
502 if (rx_queue_id == -1) {
503 intr_diff = dev_info->nb_rx_intr;
504 poll_diff = dev_info->dev->data->nb_rx_queues -
505 dev_info->nb_rx_poll;
506 wrr_len_diff = wt*dev_info->dev->data->nb_rx_queues
509 intr_diff = rxa_intr_queue(dev_info, rx_queue_id);
510 poll_diff = !rxa_polled_queue(dev_info, rx_queue_id);
511 wrr_len_diff = rxa_polled_queue(dev_info, rx_queue_id) ?
512 wt - dev_info->rx_queue[rx_queue_id].wt :
516 *nb_rx_poll = rx_adapter->num_rx_polled + poll_diff;
517 *nb_rx_intr = rx_adapter->num_rx_intr - intr_diff;
518 *nb_wrr = rx_adapter->wrr_len + wrr_len_diff;
521 /* Calculate nb_rx_* after adding rx_queue_id */
523 rxa_calc_nb_post_add(struct event_eth_rx_adapter *rx_adapter,
524 struct eth_device_info *dev_info, int rx_queue_id,
525 uint16_t wt, uint32_t *nb_rx_poll, uint32_t *nb_rx_intr,
529 rxa_calc_nb_post_add_poll(rx_adapter, dev_info, rx_queue_id,
530 wt, nb_rx_poll, nb_rx_intr, nb_wrr);
532 rxa_calc_nb_post_add_intr(rx_adapter, dev_info, rx_queue_id,
533 nb_rx_poll, nb_rx_intr, nb_wrr);
536 /* Calculate nb_rx_* after deleting rx_queue_id */
538 rxa_calc_nb_post_del(struct event_eth_rx_adapter *rx_adapter,
539 struct eth_device_info *dev_info, int rx_queue_id,
540 uint32_t *nb_rx_poll, uint32_t *nb_rx_intr,
543 rxa_calc_nb_post_poll_del(rx_adapter, dev_info, rx_queue_id, nb_rx_poll,
545 rxa_calc_nb_post_intr_del(rx_adapter, dev_info, rx_queue_id,
550 * Allocate the rx_poll array
552 static struct eth_rx_poll_entry *
553 rxa_alloc_poll(struct event_eth_rx_adapter *rx_adapter, uint32_t num_rx_polled)
557 len = RTE_ALIGN(num_rx_polled * sizeof(*rx_adapter->eth_rx_poll),
558 RTE_CACHE_LINE_SIZE);
559 return rte_zmalloc_socket(rx_adapter->mem_name,
562 rx_adapter->socket_id);
566 * Allocate the WRR array
569 rxa_alloc_wrr(struct event_eth_rx_adapter *rx_adapter, int nb_wrr)
573 len = RTE_ALIGN(nb_wrr * sizeof(*rx_adapter->wrr_sched),
574 RTE_CACHE_LINE_SIZE);
575 return rte_zmalloc_socket(rx_adapter->mem_name,
578 rx_adapter->socket_id);
582 rxa_alloc_poll_arrays(struct event_eth_rx_adapter *rx_adapter, uint32_t nb_poll,
583 uint32_t nb_wrr, struct eth_rx_poll_entry **rx_poll,
584 uint32_t **wrr_sched)
593 *rx_poll = rxa_alloc_poll(rx_adapter, nb_poll);
594 if (*rx_poll == NULL) {
599 *wrr_sched = rxa_alloc_wrr(rx_adapter, nb_wrr);
600 if (*wrr_sched == NULL) {
607 /* Precalculate WRR polling sequence for all queues in rx_adapter */
609 rxa_calc_wrr_sequence(struct event_eth_rx_adapter *rx_adapter,
610 struct eth_rx_poll_entry *rx_poll, uint32_t *rx_wrr)
618 /* Initialize variables for calculation of wrr schedule */
619 uint16_t max_wrr_pos = 0;
620 unsigned int poll_q = 0;
627 /* Generate array of all queues to poll, the size of this
630 RTE_ETH_FOREACH_DEV(d) {
631 uint16_t nb_rx_queues;
632 struct eth_device_info *dev_info =
633 &rx_adapter->eth_devices[d];
634 nb_rx_queues = dev_info->dev->data->nb_rx_queues;
635 if (dev_info->rx_queue == NULL)
637 if (dev_info->internal_event_port)
639 dev_info->wrr_len = 0;
640 for (q = 0; q < nb_rx_queues; q++) {
641 struct eth_rx_queue_info *queue_info =
642 &dev_info->rx_queue[q];
645 if (!rxa_polled_queue(dev_info, q))
648 rx_poll[poll_q].eth_dev_id = d;
649 rx_poll[poll_q].eth_rx_qid = q;
651 dev_info->wrr_len += wt;
652 max_wt = RTE_MAX(max_wt, wt);
653 gcd = (gcd) ? rxa_gcd_u16(gcd, wt) : wt;
658 /* Generate polling sequence based on weights */
661 for (i = 0; i < max_wrr_pos; i++) {
662 rx_wrr[i] = rxa_wrr_next(rx_adapter, poll_q, &cw,
663 rx_poll, max_wt, gcd, prev);
669 rxa_mtoip(struct rte_mbuf *m, struct rte_ipv4_hdr **ipv4_hdr,
670 struct rte_ipv6_hdr **ipv6_hdr)
672 struct rte_ether_hdr *eth_hdr =
673 rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
674 struct rte_vlan_hdr *vlan_hdr;
679 switch (eth_hdr->ether_type) {
680 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
681 *ipv4_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
684 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
685 *ipv6_hdr = (struct rte_ipv6_hdr *)(eth_hdr + 1);
688 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
689 vlan_hdr = (struct rte_vlan_hdr *)(eth_hdr + 1);
690 switch (vlan_hdr->eth_proto) {
691 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
692 *ipv4_hdr = (struct rte_ipv4_hdr *)(vlan_hdr + 1);
694 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
695 *ipv6_hdr = (struct rte_ipv6_hdr *)(vlan_hdr + 1);
707 /* Calculate RSS hash for IPv4/6 */
708 static inline uint32_t
709 rxa_do_softrss(struct rte_mbuf *m, const uint8_t *rss_key_be)
713 struct rte_ipv4_tuple ipv4_tuple;
714 struct rte_ipv6_tuple ipv6_tuple;
715 struct rte_ipv4_hdr *ipv4_hdr;
716 struct rte_ipv6_hdr *ipv6_hdr;
718 rxa_mtoip(m, &ipv4_hdr, &ipv6_hdr);
721 ipv4_tuple.src_addr = rte_be_to_cpu_32(ipv4_hdr->src_addr);
722 ipv4_tuple.dst_addr = rte_be_to_cpu_32(ipv4_hdr->dst_addr);
724 input_len = RTE_THASH_V4_L3_LEN;
725 } else if (ipv6_hdr) {
726 rte_thash_load_v6_addrs(ipv6_hdr,
727 (union rte_thash_tuple *)&ipv6_tuple);
729 input_len = RTE_THASH_V6_L3_LEN;
733 return rte_softrss_be(tuple, input_len, rss_key_be);
737 rxa_enq_blocked(struct event_eth_rx_adapter *rx_adapter)
739 return !!rx_adapter->enq_block_count;
743 rxa_enq_block_start_ts(struct event_eth_rx_adapter *rx_adapter)
745 if (rx_adapter->rx_enq_block_start_ts)
748 rx_adapter->enq_block_count++;
749 if (rx_adapter->enq_block_count < BLOCK_CNT_THRESHOLD)
752 rx_adapter->rx_enq_block_start_ts = rte_get_tsc_cycles();
756 rxa_enq_block_end_ts(struct event_eth_rx_adapter *rx_adapter,
757 struct rte_event_eth_rx_adapter_stats *stats)
759 if (unlikely(!stats->rx_enq_start_ts))
760 stats->rx_enq_start_ts = rte_get_tsc_cycles();
762 if (likely(!rxa_enq_blocked(rx_adapter)))
765 rx_adapter->enq_block_count = 0;
766 if (rx_adapter->rx_enq_block_start_ts) {
767 stats->rx_enq_end_ts = rte_get_tsc_cycles();
768 stats->rx_enq_block_cycles += stats->rx_enq_end_ts -
769 rx_adapter->rx_enq_block_start_ts;
770 rx_adapter->rx_enq_block_start_ts = 0;
774 /* Enqueue buffered events to event device */
775 static inline uint16_t
776 rxa_flush_event_buffer(struct event_eth_rx_adapter *rx_adapter,
777 struct eth_event_enqueue_buffer *buf,
778 struct rte_event_eth_rx_adapter_stats *stats)
780 uint16_t count = buf->last ? buf->last - buf->head : buf->count;
785 uint16_t n = rte_event_enqueue_new_burst(rx_adapter->eventdev_id,
786 rx_adapter->event_port_id,
787 &buf->events[buf->head],
790 stats->rx_enq_retry++;
794 if (buf->last && n == count) {
797 n1 = rte_event_enqueue_new_burst(rx_adapter->eventdev_id,
798 rx_adapter->event_port_id,
803 stats->rx_enq_retry++;
811 n ? rxa_enq_block_end_ts(rx_adapter, stats) :
812 rxa_enq_block_start_ts(rx_adapter);
815 stats->rx_enq_count += n;
821 rxa_init_vector(struct event_eth_rx_adapter *rx_adapter,
822 struct eth_rx_vector_data *vec)
824 vec->vector_ev->nb_elem = 0;
825 vec->vector_ev->port = vec->port;
826 vec->vector_ev->queue = vec->queue;
827 vec->vector_ev->attr_valid = true;
828 TAILQ_INSERT_TAIL(&rx_adapter->vector_list, vec, next);
831 static inline uint16_t
832 rxa_create_event_vector(struct event_eth_rx_adapter *rx_adapter,
833 struct eth_rx_queue_info *queue_info,
834 struct eth_event_enqueue_buffer *buf,
835 struct rte_mbuf **mbufs, uint16_t num)
837 struct rte_event *ev = &buf->events[buf->count];
838 struct eth_rx_vector_data *vec;
839 uint16_t filled, space, sz;
842 vec = &queue_info->vector_data;
844 if (vec->vector_ev == NULL) {
845 if (rte_mempool_get(vec->vector_pool,
846 (void **)&vec->vector_ev) < 0) {
847 rte_pktmbuf_free_bulk(mbufs, num);
850 rxa_init_vector(rx_adapter, vec);
853 if (vec->vector_ev->nb_elem == vec->max_vector_count) {
855 ev->event = vec->event;
856 ev->vec = vec->vector_ev;
859 vec->vector_ev = NULL;
860 TAILQ_REMOVE(&rx_adapter->vector_list, vec, next);
861 if (rte_mempool_get(vec->vector_pool,
862 (void **)&vec->vector_ev) < 0) {
863 rte_pktmbuf_free_bulk(mbufs, num);
866 rxa_init_vector(rx_adapter, vec);
869 space = vec->max_vector_count - vec->vector_ev->nb_elem;
870 sz = num > space ? space : num;
871 memcpy(vec->vector_ev->mbufs + vec->vector_ev->nb_elem, mbufs,
872 sizeof(void *) * sz);
873 vec->vector_ev->nb_elem += sz;
876 vec->ts = rte_rdtsc();
879 if (vec->vector_ev->nb_elem == vec->max_vector_count) {
880 ev->event = vec->event;
881 ev->vec = vec->vector_ev;
884 vec->vector_ev = NULL;
885 TAILQ_REMOVE(&rx_adapter->vector_list, vec, next);
892 rxa_buffer_mbufs(struct event_eth_rx_adapter *rx_adapter, uint16_t eth_dev_id,
893 uint16_t rx_queue_id, struct rte_mbuf **mbufs, uint16_t num,
894 struct eth_event_enqueue_buffer *buf,
895 struct rte_event_eth_rx_adapter_stats *stats)
898 struct eth_device_info *dev_info =
899 &rx_adapter->eth_devices[eth_dev_id];
900 struct eth_rx_queue_info *eth_rx_queue_info =
901 &dev_info->rx_queue[rx_queue_id];
902 uint16_t new_tail = buf->tail;
903 uint64_t event = eth_rx_queue_info->event;
904 uint32_t flow_id_mask = eth_rx_queue_info->flow_id_mask;
905 struct rte_mbuf *m = mbufs[0];
911 uint64_t ts, ts_mask;
913 if (!eth_rx_queue_info->ena_vector) {
914 ts = m->ol_flags & event_eth_rx_timestamp_dynflag ?
915 0 : rte_get_tsc_cycles();
917 /* 0xffff ffff ffff ffff if RTE_MBUF_F_RX_TIMESTAMP is set,
920 ts_mask = (uint64_t)(!(m->ol_flags &
921 event_eth_rx_timestamp_dynflag)) - 1ULL;
923 /* 0xffff ffff if RTE_MBUF_F_RX_RSS_HASH is set, otherwise 0 */
924 rss_mask = ~(((m->ol_flags & RTE_MBUF_F_RX_RSS_HASH) != 0) - 1);
925 do_rss = !rss_mask && !eth_rx_queue_info->flow_id_mask;
926 for (i = 0; i < num; i++) {
927 struct rte_event *ev;
930 *rxa_timestamp_dynfield(m) = ts |
931 (*rxa_timestamp_dynfield(m) & ts_mask);
933 ev = &buf->events[new_tail];
935 rss = do_rss ? rxa_do_softrss(m, rx_adapter->rss_key_be)
938 ev->flow_id = (rss & ~flow_id_mask) |
939 (ev->flow_id & flow_id_mask);
944 num = rxa_create_event_vector(rx_adapter, eth_rx_queue_info,
948 if (num && dev_info->cb_fn) {
951 nb_cb = dev_info->cb_fn(eth_dev_id, rx_queue_id,
953 (buf->events_size & ~buf->last_mask),
954 buf->count >= BATCH_SIZE ?
955 buf->count - BATCH_SIZE : 0,
956 &buf->events[buf->tail],
960 if (unlikely(nb_cb > num))
961 RTE_EDEV_LOG_ERR("Rx CB returned %d (> %d) events",
966 stats->rx_dropped += dropped;
974 rxa_pkt_buf_available(struct eth_event_enqueue_buffer *buf)
976 uint32_t nb_req = buf->tail + BATCH_SIZE;
979 if (nb_req <= buf->events_size)
982 if (buf->head >= BATCH_SIZE) {
984 buf->last = buf->tail;
990 return nb_req <= buf->head;
993 /* Enqueue packets from <port, q> to event buffer */
994 static inline uint32_t
995 rxa_eth_rx(struct event_eth_rx_adapter *rx_adapter, uint16_t port_id,
996 uint16_t queue_id, uint32_t rx_count, uint32_t max_rx,
997 int *rxq_empty, struct eth_event_enqueue_buffer *buf,
998 struct rte_event_eth_rx_adapter_stats *stats)
1000 struct rte_mbuf *mbufs[BATCH_SIZE];
1003 uint32_t nb_flushed = 0;
1007 /* Don't do a batch dequeue from the rx queue if there isn't
1008 * enough space in the enqueue buffer.
1010 while (rxa_pkt_buf_available(buf)) {
1011 if (buf->count >= BATCH_SIZE)
1013 rxa_flush_event_buffer(rx_adapter, buf, stats);
1015 stats->rx_poll_count++;
1016 n = rte_eth_rx_burst(port_id, queue_id, mbufs, BATCH_SIZE);
1022 rxa_buffer_mbufs(rx_adapter, port_id, queue_id, mbufs, n, buf,
1025 if (rx_count + nb_rx > max_rx)
1030 nb_flushed += rxa_flush_event_buffer(rx_adapter, buf, stats);
1032 stats->rx_packets += nb_rx;
1033 if (nb_flushed == 0)
1034 rte_event_maintain(rx_adapter->eventdev_id,
1035 rx_adapter->event_port_id, 0);
1041 rxa_intr_ring_enqueue(struct event_eth_rx_adapter *rx_adapter, void *data)
1046 union queue_data qd;
1047 struct eth_device_info *dev_info;
1048 struct eth_rx_queue_info *queue_info;
1055 dev_info = &rx_adapter->eth_devices[port_id];
1056 queue_info = &dev_info->rx_queue[queue];
1057 rte_spinlock_lock(&rx_adapter->intr_ring_lock);
1058 if (rxa_shared_intr(dev_info, queue))
1059 intr_enabled = &dev_info->shared_intr_enabled;
1061 intr_enabled = &queue_info->intr_enabled;
1063 if (*intr_enabled) {
1065 err = rte_ring_enqueue(rx_adapter->intr_ring, data);
1066 /* Entry should always be available.
1067 * The ring size equals the maximum number of interrupt
1068 * vectors supported (an interrupt vector is shared in
1069 * case of shared interrupts)
1072 RTE_EDEV_LOG_ERR("Failed to enqueue interrupt"
1073 " to ring: %s", strerror(-err));
1075 rte_eth_dev_rx_intr_disable(port_id, queue);
1077 rte_spinlock_unlock(&rx_adapter->intr_ring_lock);
1081 rxa_intr_ring_check_avail(struct event_eth_rx_adapter *rx_adapter,
1082 uint32_t num_intr_vec)
1084 if (rx_adapter->num_intr_vec + num_intr_vec >
1085 RTE_EVENT_ETH_INTR_RING_SIZE) {
1086 RTE_EDEV_LOG_ERR("Exceeded intr ring slots current"
1087 " %d needed %d limit %d", rx_adapter->num_intr_vec,
1088 num_intr_vec, RTE_EVENT_ETH_INTR_RING_SIZE);
1095 /* Delete entries for (dev, queue) from the interrupt ring */
1097 rxa_intr_ring_del_entries(struct event_eth_rx_adapter *rx_adapter,
1098 struct eth_device_info *dev_info,
1099 uint16_t rx_queue_id)
1102 union queue_data qd;
1104 rte_spinlock_lock(&rx_adapter->intr_ring_lock);
1106 n = rte_ring_count(rx_adapter->intr_ring);
1107 for (i = 0; i < n; i++) {
1108 rte_ring_dequeue(rx_adapter->intr_ring, &qd.ptr);
1109 if (!rxa_shared_intr(dev_info, rx_queue_id)) {
1110 if (qd.port == dev_info->dev->data->port_id &&
1111 qd.queue == rx_queue_id)
1114 if (qd.port == dev_info->dev->data->port_id)
1117 rte_ring_enqueue(rx_adapter->intr_ring, qd.ptr);
1120 rte_spinlock_unlock(&rx_adapter->intr_ring_lock);
1123 /* pthread callback handling interrupt mode receive queues
1124 * After receiving an Rx interrupt, it enqueues the port id and queue id of the
1125 * interrupting queue to the adapter's ring buffer for interrupt events.
1126 * These events are picked up by rxa_intr_ring_dequeue() which is invoked from
1127 * the adapter service function.
1130 rxa_intr_thread(void *arg)
1132 struct event_eth_rx_adapter *rx_adapter = arg;
1133 struct rte_epoll_event *epoll_events = rx_adapter->epoll_events;
1137 n = rte_epoll_wait(rx_adapter->epd, epoll_events,
1138 RTE_EVENT_ETH_INTR_RING_SIZE, -1);
1139 if (unlikely(n < 0))
1140 RTE_EDEV_LOG_ERR("rte_epoll_wait returned error %d",
1142 for (i = 0; i < n; i++) {
1143 rxa_intr_ring_enqueue(rx_adapter,
1144 epoll_events[i].epdata.data);
1151 /* Dequeue <port, q> from interrupt ring and enqueue received
1155 rxa_intr_ring_dequeue(struct event_eth_rx_adapter *rx_adapter)
1160 struct eth_event_enqueue_buffer *buf;
1161 struct rte_event_eth_rx_adapter_stats *stats;
1162 rte_spinlock_t *ring_lock;
1163 uint8_t max_done = 0;
1165 if (rx_adapter->num_rx_intr == 0)
1168 if (rte_ring_count(rx_adapter->intr_ring) == 0
1169 && !rx_adapter->qd_valid)
1172 buf = &rx_adapter->event_enqueue_buffer;
1173 stats = &rx_adapter->stats;
1174 ring_lock = &rx_adapter->intr_ring_lock;
1176 if (buf->count >= BATCH_SIZE)
1177 rxa_flush_event_buffer(rx_adapter, buf, stats);
1179 while (rxa_pkt_buf_available(buf)) {
1180 struct eth_device_info *dev_info;
1183 union queue_data qd = rx_adapter->qd;
1186 if (!rx_adapter->qd_valid) {
1187 struct eth_rx_queue_info *queue_info;
1189 rte_spinlock_lock(ring_lock);
1190 err = rte_ring_dequeue(rx_adapter->intr_ring, &qd.ptr);
1192 rte_spinlock_unlock(ring_lock);
1198 rx_adapter->qd = qd;
1199 rx_adapter->qd_valid = 1;
1200 dev_info = &rx_adapter->eth_devices[port];
1201 if (rxa_shared_intr(dev_info, queue))
1202 dev_info->shared_intr_enabled = 1;
1204 queue_info = &dev_info->rx_queue[queue];
1205 queue_info->intr_enabled = 1;
1207 rte_eth_dev_rx_intr_enable(port, queue);
1208 rte_spinlock_unlock(ring_lock);
1213 dev_info = &rx_adapter->eth_devices[port];
1216 if (rxa_shared_intr(dev_info, queue)) {
1220 nb_queues = dev_info->dev->data->nb_rx_queues;
1222 for (i = dev_info->next_q_idx; i < nb_queues; i++) {
1223 uint8_t enq_buffer_full;
1225 if (!rxa_intr_queue(dev_info, i))
1227 n = rxa_eth_rx(rx_adapter, port, i, nb_rx,
1228 rx_adapter->max_nb_rx,
1229 &rxq_empty, buf, stats);
1232 enq_buffer_full = !rxq_empty && n == 0;
1233 max_done = nb_rx > rx_adapter->max_nb_rx;
1235 if (enq_buffer_full || max_done) {
1236 dev_info->next_q_idx = i;
1241 rx_adapter->qd_valid = 0;
1243 /* Reinitialize for next interrupt */
1244 dev_info->next_q_idx = dev_info->multi_intr_cap ?
1245 RTE_MAX_RXTX_INTR_VEC_ID - 1 :
1248 n = rxa_eth_rx(rx_adapter, port, queue, nb_rx,
1249 rx_adapter->max_nb_rx,
1250 &rxq_empty, buf, stats);
1251 rx_adapter->qd_valid = !rxq_empty;
1253 if (nb_rx > rx_adapter->max_nb_rx)
1259 rx_adapter->stats.rx_intr_packets += nb_rx;
1263 * Polls receive queues added to the event adapter and enqueues received
1264 * packets to the event device.
1266 * The receive code enqueues initially to a temporary buffer, the
1267 * temporary buffer is drained anytime it holds >= BATCH_SIZE packets
1269 * If there isn't space available in the temporary buffer, packets from the
1270 * Rx queue aren't dequeued from the eth device, this back pressures the
1271 * eth device, in virtual device environments this back pressure is relayed to
1272 * the hypervisor's switching layer where adjustments can be made to deal with
1276 rxa_poll(struct event_eth_rx_adapter *rx_adapter)
1280 struct eth_event_enqueue_buffer *buf = NULL;
1281 struct rte_event_eth_rx_adapter_stats *stats = NULL;
1285 wrr_pos = rx_adapter->wrr_pos;
1286 max_nb_rx = rx_adapter->max_nb_rx;
1288 /* Iterate through a WRR sequence */
1289 for (num_queue = 0; num_queue < rx_adapter->wrr_len; num_queue++) {
1290 unsigned int poll_idx = rx_adapter->wrr_sched[wrr_pos];
1291 uint16_t qid = rx_adapter->eth_rx_poll[poll_idx].eth_rx_qid;
1292 uint16_t d = rx_adapter->eth_rx_poll[poll_idx].eth_dev_id;
1294 buf = rxa_event_buf_get(rx_adapter, d, qid, &stats);
1296 /* Don't do a batch dequeue from the rx queue if there isn't
1297 * enough space in the enqueue buffer.
1299 if (buf->count >= BATCH_SIZE)
1300 rxa_flush_event_buffer(rx_adapter, buf, stats);
1301 if (!rxa_pkt_buf_available(buf)) {
1302 if (rx_adapter->use_queue_event_buf)
1303 goto poll_next_entry;
1305 rx_adapter->wrr_pos = wrr_pos;
1310 nb_rx += rxa_eth_rx(rx_adapter, d, qid, nb_rx, max_nb_rx,
1312 if (nb_rx > max_nb_rx) {
1313 rx_adapter->wrr_pos =
1314 (wrr_pos + 1) % rx_adapter->wrr_len;
1319 if (++wrr_pos == rx_adapter->wrr_len)
1325 rxa_vector_expire(struct eth_rx_vector_data *vec, void *arg)
1327 struct event_eth_rx_adapter *rx_adapter = arg;
1328 struct eth_event_enqueue_buffer *buf = NULL;
1329 struct rte_event_eth_rx_adapter_stats *stats = NULL;
1330 struct rte_event *ev;
1332 buf = rxa_event_buf_get(rx_adapter, vec->port, vec->queue, &stats);
1335 rxa_flush_event_buffer(rx_adapter, buf, stats);
1337 if (vec->vector_ev->nb_elem == 0)
1339 ev = &buf->events[buf->count];
1342 ev->event = vec->event;
1343 ev->vec = vec->vector_ev;
1346 vec->vector_ev = NULL;
1351 rxa_service_func(void *args)
1353 struct event_eth_rx_adapter *rx_adapter = args;
1355 if (rte_spinlock_trylock(&rx_adapter->rx_lock) == 0)
1357 if (!rx_adapter->rxa_started) {
1358 rte_spinlock_unlock(&rx_adapter->rx_lock);
1362 if (rx_adapter->ena_vector) {
1363 if ((rte_rdtsc() - rx_adapter->prev_expiry_ts) >=
1364 rx_adapter->vector_tmo_ticks) {
1365 struct eth_rx_vector_data *vec;
1367 TAILQ_FOREACH(vec, &rx_adapter->vector_list, next) {
1368 uint64_t elapsed_time = rte_rdtsc() - vec->ts;
1370 if (elapsed_time >= vec->vector_timeout_ticks) {
1371 rxa_vector_expire(vec, rx_adapter);
1372 TAILQ_REMOVE(&rx_adapter->vector_list,
1376 rx_adapter->prev_expiry_ts = rte_rdtsc();
1380 rxa_intr_ring_dequeue(rx_adapter);
1381 rxa_poll(rx_adapter);
1383 rte_spinlock_unlock(&rx_adapter->rx_lock);
1389 rte_event_eth_rx_adapter_init(void)
1391 const char *name = RXA_ADAPTER_ARRAY;
1392 const struct rte_memzone *mz;
1395 sz = sizeof(*event_eth_rx_adapter) *
1396 RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE;
1397 sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
1399 mz = rte_memzone_lookup(name);
1401 mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0,
1402 RTE_CACHE_LINE_SIZE);
1404 RTE_EDEV_LOG_ERR("failed to reserve memzone err = %"
1410 event_eth_rx_adapter = mz->addr;
1415 rxa_memzone_lookup(void)
1417 const struct rte_memzone *mz;
1419 if (event_eth_rx_adapter == NULL) {
1420 mz = rte_memzone_lookup(RXA_ADAPTER_ARRAY);
1423 event_eth_rx_adapter = mz->addr;
1429 static inline struct event_eth_rx_adapter *
1430 rxa_id_to_adapter(uint8_t id)
1432 return event_eth_rx_adapter ?
1433 event_eth_rx_adapter[id] : NULL;
1437 rxa_default_conf_cb(uint8_t id, uint8_t dev_id,
1438 struct rte_event_eth_rx_adapter_conf *conf, void *arg)
1441 struct rte_eventdev *dev;
1442 struct rte_event_dev_config dev_conf;
1445 struct rte_event_port_conf *port_conf = arg;
1446 struct event_eth_rx_adapter *rx_adapter = rxa_id_to_adapter(id);
1448 dev = &rte_eventdevs[rx_adapter->eventdev_id];
1449 dev_conf = dev->data->dev_conf;
1451 started = dev->data->dev_started;
1453 rte_event_dev_stop(dev_id);
1454 port_id = dev_conf.nb_event_ports;
1455 dev_conf.nb_event_ports += 1;
1456 ret = rte_event_dev_configure(dev_id, &dev_conf);
1458 RTE_EDEV_LOG_ERR("failed to configure event dev %u\n",
1461 if (rte_event_dev_start(dev_id))
1467 ret = rte_event_port_setup(dev_id, port_id, port_conf);
1469 RTE_EDEV_LOG_ERR("failed to setup event port %u\n",
1474 conf->event_port_id = port_id;
1475 conf->max_nb_rx = 128;
1477 ret = rte_event_dev_start(dev_id);
1478 rx_adapter->default_cb_arg = 1;
1483 rxa_epoll_create1(void)
1487 fd = epoll_create1(EPOLL_CLOEXEC);
1488 return fd < 0 ? -errno : fd;
1495 rxa_init_epd(struct event_eth_rx_adapter *rx_adapter)
1497 if (rx_adapter->epd != INIT_FD)
1500 rx_adapter->epd = rxa_epoll_create1();
1501 if (rx_adapter->epd < 0) {
1502 int err = rx_adapter->epd;
1503 rx_adapter->epd = INIT_FD;
1504 RTE_EDEV_LOG_ERR("epoll_create1() failed, err %d", err);
1512 rxa_create_intr_thread(struct event_eth_rx_adapter *rx_adapter)
1515 char thread_name[RTE_MAX_THREAD_NAME_LEN];
1517 if (rx_adapter->intr_ring)
1520 rx_adapter->intr_ring = rte_ring_create("intr_ring",
1521 RTE_EVENT_ETH_INTR_RING_SIZE,
1522 rte_socket_id(), 0);
1523 if (!rx_adapter->intr_ring)
1526 rx_adapter->epoll_events = rte_zmalloc_socket(rx_adapter->mem_name,
1527 RTE_EVENT_ETH_INTR_RING_SIZE *
1528 sizeof(struct rte_epoll_event),
1529 RTE_CACHE_LINE_SIZE,
1530 rx_adapter->socket_id);
1531 if (!rx_adapter->epoll_events) {
1536 rte_spinlock_init(&rx_adapter->intr_ring_lock);
1538 snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN,
1539 "rx-intr-thread-%d", rx_adapter->id);
1541 err = rte_ctrl_thread_create(&rx_adapter->rx_intr_thread, thread_name,
1542 NULL, rxa_intr_thread, rx_adapter);
1546 RTE_EDEV_LOG_ERR("Failed to create interrupt thread err = %d\n", err);
1547 rte_free(rx_adapter->epoll_events);
1549 rte_ring_free(rx_adapter->intr_ring);
1550 rx_adapter->intr_ring = NULL;
1551 rx_adapter->epoll_events = NULL;
1556 rxa_destroy_intr_thread(struct event_eth_rx_adapter *rx_adapter)
1560 err = pthread_cancel(rx_adapter->rx_intr_thread);
1562 RTE_EDEV_LOG_ERR("Can't cancel interrupt thread err = %d\n",
1565 err = pthread_join(rx_adapter->rx_intr_thread, NULL);
1567 RTE_EDEV_LOG_ERR("Can't join interrupt thread err = %d\n", err);
1569 rte_free(rx_adapter->epoll_events);
1570 rte_ring_free(rx_adapter->intr_ring);
1571 rx_adapter->intr_ring = NULL;
1572 rx_adapter->epoll_events = NULL;
1577 rxa_free_intr_resources(struct event_eth_rx_adapter *rx_adapter)
1581 if (rx_adapter->num_rx_intr == 0)
1584 ret = rxa_destroy_intr_thread(rx_adapter);
1588 close(rx_adapter->epd);
1589 rx_adapter->epd = INIT_FD;
1595 rxa_disable_intr(struct event_eth_rx_adapter *rx_adapter,
1596 struct eth_device_info *dev_info, uint16_t rx_queue_id)
1599 uint16_t eth_dev_id = dev_info->dev->data->port_id;
1600 int sintr = rxa_shared_intr(dev_info, rx_queue_id);
1602 err = rte_eth_dev_rx_intr_disable(eth_dev_id, rx_queue_id);
1604 RTE_EDEV_LOG_ERR("Could not disable interrupt for Rx queue %u",
1609 err = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
1614 RTE_EDEV_LOG_ERR("Interrupt event deletion failed %d", err);
1617 dev_info->rx_queue[rx_queue_id].intr_enabled = 0;
1619 dev_info->shared_intr_enabled = 0;
1624 rxa_del_intr_queue(struct event_eth_rx_adapter *rx_adapter,
1625 struct eth_device_info *dev_info, int rx_queue_id)
1631 if (dev_info->nb_rx_intr == 0)
1635 if (rx_queue_id == -1) {
1636 s = dev_info->nb_shared_intr;
1637 for (i = 0; i < dev_info->nb_rx_intr; i++) {
1641 q = dev_info->intr_queue[i];
1642 sintr = rxa_shared_intr(dev_info, q);
1645 if (!sintr || s == 0) {
1647 err = rxa_disable_intr(rx_adapter, dev_info,
1651 rxa_intr_ring_del_entries(rx_adapter, dev_info,
1656 if (!rxa_intr_queue(dev_info, rx_queue_id))
1658 if (!rxa_shared_intr(dev_info, rx_queue_id) ||
1659 dev_info->nb_shared_intr == 1) {
1660 err = rxa_disable_intr(rx_adapter, dev_info,
1664 rxa_intr_ring_del_entries(rx_adapter, dev_info,
1668 for (i = 0; i < dev_info->nb_rx_intr; i++) {
1669 if (dev_info->intr_queue[i] == rx_queue_id) {
1670 for (; i < dev_info->nb_rx_intr - 1; i++)
1671 dev_info->intr_queue[i] =
1672 dev_info->intr_queue[i + 1];
1682 rxa_config_intr(struct event_eth_rx_adapter *rx_adapter,
1683 struct eth_device_info *dev_info, uint16_t rx_queue_id)
1686 uint16_t eth_dev_id = dev_info->dev->data->port_id;
1687 union queue_data qd;
1689 uint16_t *intr_queue;
1690 int sintr = rxa_shared_intr(dev_info, rx_queue_id);
1692 if (rxa_intr_queue(dev_info, rx_queue_id))
1695 intr_queue = dev_info->intr_queue;
1696 if (dev_info->intr_queue == NULL) {
1698 dev_info->dev->data->nb_rx_queues * sizeof(uint16_t);
1699 dev_info->intr_queue =
1701 rx_adapter->mem_name,
1704 rx_adapter->socket_id);
1705 if (dev_info->intr_queue == NULL)
1709 init_fd = rx_adapter->epd;
1710 err = rxa_init_epd(rx_adapter);
1712 goto err_free_queue;
1714 qd.port = eth_dev_id;
1715 qd.queue = rx_queue_id;
1717 err = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
1722 RTE_EDEV_LOG_ERR("Failed to add interrupt event for"
1723 " Rx Queue %u err %d", rx_queue_id, err);
1727 err = rte_eth_dev_rx_intr_enable(eth_dev_id, rx_queue_id);
1729 RTE_EDEV_LOG_ERR("Could not enable interrupt for"
1730 " Rx Queue %u err %d", rx_queue_id, err);
1735 err = rxa_create_intr_thread(rx_adapter);
1738 dev_info->shared_intr_enabled = 1;
1740 dev_info->rx_queue[rx_queue_id].intr_enabled = 1;
1745 err = rte_eth_dev_rx_intr_disable(eth_dev_id, rx_queue_id);
1747 RTE_EDEV_LOG_ERR("Could not disable interrupt for"
1748 " Rx Queue %u err %d", rx_queue_id, err);
1750 err1 = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
1755 RTE_EDEV_LOG_ERR("Could not delete event for"
1756 " Rx Queue %u err %d", rx_queue_id, err1);
1759 if (init_fd == INIT_FD) {
1760 close(rx_adapter->epd);
1761 rx_adapter->epd = -1;
1764 if (intr_queue == NULL)
1765 rte_free(dev_info->intr_queue);
1771 rxa_add_intr_queue(struct event_eth_rx_adapter *rx_adapter,
1772 struct eth_device_info *dev_info, int rx_queue_id)
1777 int shared_done = (dev_info->nb_shared_intr > 0);
1779 if (rx_queue_id != -1) {
1780 if (rxa_shared_intr(dev_info, rx_queue_id) && shared_done)
1782 return rxa_config_intr(rx_adapter, dev_info, rx_queue_id);
1786 for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++) {
1788 if (rxa_shared_intr(dev_info, i) && shared_done)
1791 err = rxa_config_intr(rx_adapter, dev_info, i);
1793 shared_done = err == 0 && rxa_shared_intr(dev_info, i);
1796 dev_info->shared_intr_enabled = 1;
1805 shared_done = (dev_info->nb_shared_intr > 0);
1806 for (j = 0; j < i; j++) {
1807 if (rxa_intr_queue(dev_info, j))
1809 if (rxa_shared_intr(dev_info, j) && si != j)
1811 err = rxa_disable_intr(rx_adapter, dev_info, j);
1821 rxa_init_service(struct event_eth_rx_adapter *rx_adapter, uint8_t id)
1824 struct rte_service_spec service;
1825 struct rte_event_eth_rx_adapter_conf rx_adapter_conf;
1827 if (rx_adapter->service_inited)
1830 memset(&service, 0, sizeof(service));
1831 snprintf(service.name, ETH_RX_ADAPTER_SERVICE_NAME_LEN,
1832 "rte_event_eth_rx_adapter_%d", id);
1833 service.socket_id = rx_adapter->socket_id;
1834 service.callback = rxa_service_func;
1835 service.callback_userdata = rx_adapter;
1836 /* Service function handles locking for queue add/del updates */
1837 service.capabilities = RTE_SERVICE_CAP_MT_SAFE;
1838 ret = rte_service_component_register(&service, &rx_adapter->service_id);
1840 RTE_EDEV_LOG_ERR("failed to register service %s err = %" PRId32,
1845 ret = rx_adapter->conf_cb(id, rx_adapter->eventdev_id,
1846 &rx_adapter_conf, rx_adapter->conf_arg);
1848 RTE_EDEV_LOG_ERR("configuration callback failed err = %" PRId32,
1852 rx_adapter->event_port_id = rx_adapter_conf.event_port_id;
1853 rx_adapter->max_nb_rx = rx_adapter_conf.max_nb_rx;
1854 rx_adapter->service_inited = 1;
1855 rx_adapter->epd = INIT_FD;
1859 rte_service_component_unregister(rx_adapter->service_id);
1864 rxa_update_queue(struct event_eth_rx_adapter *rx_adapter,
1865 struct eth_device_info *dev_info, int32_t rx_queue_id,
1868 struct eth_rx_queue_info *queue_info;
1872 if (dev_info->rx_queue == NULL)
1875 if (rx_queue_id == -1) {
1876 for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
1877 rxa_update_queue(rx_adapter, dev_info, i, add);
1879 queue_info = &dev_info->rx_queue[rx_queue_id];
1880 enabled = queue_info->queue_enabled;
1882 rx_adapter->nb_queues += !enabled;
1883 dev_info->nb_dev_queues += !enabled;
1885 rx_adapter->nb_queues -= enabled;
1886 dev_info->nb_dev_queues -= enabled;
1888 queue_info->queue_enabled = !!add;
1893 rxa_set_vector_data(struct eth_rx_queue_info *queue_info, uint16_t vector_count,
1894 uint64_t vector_ns, struct rte_mempool *mp, uint32_t qid,
1897 #define NSEC2TICK(__ns, __freq) (((__ns) * (__freq)) / 1E9)
1898 struct eth_rx_vector_data *vector_data;
1901 vector_data = &queue_info->vector_data;
1902 vector_data->max_vector_count = vector_count;
1903 vector_data->port = port_id;
1904 vector_data->queue = qid;
1905 vector_data->vector_pool = mp;
1906 vector_data->vector_timeout_ticks =
1907 NSEC2TICK(vector_ns, rte_get_timer_hz());
1908 vector_data->ts = 0;
1909 flow_id = queue_info->event & 0xFFFFF;
1911 flow_id == 0 ? (qid & 0xFFF) | (port_id & 0xFF) << 12 : flow_id;
1912 vector_data->event = (queue_info->event & ~0xFFFFF) | flow_id;
1916 rxa_sw_del(struct event_eth_rx_adapter *rx_adapter,
1917 struct eth_device_info *dev_info, int32_t rx_queue_id)
1919 struct eth_rx_vector_data *vec;
1925 if (rx_adapter->nb_queues == 0)
1928 if (rx_queue_id == -1) {
1929 uint16_t nb_rx_queues;
1932 nb_rx_queues = dev_info->dev->data->nb_rx_queues;
1933 for (i = 0; i < nb_rx_queues; i++)
1934 rxa_sw_del(rx_adapter, dev_info, i);
1938 /* Push all the partial event vectors to event device. */
1939 TAILQ_FOREACH(vec, &rx_adapter->vector_list, next) {
1940 if (vec->queue != rx_queue_id)
1942 rxa_vector_expire(vec, rx_adapter);
1943 TAILQ_REMOVE(&rx_adapter->vector_list, vec, next);
1946 pollq = rxa_polled_queue(dev_info, rx_queue_id);
1947 intrq = rxa_intr_queue(dev_info, rx_queue_id);
1948 sintrq = rxa_shared_intr(dev_info, rx_queue_id);
1949 rxa_update_queue(rx_adapter, dev_info, rx_queue_id, 0);
1950 rx_adapter->num_rx_polled -= pollq;
1951 dev_info->nb_rx_poll -= pollq;
1952 rx_adapter->num_rx_intr -= intrq;
1953 dev_info->nb_rx_intr -= intrq;
1954 dev_info->nb_shared_intr -= intrq && sintrq;
1955 if (rx_adapter->use_queue_event_buf) {
1956 struct eth_event_enqueue_buffer *event_buf =
1957 dev_info->rx_queue[rx_queue_id].event_buf;
1958 struct rte_event_eth_rx_adapter_stats *stats =
1959 dev_info->rx_queue[rx_queue_id].stats;
1960 rte_free(event_buf->events);
1961 rte_free(event_buf);
1963 dev_info->rx_queue[rx_queue_id].event_buf = NULL;
1964 dev_info->rx_queue[rx_queue_id].stats = NULL;
1969 rxa_add_queue(struct event_eth_rx_adapter *rx_adapter,
1970 struct eth_device_info *dev_info, int32_t rx_queue_id,
1971 const struct rte_event_eth_rx_adapter_queue_conf *conf)
1973 struct eth_rx_queue_info *queue_info;
1974 const struct rte_event *ev = &conf->ev;
1978 struct rte_event *qi_ev;
1979 struct eth_event_enqueue_buffer *new_rx_buf = NULL;
1980 struct rte_event_eth_rx_adapter_stats *stats = NULL;
1981 uint16_t eth_dev_id = dev_info->dev->data->port_id;
1984 if (rx_queue_id == -1) {
1985 uint16_t nb_rx_queues;
1988 nb_rx_queues = dev_info->dev->data->nb_rx_queues;
1989 for (i = 0; i < nb_rx_queues; i++) {
1990 ret = rxa_add_queue(rx_adapter, dev_info, i, conf);
1997 pollq = rxa_polled_queue(dev_info, rx_queue_id);
1998 intrq = rxa_intr_queue(dev_info, rx_queue_id);
1999 sintrq = rxa_shared_intr(dev_info, rx_queue_id);
2001 queue_info = &dev_info->rx_queue[rx_queue_id];
2002 queue_info->wt = conf->servicing_weight;
2004 qi_ev = (struct rte_event *)&queue_info->event;
2005 qi_ev->event = ev->event;
2006 qi_ev->op = RTE_EVENT_OP_NEW;
2007 qi_ev->event_type = RTE_EVENT_TYPE_ETH_RX_ADAPTER;
2008 qi_ev->sub_event_type = 0;
2010 if (conf->rx_queue_flags &
2011 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID) {
2012 queue_info->flow_id_mask = ~0;
2016 if (conf->rx_queue_flags &
2017 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR) {
2018 queue_info->ena_vector = 1;
2019 qi_ev->event_type = RTE_EVENT_TYPE_ETH_RX_ADAPTER_VECTOR;
2020 rxa_set_vector_data(queue_info, conf->vector_sz,
2021 conf->vector_timeout_ns, conf->vector_mp,
2022 rx_queue_id, dev_info->dev->data->port_id);
2023 rx_adapter->ena_vector = 1;
2024 rx_adapter->vector_tmo_ticks =
2025 rx_adapter->vector_tmo_ticks ?
2026 RTE_MIN(queue_info->vector_data
2027 .vector_timeout_ticks >>
2029 rx_adapter->vector_tmo_ticks) :
2030 queue_info->vector_data.vector_timeout_ticks >>
2034 rxa_update_queue(rx_adapter, dev_info, rx_queue_id, 1);
2035 if (rxa_polled_queue(dev_info, rx_queue_id)) {
2036 rx_adapter->num_rx_polled += !pollq;
2037 dev_info->nb_rx_poll += !pollq;
2038 rx_adapter->num_rx_intr -= intrq;
2039 dev_info->nb_rx_intr -= intrq;
2040 dev_info->nb_shared_intr -= intrq && sintrq;
2043 if (rxa_intr_queue(dev_info, rx_queue_id)) {
2044 rx_adapter->num_rx_polled -= pollq;
2045 dev_info->nb_rx_poll -= pollq;
2046 rx_adapter->num_rx_intr += !intrq;
2047 dev_info->nb_rx_intr += !intrq;
2048 dev_info->nb_shared_intr += !intrq && sintrq;
2049 if (dev_info->nb_shared_intr == 1) {
2050 if (dev_info->multi_intr_cap)
2051 dev_info->next_q_idx =
2052 RTE_MAX_RXTX_INTR_VEC_ID - 1;
2054 dev_info->next_q_idx = 0;
2058 if (!rx_adapter->use_queue_event_buf)
2061 new_rx_buf = rte_zmalloc_socket("rx_buffer_meta",
2062 sizeof(*new_rx_buf), 0,
2063 rte_eth_dev_socket_id(eth_dev_id));
2064 if (new_rx_buf == NULL) {
2065 RTE_EDEV_LOG_ERR("Failed to allocate event buffer meta for "
2066 "dev_id: %d queue_id: %d",
2067 eth_dev_id, rx_queue_id);
2071 new_rx_buf->events_size = RTE_ALIGN(conf->event_buf_size, BATCH_SIZE);
2072 new_rx_buf->events_size += (2 * BATCH_SIZE);
2073 new_rx_buf->events = rte_zmalloc_socket("rx_buffer",
2074 sizeof(struct rte_event) *
2075 new_rx_buf->events_size, 0,
2076 rte_eth_dev_socket_id(eth_dev_id));
2077 if (new_rx_buf->events == NULL) {
2078 rte_free(new_rx_buf);
2079 RTE_EDEV_LOG_ERR("Failed to allocate event buffer for "
2080 "dev_id: %d queue_id: %d",
2081 eth_dev_id, rx_queue_id);
2085 queue_info->event_buf = new_rx_buf;
2087 /* Allocate storage for adapter queue stats */
2088 stats = rte_zmalloc_socket("rx_queue_stats",
2090 rte_eth_dev_socket_id(eth_dev_id));
2091 if (stats == NULL) {
2092 rte_free(new_rx_buf->events);
2093 rte_free(new_rx_buf);
2094 RTE_EDEV_LOG_ERR("Failed to allocate stats storage for"
2095 " dev_id: %d queue_id: %d",
2096 eth_dev_id, rx_queue_id);
2100 queue_info->stats = stats;
2106 rxa_sw_add(struct event_eth_rx_adapter *rx_adapter, uint16_t eth_dev_id,
2108 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
2110 struct eth_device_info *dev_info = &rx_adapter->eth_devices[eth_dev_id];
2111 struct rte_event_eth_rx_adapter_queue_conf temp_conf;
2113 struct eth_rx_poll_entry *rx_poll;
2114 struct eth_rx_queue_info *rx_queue;
2116 uint16_t nb_rx_queues;
2117 uint32_t nb_rx_poll, nb_wrr;
2118 uint32_t nb_rx_intr;
2122 if (queue_conf->servicing_weight == 0) {
2123 struct rte_eth_dev_data *data = dev_info->dev->data;
2125 temp_conf = *queue_conf;
2126 if (!data->dev_conf.intr_conf.rxq) {
2127 /* If Rx interrupts are disabled set wt = 1 */
2128 temp_conf.servicing_weight = 1;
2130 queue_conf = &temp_conf;
2132 if (queue_conf->servicing_weight == 0 &&
2133 rx_adapter->use_queue_event_buf) {
2135 RTE_EDEV_LOG_ERR("Use of queue level event buffer "
2136 "not supported for interrupt queues "
2137 "dev_id: %d queue_id: %d",
2138 eth_dev_id, rx_queue_id);
2143 nb_rx_queues = dev_info->dev->data->nb_rx_queues;
2144 rx_queue = dev_info->rx_queue;
2145 wt = queue_conf->servicing_weight;
2147 if (dev_info->rx_queue == NULL) {
2148 dev_info->rx_queue =
2149 rte_zmalloc_socket(rx_adapter->mem_name,
2151 sizeof(struct eth_rx_queue_info), 0,
2152 rx_adapter->socket_id);
2153 if (dev_info->rx_queue == NULL)
2159 rxa_calc_nb_post_add(rx_adapter, dev_info, rx_queue_id,
2160 queue_conf->servicing_weight,
2161 &nb_rx_poll, &nb_rx_intr, &nb_wrr);
2163 if (dev_info->dev->intr_handle)
2164 dev_info->multi_intr_cap =
2165 rte_intr_cap_multiple(dev_info->dev->intr_handle);
2167 ret = rxa_alloc_poll_arrays(rx_adapter, nb_rx_poll, nb_wrr,
2170 goto err_free_rxqueue;
2173 num_intr_vec = rxa_nb_intr_vect(dev_info, rx_queue_id, 1);
2175 ret = rxa_intr_ring_check_avail(rx_adapter, num_intr_vec);
2177 goto err_free_rxqueue;
2179 ret = rxa_add_intr_queue(rx_adapter, dev_info, rx_queue_id);
2181 goto err_free_rxqueue;
2185 if (rx_adapter->num_rx_intr > nb_rx_intr) {
2186 num_intr_vec = rxa_nb_intr_vect(dev_info,
2188 /* interrupt based queues are being converted to
2189 * poll mode queues, delete the interrupt configuration
2192 ret = rxa_del_intr_queue(rx_adapter,
2193 dev_info, rx_queue_id);
2195 goto err_free_rxqueue;
2199 if (nb_rx_intr == 0) {
2200 ret = rxa_free_intr_resources(rx_adapter);
2202 goto err_free_rxqueue;
2208 if (rx_queue_id == -1) {
2209 for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
2210 dev_info->intr_queue[i] = i;
2212 if (!rxa_intr_queue(dev_info, rx_queue_id))
2213 dev_info->intr_queue[nb_rx_intr - 1] =
2220 ret = rxa_add_queue(rx_adapter, dev_info, rx_queue_id, queue_conf);
2222 goto err_free_rxqueue;
2223 rxa_calc_wrr_sequence(rx_adapter, rx_poll, rx_wrr);
2225 rte_free(rx_adapter->eth_rx_poll);
2226 rte_free(rx_adapter->wrr_sched);
2228 rx_adapter->eth_rx_poll = rx_poll;
2229 rx_adapter->wrr_sched = rx_wrr;
2230 rx_adapter->wrr_len = nb_wrr;
2231 rx_adapter->num_intr_vec += num_intr_vec;
2235 if (rx_queue == NULL) {
2236 rte_free(dev_info->rx_queue);
2237 dev_info->rx_queue = NULL;
2247 rxa_ctrl(uint8_t id, int start)
2249 struct event_eth_rx_adapter *rx_adapter;
2250 struct rte_eventdev *dev;
2251 struct eth_device_info *dev_info;
2253 int use_service = 0;
2256 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2257 rx_adapter = rxa_id_to_adapter(id);
2258 if (rx_adapter == NULL)
2261 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2263 RTE_ETH_FOREACH_DEV(i) {
2264 dev_info = &rx_adapter->eth_devices[i];
2265 /* if start check for num dev queues */
2266 if (start && !dev_info->nb_dev_queues)
2268 /* if stop check if dev has been started */
2269 if (stop && !dev_info->dev_rx_started)
2271 use_service |= !dev_info->internal_event_port;
2272 dev_info->dev_rx_started = start;
2273 if (dev_info->internal_event_port == 0)
2275 start ? (*dev->dev_ops->eth_rx_adapter_start)(dev,
2276 &rte_eth_devices[i]) :
2277 (*dev->dev_ops->eth_rx_adapter_stop)(dev,
2278 &rte_eth_devices[i]);
2282 rte_spinlock_lock(&rx_adapter->rx_lock);
2283 rx_adapter->rxa_started = start;
2284 rte_service_runstate_set(rx_adapter->service_id, start);
2285 rte_spinlock_unlock(&rx_adapter->rx_lock);
2292 rxa_create(uint8_t id, uint8_t dev_id,
2293 struct rte_event_eth_rx_adapter_params *rxa_params,
2294 rte_event_eth_rx_adapter_conf_cb conf_cb,
2297 struct event_eth_rx_adapter *rx_adapter;
2298 struct eth_event_enqueue_buffer *buf;
2299 struct rte_event *events;
2303 char mem_name[ETH_RX_ADAPTER_SERVICE_NAME_LEN];
2304 const uint8_t default_rss_key[] = {
2305 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
2306 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
2307 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
2308 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
2309 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
2312 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2313 RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
2315 if (conf_cb == NULL)
2318 if (event_eth_rx_adapter == NULL) {
2319 ret = rte_event_eth_rx_adapter_init();
2324 rx_adapter = rxa_id_to_adapter(id);
2325 if (rx_adapter != NULL) {
2326 RTE_EDEV_LOG_ERR("Eth Rx adapter exists id = %" PRIu8, id);
2330 socket_id = rte_event_dev_socket_id(dev_id);
2331 snprintf(mem_name, ETH_RX_ADAPTER_MEM_NAME_LEN,
2332 "rte_event_eth_rx_adapter_%d",
2335 rx_adapter = rte_zmalloc_socket(mem_name, sizeof(*rx_adapter),
2336 RTE_CACHE_LINE_SIZE, socket_id);
2337 if (rx_adapter == NULL) {
2338 RTE_EDEV_LOG_ERR("failed to get mem for rx adapter");
2342 rx_adapter->eventdev_id = dev_id;
2343 rx_adapter->socket_id = socket_id;
2344 rx_adapter->conf_cb = conf_cb;
2345 rx_adapter->conf_arg = conf_arg;
2346 rx_adapter->id = id;
2347 TAILQ_INIT(&rx_adapter->vector_list);
2348 strcpy(rx_adapter->mem_name, mem_name);
2349 rx_adapter->eth_devices = rte_zmalloc_socket(rx_adapter->mem_name,
2351 sizeof(struct eth_device_info), 0,
2353 rte_convert_rss_key((const uint32_t *)default_rss_key,
2354 (uint32_t *)rx_adapter->rss_key_be,
2355 RTE_DIM(default_rss_key));
2357 if (rx_adapter->eth_devices == NULL) {
2358 RTE_EDEV_LOG_ERR("failed to get mem for eth devices\n");
2359 rte_free(rx_adapter);
2363 rte_spinlock_init(&rx_adapter->rx_lock);
2365 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
2366 rx_adapter->eth_devices[i].dev = &rte_eth_devices[i];
2368 /* Rx adapter event buffer allocation */
2369 rx_adapter->use_queue_event_buf = rxa_params->use_queue_event_buf;
2371 if (!rx_adapter->use_queue_event_buf) {
2372 buf = &rx_adapter->event_enqueue_buffer;
2373 buf->events_size = rxa_params->event_buf_size;
2375 events = rte_zmalloc_socket(rx_adapter->mem_name,
2376 buf->events_size * sizeof(*events),
2378 if (events == NULL) {
2379 RTE_EDEV_LOG_ERR("Failed to allocate memory "
2380 "for adapter event buffer");
2381 rte_free(rx_adapter->eth_devices);
2382 rte_free(rx_adapter);
2386 rx_adapter->event_enqueue_buffer.events = events;
2389 event_eth_rx_adapter[id] = rx_adapter;
2391 if (conf_cb == rxa_default_conf_cb)
2392 rx_adapter->default_cb_arg = 1;
2394 if (rte_mbuf_dyn_rx_timestamp_register(
2395 &event_eth_rx_timestamp_dynfield_offset,
2396 &event_eth_rx_timestamp_dynflag) != 0) {
2397 RTE_EDEV_LOG_ERR("Error registering timestamp field in mbuf\n");
2401 rte_eventdev_trace_eth_rx_adapter_create(id, dev_id, conf_cb,
2407 rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id,
2408 rte_event_eth_rx_adapter_conf_cb conf_cb,
2411 struct rte_event_eth_rx_adapter_params rxa_params = {0};
2413 /* use default values for adapter params */
2414 rxa_params.event_buf_size = ETH_EVENT_BUFFER_SIZE;
2415 rxa_params.use_queue_event_buf = false;
2417 return rxa_create(id, dev_id, &rxa_params, conf_cb, conf_arg);
2421 rte_event_eth_rx_adapter_create_with_params(uint8_t id, uint8_t dev_id,
2422 struct rte_event_port_conf *port_config,
2423 struct rte_event_eth_rx_adapter_params *rxa_params)
2425 struct rte_event_port_conf *pc;
2427 struct rte_event_eth_rx_adapter_params temp_params = {0};
2429 if (port_config == NULL)
2432 if (rxa_params == NULL) {
2433 /* use default values if rxa_params is NULL */
2434 rxa_params = &temp_params;
2435 rxa_params->event_buf_size = ETH_EVENT_BUFFER_SIZE;
2436 rxa_params->use_queue_event_buf = false;
2437 } else if ((!rxa_params->use_queue_event_buf &&
2438 rxa_params->event_buf_size == 0) ||
2439 (rxa_params->use_queue_event_buf &&
2440 rxa_params->event_buf_size != 0)) {
2441 RTE_EDEV_LOG_ERR("Invalid adapter params\n");
2443 } else if (!rxa_params->use_queue_event_buf) {
2444 /* adjust event buff size with BATCH_SIZE used for fetching
2445 * packets from NIC rx queues to get full buffer utilization
2446 * and prevent unnecessary rollovers.
2449 rxa_params->event_buf_size =
2450 RTE_ALIGN(rxa_params->event_buf_size, BATCH_SIZE);
2451 rxa_params->event_buf_size += (BATCH_SIZE + BATCH_SIZE);
2454 pc = rte_malloc(NULL, sizeof(*pc), 0);
2460 ret = rxa_create(id, dev_id, rxa_params, rxa_default_conf_cb, pc);
2468 rte_event_eth_rx_adapter_create(uint8_t id, uint8_t dev_id,
2469 struct rte_event_port_conf *port_config)
2471 struct rte_event_port_conf *pc;
2474 if (port_config == NULL)
2477 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2479 pc = rte_malloc(NULL, sizeof(*pc), 0);
2484 ret = rte_event_eth_rx_adapter_create_ext(id, dev_id,
2485 rxa_default_conf_cb,
2493 rte_event_eth_rx_adapter_free(uint8_t id)
2495 struct event_eth_rx_adapter *rx_adapter;
2497 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2499 rx_adapter = rxa_id_to_adapter(id);
2500 if (rx_adapter == NULL)
2503 if (rx_adapter->nb_queues) {
2504 RTE_EDEV_LOG_ERR("%" PRIu16 " Rx queues not deleted",
2505 rx_adapter->nb_queues);
2509 if (rx_adapter->default_cb_arg)
2510 rte_free(rx_adapter->conf_arg);
2511 rte_free(rx_adapter->eth_devices);
2512 if (!rx_adapter->use_queue_event_buf)
2513 rte_free(rx_adapter->event_enqueue_buffer.events);
2514 rte_free(rx_adapter);
2515 event_eth_rx_adapter[id] = NULL;
2517 rte_eventdev_trace_eth_rx_adapter_free(id);
2522 rte_event_eth_rx_adapter_queue_add(uint8_t id,
2523 uint16_t eth_dev_id,
2524 int32_t rx_queue_id,
2525 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
2529 struct event_eth_rx_adapter *rx_adapter;
2530 struct rte_eventdev *dev;
2531 struct eth_device_info *dev_info;
2532 struct rte_event_eth_rx_adapter_vector_limits limits;
2534 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2535 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
2537 rx_adapter = rxa_id_to_adapter(id);
2538 if ((rx_adapter == NULL) || (queue_conf == NULL))
2541 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2542 ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
2546 RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
2547 "eth port %" PRIu16, id, eth_dev_id);
2551 if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) == 0
2552 && (queue_conf->rx_queue_flags &
2553 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID)) {
2554 RTE_EDEV_LOG_ERR("Flow ID override is not supported,"
2555 " eth port: %" PRIu16 " adapter id: %" PRIu8,
2560 if (queue_conf->rx_queue_flags &
2561 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR) {
2563 if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) == 0) {
2564 RTE_EDEV_LOG_ERR("Event vectorization is not supported,"
2565 " eth port: %" PRIu16
2566 " adapter id: %" PRIu8,
2571 ret = rte_event_eth_rx_adapter_vector_limits_get(
2572 rx_adapter->eventdev_id, eth_dev_id, &limits);
2574 RTE_EDEV_LOG_ERR("Failed to get event device vector limits,"
2575 " eth port: %" PRIu16
2576 " adapter id: %" PRIu8,
2580 if (queue_conf->vector_sz < limits.min_sz ||
2581 queue_conf->vector_sz > limits.max_sz ||
2582 queue_conf->vector_timeout_ns < limits.min_timeout_ns ||
2583 queue_conf->vector_timeout_ns > limits.max_timeout_ns ||
2584 queue_conf->vector_mp == NULL) {
2585 RTE_EDEV_LOG_ERR("Invalid event vector configuration,"
2586 " eth port: %" PRIu16
2587 " adapter id: %" PRIu8,
2591 if (queue_conf->vector_mp->elt_size <
2592 (sizeof(struct rte_event_vector) +
2593 (sizeof(uintptr_t) * queue_conf->vector_sz))) {
2594 RTE_EDEV_LOG_ERR("Invalid event vector configuration,"
2595 " eth port: %" PRIu16
2596 " adapter id: %" PRIu8,
2602 if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ) == 0 &&
2603 (rx_queue_id != -1)) {
2604 RTE_EDEV_LOG_ERR("Rx queues can only be connected to single "
2605 "event queue, eth port: %" PRIu16 " adapter id: %"
2606 PRIu8, eth_dev_id, id);
2610 if (rx_queue_id != -1 && (uint16_t)rx_queue_id >=
2611 rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
2612 RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16,
2613 (uint16_t)rx_queue_id);
2617 if ((rx_adapter->use_queue_event_buf &&
2618 queue_conf->event_buf_size == 0) ||
2619 (!rx_adapter->use_queue_event_buf &&
2620 queue_conf->event_buf_size != 0)) {
2621 RTE_EDEV_LOG_ERR("Invalid Event buffer size for the queue");
2625 dev_info = &rx_adapter->eth_devices[eth_dev_id];
2627 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
2628 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->eth_rx_adapter_queue_add,
2630 if (dev_info->rx_queue == NULL) {
2631 dev_info->rx_queue =
2632 rte_zmalloc_socket(rx_adapter->mem_name,
2633 dev_info->dev->data->nb_rx_queues *
2634 sizeof(struct eth_rx_queue_info), 0,
2635 rx_adapter->socket_id);
2636 if (dev_info->rx_queue == NULL)
2640 ret = (*dev->dev_ops->eth_rx_adapter_queue_add)(dev,
2641 &rte_eth_devices[eth_dev_id],
2642 rx_queue_id, queue_conf);
2644 dev_info->internal_event_port = 1;
2645 rxa_update_queue(rx_adapter,
2646 &rx_adapter->eth_devices[eth_dev_id],
2651 rte_spinlock_lock(&rx_adapter->rx_lock);
2652 dev_info->internal_event_port = 0;
2653 ret = rxa_init_service(rx_adapter, id);
2655 uint32_t service_id = rx_adapter->service_id;
2656 ret = rxa_sw_add(rx_adapter, eth_dev_id, rx_queue_id,
2658 rte_service_component_runstate_set(service_id,
2659 rxa_sw_adapter_queue_count(rx_adapter));
2661 rte_spinlock_unlock(&rx_adapter->rx_lock);
2664 rte_eventdev_trace_eth_rx_adapter_queue_add(id, eth_dev_id,
2665 rx_queue_id, queue_conf, ret);
2673 rxa_sw_vector_limits(struct rte_event_eth_rx_adapter_vector_limits *limits)
2675 limits->max_sz = MAX_VECTOR_SIZE;
2676 limits->min_sz = MIN_VECTOR_SIZE;
2677 limits->max_timeout_ns = MAX_VECTOR_NS;
2678 limits->min_timeout_ns = MIN_VECTOR_NS;
2684 rte_event_eth_rx_adapter_queue_del(uint8_t id, uint16_t eth_dev_id,
2685 int32_t rx_queue_id)
2688 struct rte_eventdev *dev;
2689 struct event_eth_rx_adapter *rx_adapter;
2690 struct eth_device_info *dev_info;
2692 uint32_t nb_rx_poll = 0;
2693 uint32_t nb_wrr = 0;
2694 uint32_t nb_rx_intr;
2695 struct eth_rx_poll_entry *rx_poll = NULL;
2696 uint32_t *rx_wrr = NULL;
2699 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2700 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
2702 rx_adapter = rxa_id_to_adapter(id);
2703 if (rx_adapter == NULL)
2706 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2707 ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
2713 if (rx_queue_id != -1 && (uint16_t)rx_queue_id >=
2714 rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
2715 RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16,
2716 (uint16_t)rx_queue_id);
2720 dev_info = &rx_adapter->eth_devices[eth_dev_id];
2722 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
2723 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->eth_rx_adapter_queue_del,
2725 ret = (*dev->dev_ops->eth_rx_adapter_queue_del)(dev,
2726 &rte_eth_devices[eth_dev_id],
2729 rxa_update_queue(rx_adapter,
2730 &rx_adapter->eth_devices[eth_dev_id],
2733 if (dev_info->nb_dev_queues == 0) {
2734 rte_free(dev_info->rx_queue);
2735 dev_info->rx_queue = NULL;
2739 rxa_calc_nb_post_del(rx_adapter, dev_info, rx_queue_id,
2740 &nb_rx_poll, &nb_rx_intr, &nb_wrr);
2742 ret = rxa_alloc_poll_arrays(rx_adapter, nb_rx_poll, nb_wrr,
2747 rte_spinlock_lock(&rx_adapter->rx_lock);
2750 if (rx_adapter->num_rx_intr > nb_rx_intr) {
2752 num_intr_vec = rxa_nb_intr_vect(dev_info,
2754 ret = rxa_del_intr_queue(rx_adapter, dev_info,
2760 if (nb_rx_intr == 0) {
2761 ret = rxa_free_intr_resources(rx_adapter);
2766 rxa_sw_del(rx_adapter, dev_info, rx_queue_id);
2767 rxa_calc_wrr_sequence(rx_adapter, rx_poll, rx_wrr);
2769 rte_free(rx_adapter->eth_rx_poll);
2770 rte_free(rx_adapter->wrr_sched);
2772 if (nb_rx_intr == 0) {
2773 rte_free(dev_info->intr_queue);
2774 dev_info->intr_queue = NULL;
2777 rx_adapter->eth_rx_poll = rx_poll;
2778 rx_adapter->wrr_sched = rx_wrr;
2779 rx_adapter->wrr_len = nb_wrr;
2781 * reset next poll start position (wrr_pos) to avoid buffer
2782 * overrun when wrr_len is reduced in case of queue delete
2784 rx_adapter->wrr_pos = 0;
2785 rx_adapter->num_intr_vec += num_intr_vec;
2787 if (dev_info->nb_dev_queues == 0) {
2788 rte_free(dev_info->rx_queue);
2789 dev_info->rx_queue = NULL;
2792 rte_spinlock_unlock(&rx_adapter->rx_lock);
2799 rte_service_component_runstate_set(rx_adapter->service_id,
2800 rxa_sw_adapter_queue_count(rx_adapter));
2803 rte_eventdev_trace_eth_rx_adapter_queue_del(id, eth_dev_id,
2809 rte_event_eth_rx_adapter_vector_limits_get(
2810 uint8_t dev_id, uint16_t eth_port_id,
2811 struct rte_event_eth_rx_adapter_vector_limits *limits)
2813 struct rte_eventdev *dev;
2817 RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
2818 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_port_id, -EINVAL);
2823 dev = &rte_eventdevs[dev_id];
2825 ret = rte_event_eth_rx_adapter_caps_get(dev_id, eth_port_id, &cap);
2827 RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
2828 "eth port %" PRIu16,
2829 dev_id, eth_port_id);
2833 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
2834 RTE_FUNC_PTR_OR_ERR_RET(
2835 *dev->dev_ops->eth_rx_adapter_vector_limits_get,
2837 ret = dev->dev_ops->eth_rx_adapter_vector_limits_get(
2838 dev, &rte_eth_devices[eth_port_id], limits);
2840 ret = rxa_sw_vector_limits(limits);
2847 rte_event_eth_rx_adapter_start(uint8_t id)
2849 rte_eventdev_trace_eth_rx_adapter_start(id);
2850 return rxa_ctrl(id, 1);
2854 rte_event_eth_rx_adapter_stop(uint8_t id)
2856 rte_eventdev_trace_eth_rx_adapter_stop(id);
2857 return rxa_ctrl(id, 0);
2861 rxa_queue_stats_reset(struct eth_rx_queue_info *queue_info)
2863 struct rte_event_eth_rx_adapter_stats *q_stats;
2865 q_stats = queue_info->stats;
2866 memset(q_stats, 0, sizeof(*q_stats));
2870 rte_event_eth_rx_adapter_stats_get(uint8_t id,
2871 struct rte_event_eth_rx_adapter_stats *stats)
2873 struct event_eth_rx_adapter *rx_adapter;
2874 struct eth_event_enqueue_buffer *buf;
2875 struct rte_event_eth_rx_adapter_stats dev_stats_sum = { 0 };
2876 struct rte_event_eth_rx_adapter_stats dev_stats;
2877 struct rte_eventdev *dev;
2878 struct eth_device_info *dev_info;
2879 struct eth_rx_queue_info *queue_info;
2880 struct rte_event_eth_rx_adapter_stats *q_stats;
2884 if (rxa_memzone_lookup())
2887 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2889 rx_adapter = rxa_id_to_adapter(id);
2890 if (rx_adapter == NULL || stats == NULL)
2893 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2894 memset(stats, 0, sizeof(*stats));
2896 if (rx_adapter->service_inited)
2897 *stats = rx_adapter->stats;
2899 RTE_ETH_FOREACH_DEV(i) {
2900 dev_info = &rx_adapter->eth_devices[i];
2902 if (rx_adapter->use_queue_event_buf && dev_info->rx_queue) {
2904 for (j = 0; j < dev_info->dev->data->nb_rx_queues;
2906 queue_info = &dev_info->rx_queue[j];
2907 if (!queue_info->queue_enabled)
2909 q_stats = queue_info->stats;
2911 stats->rx_packets += q_stats->rx_packets;
2912 stats->rx_poll_count += q_stats->rx_poll_count;
2913 stats->rx_enq_count += q_stats->rx_enq_count;
2914 stats->rx_enq_retry += q_stats->rx_enq_retry;
2915 stats->rx_dropped += q_stats->rx_dropped;
2916 stats->rx_enq_block_cycles +=
2917 q_stats->rx_enq_block_cycles;
2921 if (dev_info->internal_event_port == 0 ||
2922 dev->dev_ops->eth_rx_adapter_stats_get == NULL)
2924 ret = (*dev->dev_ops->eth_rx_adapter_stats_get)(dev,
2925 &rte_eth_devices[i],
2929 dev_stats_sum.rx_packets += dev_stats.rx_packets;
2930 dev_stats_sum.rx_enq_count += dev_stats.rx_enq_count;
2933 buf = &rx_adapter->event_enqueue_buffer;
2934 stats->rx_packets += dev_stats_sum.rx_packets;
2935 stats->rx_enq_count += dev_stats_sum.rx_enq_count;
2936 stats->rx_event_buf_count = buf->count;
2937 stats->rx_event_buf_size = buf->events_size;
2943 rte_event_eth_rx_adapter_queue_stats_get(uint8_t id,
2944 uint16_t eth_dev_id,
2945 uint16_t rx_queue_id,
2946 struct rte_event_eth_rx_adapter_queue_stats *stats)
2948 struct event_eth_rx_adapter *rx_adapter;
2949 struct eth_device_info *dev_info;
2950 struct eth_rx_queue_info *queue_info;
2951 struct eth_event_enqueue_buffer *event_buf;
2952 struct rte_event_eth_rx_adapter_stats *q_stats;
2953 struct rte_eventdev *dev;
2955 if (rxa_memzone_lookup())
2958 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2959 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
2961 rx_adapter = rxa_id_to_adapter(id);
2963 if (rx_adapter == NULL || stats == NULL)
2966 if (!rx_adapter->use_queue_event_buf)
2969 if (rx_queue_id >= rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
2970 RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16, rx_queue_id);
2974 dev_info = &rx_adapter->eth_devices[eth_dev_id];
2975 if (dev_info->rx_queue == NULL ||
2976 !dev_info->rx_queue[rx_queue_id].queue_enabled) {
2977 RTE_EDEV_LOG_ERR("Rx queue %u not added", rx_queue_id);
2981 queue_info = &dev_info->rx_queue[rx_queue_id];
2982 event_buf = queue_info->event_buf;
2983 q_stats = queue_info->stats;
2985 stats->rx_event_buf_count = event_buf->count;
2986 stats->rx_event_buf_size = event_buf->events_size;
2987 stats->rx_packets = q_stats->rx_packets;
2988 stats->rx_poll_count = q_stats->rx_poll_count;
2989 stats->rx_dropped = q_stats->rx_dropped;
2991 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2992 if (dev->dev_ops->eth_rx_adapter_queue_stats_get != NULL) {
2993 return (*dev->dev_ops->eth_rx_adapter_queue_stats_get)(dev,
2994 &rte_eth_devices[eth_dev_id],
2995 rx_queue_id, stats);
3002 rte_event_eth_rx_adapter_stats_reset(uint8_t id)
3004 struct event_eth_rx_adapter *rx_adapter;
3005 struct rte_eventdev *dev;
3006 struct eth_device_info *dev_info;
3007 struct eth_rx_queue_info *queue_info;
3010 if (rxa_memzone_lookup())
3013 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
3015 rx_adapter = rxa_id_to_adapter(id);
3016 if (rx_adapter == NULL)
3019 dev = &rte_eventdevs[rx_adapter->eventdev_id];
3021 RTE_ETH_FOREACH_DEV(i) {
3022 dev_info = &rx_adapter->eth_devices[i];
3024 if (rx_adapter->use_queue_event_buf && dev_info->rx_queue) {
3026 for (j = 0; j < dev_info->dev->data->nb_rx_queues;
3028 queue_info = &dev_info->rx_queue[j];
3029 if (!queue_info->queue_enabled)
3031 rxa_queue_stats_reset(queue_info);
3035 if (dev_info->internal_event_port == 0 ||
3036 dev->dev_ops->eth_rx_adapter_stats_reset == NULL)
3038 (*dev->dev_ops->eth_rx_adapter_stats_reset)(dev,
3039 &rte_eth_devices[i]);
3042 memset(&rx_adapter->stats, 0, sizeof(rx_adapter->stats));
3048 rte_event_eth_rx_adapter_queue_stats_reset(uint8_t id,
3049 uint16_t eth_dev_id,
3050 uint16_t rx_queue_id)
3052 struct event_eth_rx_adapter *rx_adapter;
3053 struct eth_device_info *dev_info;
3054 struct eth_rx_queue_info *queue_info;
3055 struct rte_eventdev *dev;
3057 if (rxa_memzone_lookup())
3060 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
3061 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
3063 rx_adapter = rxa_id_to_adapter(id);
3064 if (rx_adapter == NULL)
3067 if (!rx_adapter->use_queue_event_buf)
3070 if (rx_queue_id >= rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
3071 RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16, rx_queue_id);
3075 dev_info = &rx_adapter->eth_devices[eth_dev_id];
3077 if (dev_info->rx_queue == NULL ||
3078 !dev_info->rx_queue[rx_queue_id].queue_enabled) {
3079 RTE_EDEV_LOG_ERR("Rx queue %u not added", rx_queue_id);
3083 queue_info = &dev_info->rx_queue[rx_queue_id];
3084 rxa_queue_stats_reset(queue_info);
3086 dev = &rte_eventdevs[rx_adapter->eventdev_id];
3087 if (dev->dev_ops->eth_rx_adapter_queue_stats_reset != NULL) {
3088 return (*dev->dev_ops->eth_rx_adapter_queue_stats_reset)(dev,
3089 &rte_eth_devices[eth_dev_id],
3097 rte_event_eth_rx_adapter_service_id_get(uint8_t id, uint32_t *service_id)
3099 struct event_eth_rx_adapter *rx_adapter;
3101 if (rxa_memzone_lookup())
3104 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
3106 rx_adapter = rxa_id_to_adapter(id);
3107 if (rx_adapter == NULL || service_id == NULL)
3110 if (rx_adapter->service_inited)
3111 *service_id = rx_adapter->service_id;
3113 return rx_adapter->service_inited ? 0 : -ESRCH;
3117 rte_event_eth_rx_adapter_cb_register(uint8_t id,
3118 uint16_t eth_dev_id,
3119 rte_event_eth_rx_adapter_cb_fn cb_fn,
3122 struct event_eth_rx_adapter *rx_adapter;
3123 struct eth_device_info *dev_info;
3127 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
3128 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
3130 rx_adapter = rxa_id_to_adapter(id);
3131 if (rx_adapter == NULL)
3134 dev_info = &rx_adapter->eth_devices[eth_dev_id];
3135 if (dev_info->rx_queue == NULL)
3138 ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
3142 RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
3143 "eth port %" PRIu16, id, eth_dev_id);
3147 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
3148 RTE_EDEV_LOG_ERR("Rx callback not supported for eth port %"
3149 PRIu16, eth_dev_id);
3153 rte_spinlock_lock(&rx_adapter->rx_lock);
3154 dev_info->cb_fn = cb_fn;
3155 dev_info->cb_arg = cb_arg;
3156 rte_spinlock_unlock(&rx_adapter->rx_lock);
3162 rte_event_eth_rx_adapter_queue_conf_get(uint8_t id,
3163 uint16_t eth_dev_id,
3164 uint16_t rx_queue_id,
3165 struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
3167 struct rte_eventdev *dev;
3168 struct event_eth_rx_adapter *rx_adapter;
3169 struct eth_device_info *dev_info;
3170 struct eth_rx_queue_info *queue_info;
3171 struct rte_event *qi_ev;
3174 if (rxa_memzone_lookup())
3177 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
3178 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
3180 if (rx_queue_id >= rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
3181 RTE_EDEV_LOG_ERR("Invalid rx queue_id %u", rx_queue_id);
3185 if (queue_conf == NULL) {
3186 RTE_EDEV_LOG_ERR("Rx queue conf struct cannot be NULL");
3190 rx_adapter = rxa_id_to_adapter(id);
3191 if (rx_adapter == NULL)
3194 dev_info = &rx_adapter->eth_devices[eth_dev_id];
3195 if (dev_info->rx_queue == NULL ||
3196 !dev_info->rx_queue[rx_queue_id].queue_enabled) {
3197 RTE_EDEV_LOG_ERR("Rx queue %u not added", rx_queue_id);
3201 queue_info = &dev_info->rx_queue[rx_queue_id];
3202 qi_ev = (struct rte_event *)&queue_info->event;
3204 memset(queue_conf, 0, sizeof(*queue_conf));
3205 queue_conf->rx_queue_flags = 0;
3206 if (queue_info->flow_id_mask != 0)
3207 queue_conf->rx_queue_flags |=
3208 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID;
3209 queue_conf->servicing_weight = queue_info->wt;
3211 memcpy(&queue_conf->ev, qi_ev, sizeof(*qi_ev));
3213 dev = &rte_eventdevs[rx_adapter->eventdev_id];
3214 if (dev->dev_ops->eth_rx_adapter_queue_conf_get != NULL) {
3215 ret = (*dev->dev_ops->eth_rx_adapter_queue_conf_get)(dev,
3216 &rte_eth_devices[eth_dev_id],
3225 #define RXA_ADD_DICT(stats, s) rte_tel_data_add_dict_u64(d, #s, stats.s)
3228 handle_rxa_stats(const char *cmd __rte_unused,
3230 struct rte_tel_data *d)
3232 uint8_t rx_adapter_id;
3233 struct rte_event_eth_rx_adapter_stats rx_adptr_stats;
3235 if (params == NULL || strlen(params) == 0 || !isdigit(*params))
3238 /* Get Rx adapter ID from parameter string */
3239 rx_adapter_id = atoi(params);
3240 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(rx_adapter_id, -EINVAL);
3242 /* Get Rx adapter stats */
3243 if (rte_event_eth_rx_adapter_stats_get(rx_adapter_id,
3245 RTE_EDEV_LOG_ERR("Failed to get Rx adapter stats\n");
3249 rte_tel_data_start_dict(d);
3250 rte_tel_data_add_dict_u64(d, "rx_adapter_id", rx_adapter_id);
3251 RXA_ADD_DICT(rx_adptr_stats, rx_packets);
3252 RXA_ADD_DICT(rx_adptr_stats, rx_poll_count);
3253 RXA_ADD_DICT(rx_adptr_stats, rx_dropped);
3254 RXA_ADD_DICT(rx_adptr_stats, rx_enq_retry);
3255 RXA_ADD_DICT(rx_adptr_stats, rx_event_buf_count);
3256 RXA_ADD_DICT(rx_adptr_stats, rx_event_buf_size);
3257 RXA_ADD_DICT(rx_adptr_stats, rx_enq_count);
3258 RXA_ADD_DICT(rx_adptr_stats, rx_enq_start_ts);
3259 RXA_ADD_DICT(rx_adptr_stats, rx_enq_block_cycles);
3260 RXA_ADD_DICT(rx_adptr_stats, rx_enq_end_ts);
3261 RXA_ADD_DICT(rx_adptr_stats, rx_intr_packets);
3267 handle_rxa_stats_reset(const char *cmd __rte_unused,
3269 struct rte_tel_data *d __rte_unused)
3271 uint8_t rx_adapter_id;
3273 if (params == NULL || strlen(params) == 0 || ~isdigit(*params))
3276 /* Get Rx adapter ID from parameter string */
3277 rx_adapter_id = atoi(params);
3278 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(rx_adapter_id, -EINVAL);
3280 /* Reset Rx adapter stats */
3281 if (rte_event_eth_rx_adapter_stats_reset(rx_adapter_id)) {
3282 RTE_EDEV_LOG_ERR("Failed to reset Rx adapter stats\n");
3290 handle_rxa_get_queue_conf(const char *cmd __rte_unused,
3292 struct rte_tel_data *d)
3294 uint8_t rx_adapter_id;
3295 uint16_t rx_queue_id;
3297 char *token, *l_params;
3298 struct rte_event_eth_rx_adapter_queue_conf queue_conf;
3300 if (params == NULL || strlen(params) == 0 || !isdigit(*params))
3303 /* Get Rx adapter ID from parameter string */
3304 l_params = strdup(params);
3305 token = strtok(l_params, ",");
3306 rx_adapter_id = strtoul(token, NULL, 10);
3307 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(rx_adapter_id, -EINVAL);
3309 token = strtok(NULL, ",");
3310 if (token == NULL || strlen(token) == 0 || !isdigit(*token))
3313 /* Get device ID from parameter string */
3314 eth_dev_id = strtoul(token, NULL, 10);
3315 RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(eth_dev_id, -EINVAL);
3317 token = strtok(NULL, ",");
3318 if (token == NULL || strlen(token) == 0 || !isdigit(*token))
3321 /* Get Rx queue ID from parameter string */
3322 rx_queue_id = strtoul(token, NULL, 10);
3323 if (rx_queue_id >= rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
3324 RTE_EDEV_LOG_ERR("Invalid rx queue_id %u", rx_queue_id);
3328 token = strtok(NULL, "\0");
3330 RTE_EDEV_LOG_ERR("Extra parameters passed to eventdev"
3331 " telemetry command, igrnoring");
3333 if (rte_event_eth_rx_adapter_queue_conf_get(rx_adapter_id, eth_dev_id,
3334 rx_queue_id, &queue_conf)) {
3335 RTE_EDEV_LOG_ERR("Failed to get Rx adapter queue config");
3339 rte_tel_data_start_dict(d);
3340 rte_tel_data_add_dict_u64(d, "rx_adapter_id", rx_adapter_id);
3341 rte_tel_data_add_dict_u64(d, "eth_dev_id", eth_dev_id);
3342 rte_tel_data_add_dict_u64(d, "rx_queue_id", rx_queue_id);
3343 RXA_ADD_DICT(queue_conf, rx_queue_flags);
3344 RXA_ADD_DICT(queue_conf, servicing_weight);
3345 RXA_ADD_DICT(queue_conf.ev, queue_id);
3346 RXA_ADD_DICT(queue_conf.ev, sched_type);
3347 RXA_ADD_DICT(queue_conf.ev, priority);
3348 RXA_ADD_DICT(queue_conf.ev, flow_id);
3354 handle_rxa_get_queue_stats(const char *cmd __rte_unused,
3356 struct rte_tel_data *d)
3358 uint8_t rx_adapter_id;
3359 uint16_t rx_queue_id;
3361 char *token, *l_params;
3362 struct rte_event_eth_rx_adapter_queue_stats q_stats;
3364 if (params == NULL || strlen(params) == 0 || !isdigit(*params))
3367 /* Get Rx adapter ID from parameter string */
3368 l_params = strdup(params);
3369 token = strtok(l_params, ",");
3370 rx_adapter_id = strtoul(token, NULL, 10);
3371 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(rx_adapter_id, -EINVAL);
3373 token = strtok(NULL, ",");
3374 if (token == NULL || strlen(token) == 0 || !isdigit(*token))
3377 /* Get device ID from parameter string */
3378 eth_dev_id = strtoul(token, NULL, 10);
3379 RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(eth_dev_id, -EINVAL);
3381 token = strtok(NULL, ",");
3382 if (token == NULL || strlen(token) == 0 || !isdigit(*token))
3385 /* Get Rx queue ID from parameter string */
3386 rx_queue_id = strtoul(token, NULL, 10);
3387 if (rx_queue_id >= rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
3388 RTE_EDEV_LOG_ERR("Invalid rx queue_id %u", rx_queue_id);
3392 token = strtok(NULL, "\0");
3394 RTE_EDEV_LOG_ERR("Extra parameters passed to eventdev"
3395 " telemetry command, igrnoring");
3397 if (rte_event_eth_rx_adapter_queue_stats_get(rx_adapter_id, eth_dev_id,
3398 rx_queue_id, &q_stats)) {
3399 RTE_EDEV_LOG_ERR("Failed to get Rx adapter queue stats");
3403 rte_tel_data_start_dict(d);
3404 rte_tel_data_add_dict_u64(d, "rx_adapter_id", rx_adapter_id);
3405 rte_tel_data_add_dict_u64(d, "eth_dev_id", eth_dev_id);
3406 rte_tel_data_add_dict_u64(d, "rx_queue_id", rx_queue_id);
3407 RXA_ADD_DICT(q_stats, rx_event_buf_count);
3408 RXA_ADD_DICT(q_stats, rx_event_buf_size);
3409 RXA_ADD_DICT(q_stats, rx_poll_count);
3410 RXA_ADD_DICT(q_stats, rx_packets);
3411 RXA_ADD_DICT(q_stats, rx_dropped);
3417 handle_rxa_queue_stats_reset(const char *cmd __rte_unused,
3419 struct rte_tel_data *d __rte_unused)
3421 uint8_t rx_adapter_id;
3422 uint16_t rx_queue_id;
3424 char *token, *l_params;
3426 if (params == NULL || strlen(params) == 0 || !isdigit(*params))
3429 /* Get Rx adapter ID from parameter string */
3430 l_params = strdup(params);
3431 token = strtok(l_params, ",");
3432 rx_adapter_id = strtoul(token, NULL, 10);
3433 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(rx_adapter_id, -EINVAL);
3435 token = strtok(NULL, ",");
3436 if (token == NULL || strlen(token) == 0 || !isdigit(*token))
3439 /* Get device ID from parameter string */
3440 eth_dev_id = strtoul(token, NULL, 10);
3441 RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(eth_dev_id, -EINVAL);
3443 token = strtok(NULL, ",");
3444 if (token == NULL || strlen(token) == 0 || !isdigit(*token))
3447 /* Get Rx queue ID from parameter string */
3448 rx_queue_id = strtoul(token, NULL, 10);
3449 if (rx_queue_id >= rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
3450 RTE_EDEV_LOG_ERR("Invalid rx queue_id %u", rx_queue_id);
3454 token = strtok(NULL, "\0");
3456 RTE_EDEV_LOG_ERR("Extra parameters passed to eventdev"
3457 " telemetry command, igrnoring");
3459 if (rte_event_eth_rx_adapter_queue_stats_reset(rx_adapter_id,
3462 RTE_EDEV_LOG_ERR("Failed to reset Rx adapter queue stats");
3469 RTE_INIT(rxa_init_telemetry)
3471 rte_telemetry_register_cmd("/eventdev/rxa_stats",
3473 "Returns Rx adapter stats. Parameter: rxa_id");
3475 rte_telemetry_register_cmd("/eventdev/rxa_stats_reset",
3476 handle_rxa_stats_reset,
3477 "Reset Rx adapter stats. Parameter: rxa_id");
3479 rte_telemetry_register_cmd("/eventdev/rxa_queue_conf",
3480 handle_rxa_get_queue_conf,
3481 "Returns Rx queue config. Parameter: rxa_id, dev_id, queue_id");
3483 rte_telemetry_register_cmd("/eventdev/rxa_queue_stats",
3484 handle_rxa_get_queue_stats,
3485 "Returns Rx queue stats. Parameter: rxa_id, dev_id, queue_id");
3487 rte_telemetry_register_cmd("/eventdev/rxa_queue_stats_reset",
3488 handle_rxa_queue_stats_reset,
3489 "Reset Rx queue stats. Parameter: rxa_id, dev_id, queue_id");