1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation.
10 #include <rte_cycles.h>
11 #include <rte_common.h>
13 #include <rte_errno.h>
14 #include <ethdev_driver.h>
16 #include <rte_malloc.h>
17 #include <rte_service_component.h>
18 #include <rte_thash.h>
19 #include <rte_interrupts.h>
20 #include <rte_mbuf_dyn.h>
21 #include <rte_telemetry.h>
23 #include "rte_eventdev.h"
24 #include "eventdev_pmd.h"
25 #include "rte_eventdev_trace.h"
26 #include "rte_event_eth_rx_adapter.h"
29 #define BLOCK_CNT_THRESHOLD 10
30 #define ETH_EVENT_BUFFER_SIZE (6*BATCH_SIZE)
31 #define MAX_VECTOR_SIZE 1024
32 #define MIN_VECTOR_SIZE 4
33 #define MAX_VECTOR_NS 1E9
34 #define MIN_VECTOR_NS 1E5
36 #define ETH_RX_ADAPTER_SERVICE_NAME_LEN 32
37 #define ETH_RX_ADAPTER_MEM_NAME_LEN 32
39 #define RSS_KEY_SIZE 40
40 /* value written to intr thread pipe to signal thread exit */
41 #define ETH_BRIDGE_INTR_THREAD_EXIT 1
42 /* Sentinel value to detect initialized file handle */
45 #define RXA_ADAPTER_ARRAY "rte_event_eth_rx_adapter_array"
48 * Used to store port and queue ID of interrupting Rx queue
60 * There is an instance of this struct per polled Rx queue added to the
63 struct eth_rx_poll_entry {
64 /* Eth port to poll */
66 /* Eth rx queue to poll */
70 struct eth_rx_vector_data {
71 TAILQ_ENTRY(eth_rx_vector_data) next;
74 uint16_t max_vector_count;
77 uint64_t vector_timeout_ticks;
78 struct rte_mempool *vector_pool;
79 struct rte_event_vector *vector_ev;
80 } __rte_cache_aligned;
82 TAILQ_HEAD(eth_rx_vector_data_list, eth_rx_vector_data);
84 /* Instance per adapter */
85 struct rte_eth_event_enqueue_buffer {
86 /* Count of events in this buffer */
88 /* Array of events in this buffer */
89 struct rte_event *events;
90 /* size of event buffer */
92 /* Event enqueue happens from head */
94 /* New packets from rte_eth_rx_burst is enqued from tail */
96 /* last element in the buffer before rollover */
101 struct rte_event_eth_rx_adapter {
103 uint8_t rss_key_be[RSS_KEY_SIZE];
104 /* Event device identifier */
106 /* Event port identifier */
107 uint8_t event_port_id;
108 /* Flag indicating per rxq event buffer */
109 bool use_queue_event_buf;
110 /* Per ethernet device structure */
111 struct eth_device_info *eth_devices;
112 /* Lock to serialize config updates with service function */
113 rte_spinlock_t rx_lock;
114 /* Max mbufs processed in any service function invocation */
116 /* Receive queues that need to be polled */
117 struct eth_rx_poll_entry *eth_rx_poll;
118 /* Size of the eth_rx_poll array */
119 uint16_t num_rx_polled;
120 /* Weighted round robin schedule */
122 /* wrr_sched[] size */
124 /* Next entry in wrr[] to begin polling */
126 /* Event burst buffer */
127 struct rte_eth_event_enqueue_buffer event_enqueue_buffer;
128 /* Vector enable flag */
130 /* Timestamp of previous vector expiry list traversal */
131 uint64_t prev_expiry_ts;
132 /* Minimum ticks to wait before traversing expiry list */
133 uint64_t vector_tmo_ticks;
135 struct eth_rx_vector_data_list vector_list;
136 /* Per adapter stats */
137 struct rte_event_eth_rx_adapter_stats stats;
138 /* Block count, counts up to BLOCK_CNT_THRESHOLD */
139 uint16_t enq_block_count;
141 uint64_t rx_enq_block_start_ts;
142 /* epoll fd used to wait for Rx interrupts */
144 /* Num of interrupt driven interrupt queues */
145 uint32_t num_rx_intr;
146 /* Used to send <dev id, queue id> of interrupting Rx queues from
147 * the interrupt thread to the Rx thread
149 struct rte_ring *intr_ring;
150 /* Rx Queue data (dev id, queue id) for the last non-empty
154 /* queue_data is valid */
156 /* Interrupt ring lock, synchronizes Rx thread
157 * and interrupt thread
159 rte_spinlock_t intr_ring_lock;
160 /* event array passed to rte_poll_wait */
161 struct rte_epoll_event *epoll_events;
162 /* Count of interrupt vectors in use */
163 uint32_t num_intr_vec;
164 /* Thread blocked on Rx interrupts */
165 pthread_t rx_intr_thread;
166 /* Configuration callback for rte_service configuration */
167 rte_event_eth_rx_adapter_conf_cb conf_cb;
168 /* Configuration callback argument */
170 /* Set if default_cb is being used */
172 /* Service initialization state */
173 uint8_t service_inited;
174 /* Total count of Rx queues in adapter */
176 /* Memory allocation name */
177 char mem_name[ETH_RX_ADAPTER_MEM_NAME_LEN];
178 /* Socket identifier cached from eventdev */
180 /* Per adapter EAL service */
182 /* Adapter started flag */
186 } __rte_cache_aligned;
189 struct eth_device_info {
190 struct rte_eth_dev *dev;
191 struct eth_rx_queue_info *rx_queue;
193 rte_event_eth_rx_adapter_cb_fn cb_fn;
194 /* Rx callback argument */
196 /* Set if ethdev->eventdev packet transfer uses a
199 uint8_t internal_event_port;
200 /* Set if the adapter is processing rx queues for
201 * this eth device and packet processing has been
202 * started, allows for the code to know if the PMD
203 * rx_adapter_stop callback needs to be invoked
205 uint8_t dev_rx_started;
206 /* Number of queues added for this device */
207 uint16_t nb_dev_queues;
208 /* Number of poll based queues
209 * If nb_rx_poll > 0, the start callback will
210 * be invoked if not already invoked
213 /* Number of interrupt based queues
214 * If nb_rx_intr > 0, the start callback will
215 * be invoked if not already invoked.
218 /* Number of queues that use the shared interrupt */
219 uint16_t nb_shared_intr;
220 /* sum(wrr(q)) for all queues within the device
221 * useful when deleting all device queues
224 /* Intr based queue index to start polling from, this is used
225 * if the number of shared interrupts is non-zero
228 /* Intr based queue indices */
229 uint16_t *intr_queue;
230 /* device generates per Rx queue interrupt for queue index
231 * for queue indices < RTE_MAX_RXTX_INTR_VEC_ID - 1
234 /* shared interrupt enabled */
235 int shared_intr_enabled;
239 struct eth_rx_queue_info {
240 int queue_enabled; /* True if added */
243 uint16_t wt; /* Polling weight */
244 uint32_t flow_id_mask; /* Set to ~0 if app provides flow id else 0 */
246 struct eth_rx_vector_data vector_data;
247 struct rte_eth_event_enqueue_buffer *event_buf;
250 static struct rte_event_eth_rx_adapter **event_eth_rx_adapter;
252 /* Enable dynamic timestamp field in mbuf */
253 static uint64_t event_eth_rx_timestamp_dynflag;
254 static int event_eth_rx_timestamp_dynfield_offset = -1;
256 static inline rte_mbuf_timestamp_t *
257 rxa_timestamp_dynfield(struct rte_mbuf *mbuf)
259 return RTE_MBUF_DYNFIELD(mbuf,
260 event_eth_rx_timestamp_dynfield_offset, rte_mbuf_timestamp_t *);
264 rxa_validate_id(uint8_t id)
266 return id < RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE;
269 static inline struct rte_eth_event_enqueue_buffer *
270 rxa_event_buf_get(struct rte_event_eth_rx_adapter *rx_adapter,
271 uint16_t eth_dev_id, uint16_t rx_queue_id)
273 if (rx_adapter->use_queue_event_buf) {
274 struct eth_device_info *dev_info =
275 &rx_adapter->eth_devices[eth_dev_id];
276 return dev_info->rx_queue[rx_queue_id].event_buf;
278 return &rx_adapter->event_enqueue_buffer;
281 #define RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \
282 if (!rxa_validate_id(id)) { \
283 RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d\n", id); \
289 rxa_sw_adapter_queue_count(struct rte_event_eth_rx_adapter *rx_adapter)
291 return rx_adapter->num_rx_polled + rx_adapter->num_rx_intr;
294 /* Greatest common divisor */
295 static uint16_t rxa_gcd_u16(uint16_t a, uint16_t b)
299 return r ? rxa_gcd_u16(b, r) : b;
302 /* Returns the next queue in the polling sequence
304 * http://kb.linuxvirtualserver.org/wiki/Weighted_Round-Robin_Scheduling
307 rxa_wrr_next(struct rte_event_eth_rx_adapter *rx_adapter,
308 unsigned int n, int *cw,
309 struct eth_rx_poll_entry *eth_rx_poll, uint16_t max_wt,
310 uint16_t gcd, int prev)
326 q = eth_rx_poll[i].eth_rx_qid;
327 d = eth_rx_poll[i].eth_dev_id;
328 w = rx_adapter->eth_devices[d].rx_queue[q].wt;
336 rxa_shared_intr(struct eth_device_info *dev_info,
341 if (dev_info->dev->intr_handle == NULL)
344 multi_intr_cap = rte_intr_cap_multiple(dev_info->dev->intr_handle);
345 return !multi_intr_cap ||
346 rx_queue_id >= RTE_MAX_RXTX_INTR_VEC_ID - 1;
350 rxa_intr_queue(struct eth_device_info *dev_info,
353 struct eth_rx_queue_info *queue_info;
355 queue_info = &dev_info->rx_queue[rx_queue_id];
356 return dev_info->rx_queue &&
357 !dev_info->internal_event_port &&
358 queue_info->queue_enabled && queue_info->wt == 0;
362 rxa_polled_queue(struct eth_device_info *dev_info,
365 struct eth_rx_queue_info *queue_info;
367 queue_info = &dev_info->rx_queue[rx_queue_id];
368 return !dev_info->internal_event_port &&
369 dev_info->rx_queue &&
370 queue_info->queue_enabled && queue_info->wt != 0;
373 /* Calculate change in number of vectors after Rx queue ID is add/deleted */
375 rxa_nb_intr_vect(struct eth_device_info *dev_info, int rx_queue_id, int add)
381 nbq = dev_info->dev->data->nb_rx_queues;
382 n = 0; /* non shared count */
383 s = 0; /* shared count */
385 if (rx_queue_id == -1) {
386 for (i = 0; i < nbq; i++) {
387 if (!rxa_shared_intr(dev_info, i))
388 n += add ? !rxa_intr_queue(dev_info, i) :
389 rxa_intr_queue(dev_info, i);
391 s += add ? !rxa_intr_queue(dev_info, i) :
392 rxa_intr_queue(dev_info, i);
396 if ((add && dev_info->nb_shared_intr == 0) ||
397 (!add && dev_info->nb_shared_intr))
401 if (!rxa_shared_intr(dev_info, rx_queue_id))
402 n = add ? !rxa_intr_queue(dev_info, rx_queue_id) :
403 rxa_intr_queue(dev_info, rx_queue_id);
405 n = add ? !dev_info->nb_shared_intr :
406 dev_info->nb_shared_intr == 1;
412 /* Calculate nb_rx_intr after deleting interrupt mode rx queues
415 rxa_calc_nb_post_intr_del(struct rte_event_eth_rx_adapter *rx_adapter,
416 struct eth_device_info *dev_info,
418 uint32_t *nb_rx_intr)
422 if (rx_queue_id == -1)
423 intr_diff = dev_info->nb_rx_intr;
425 intr_diff = rxa_intr_queue(dev_info, rx_queue_id);
427 *nb_rx_intr = rx_adapter->num_rx_intr - intr_diff;
430 /* Calculate nb_rx_* after adding interrupt mode rx queues, newly added
431 * interrupt queues could currently be poll mode Rx queues
434 rxa_calc_nb_post_add_intr(struct rte_event_eth_rx_adapter *rx_adapter,
435 struct eth_device_info *dev_info,
437 uint32_t *nb_rx_poll,
438 uint32_t *nb_rx_intr,
443 uint32_t wrr_len_diff;
445 if (rx_queue_id == -1) {
446 intr_diff = dev_info->dev->data->nb_rx_queues -
447 dev_info->nb_rx_intr;
448 poll_diff = dev_info->nb_rx_poll;
449 wrr_len_diff = dev_info->wrr_len;
451 intr_diff = !rxa_intr_queue(dev_info, rx_queue_id);
452 poll_diff = rxa_polled_queue(dev_info, rx_queue_id);
453 wrr_len_diff = poll_diff ? dev_info->rx_queue[rx_queue_id].wt :
457 *nb_rx_intr = rx_adapter->num_rx_intr + intr_diff;
458 *nb_rx_poll = rx_adapter->num_rx_polled - poll_diff;
459 *nb_wrr = rx_adapter->wrr_len - wrr_len_diff;
462 /* Calculate size of the eth_rx_poll and wrr_sched arrays
463 * after deleting poll mode rx queues
466 rxa_calc_nb_post_poll_del(struct rte_event_eth_rx_adapter *rx_adapter,
467 struct eth_device_info *dev_info,
469 uint32_t *nb_rx_poll,
473 uint32_t wrr_len_diff;
475 if (rx_queue_id == -1) {
476 poll_diff = dev_info->nb_rx_poll;
477 wrr_len_diff = dev_info->wrr_len;
479 poll_diff = rxa_polled_queue(dev_info, rx_queue_id);
480 wrr_len_diff = poll_diff ? dev_info->rx_queue[rx_queue_id].wt :
484 *nb_rx_poll = rx_adapter->num_rx_polled - poll_diff;
485 *nb_wrr = rx_adapter->wrr_len - wrr_len_diff;
488 /* Calculate nb_rx_* after adding poll mode rx queues
491 rxa_calc_nb_post_add_poll(struct rte_event_eth_rx_adapter *rx_adapter,
492 struct eth_device_info *dev_info,
495 uint32_t *nb_rx_poll,
496 uint32_t *nb_rx_intr,
501 uint32_t wrr_len_diff;
503 if (rx_queue_id == -1) {
504 intr_diff = dev_info->nb_rx_intr;
505 poll_diff = dev_info->dev->data->nb_rx_queues -
506 dev_info->nb_rx_poll;
507 wrr_len_diff = wt*dev_info->dev->data->nb_rx_queues
510 intr_diff = rxa_intr_queue(dev_info, rx_queue_id);
511 poll_diff = !rxa_polled_queue(dev_info, rx_queue_id);
512 wrr_len_diff = rxa_polled_queue(dev_info, rx_queue_id) ?
513 wt - dev_info->rx_queue[rx_queue_id].wt :
517 *nb_rx_poll = rx_adapter->num_rx_polled + poll_diff;
518 *nb_rx_intr = rx_adapter->num_rx_intr - intr_diff;
519 *nb_wrr = rx_adapter->wrr_len + wrr_len_diff;
522 /* Calculate nb_rx_* after adding rx_queue_id */
524 rxa_calc_nb_post_add(struct rte_event_eth_rx_adapter *rx_adapter,
525 struct eth_device_info *dev_info,
528 uint32_t *nb_rx_poll,
529 uint32_t *nb_rx_intr,
533 rxa_calc_nb_post_add_poll(rx_adapter, dev_info, rx_queue_id,
534 wt, nb_rx_poll, nb_rx_intr, nb_wrr);
536 rxa_calc_nb_post_add_intr(rx_adapter, dev_info, rx_queue_id,
537 nb_rx_poll, nb_rx_intr, nb_wrr);
540 /* Calculate nb_rx_* after deleting rx_queue_id */
542 rxa_calc_nb_post_del(struct rte_event_eth_rx_adapter *rx_adapter,
543 struct eth_device_info *dev_info,
545 uint32_t *nb_rx_poll,
546 uint32_t *nb_rx_intr,
549 rxa_calc_nb_post_poll_del(rx_adapter, dev_info, rx_queue_id, nb_rx_poll,
551 rxa_calc_nb_post_intr_del(rx_adapter, dev_info, rx_queue_id,
556 * Allocate the rx_poll array
558 static struct eth_rx_poll_entry *
559 rxa_alloc_poll(struct rte_event_eth_rx_adapter *rx_adapter,
560 uint32_t num_rx_polled)
564 len = RTE_ALIGN(num_rx_polled * sizeof(*rx_adapter->eth_rx_poll),
565 RTE_CACHE_LINE_SIZE);
566 return rte_zmalloc_socket(rx_adapter->mem_name,
569 rx_adapter->socket_id);
573 * Allocate the WRR array
576 rxa_alloc_wrr(struct rte_event_eth_rx_adapter *rx_adapter, int nb_wrr)
580 len = RTE_ALIGN(nb_wrr * sizeof(*rx_adapter->wrr_sched),
581 RTE_CACHE_LINE_SIZE);
582 return rte_zmalloc_socket(rx_adapter->mem_name,
585 rx_adapter->socket_id);
589 rxa_alloc_poll_arrays(struct rte_event_eth_rx_adapter *rx_adapter,
592 struct eth_rx_poll_entry **rx_poll,
593 uint32_t **wrr_sched)
602 *rx_poll = rxa_alloc_poll(rx_adapter, nb_poll);
603 if (*rx_poll == NULL) {
608 *wrr_sched = rxa_alloc_wrr(rx_adapter, nb_wrr);
609 if (*wrr_sched == NULL) {
616 /* Precalculate WRR polling sequence for all queues in rx_adapter */
618 rxa_calc_wrr_sequence(struct rte_event_eth_rx_adapter *rx_adapter,
619 struct eth_rx_poll_entry *rx_poll,
628 /* Initialize variables for calculation of wrr schedule */
629 uint16_t max_wrr_pos = 0;
630 unsigned int poll_q = 0;
637 /* Generate array of all queues to poll, the size of this
640 RTE_ETH_FOREACH_DEV(d) {
641 uint16_t nb_rx_queues;
642 struct eth_device_info *dev_info =
643 &rx_adapter->eth_devices[d];
644 nb_rx_queues = dev_info->dev->data->nb_rx_queues;
645 if (dev_info->rx_queue == NULL)
647 if (dev_info->internal_event_port)
649 dev_info->wrr_len = 0;
650 for (q = 0; q < nb_rx_queues; q++) {
651 struct eth_rx_queue_info *queue_info =
652 &dev_info->rx_queue[q];
655 if (!rxa_polled_queue(dev_info, q))
658 rx_poll[poll_q].eth_dev_id = d;
659 rx_poll[poll_q].eth_rx_qid = q;
661 dev_info->wrr_len += wt;
662 max_wt = RTE_MAX(max_wt, wt);
663 gcd = (gcd) ? rxa_gcd_u16(gcd, wt) : wt;
668 /* Generate polling sequence based on weights */
671 for (i = 0; i < max_wrr_pos; i++) {
672 rx_wrr[i] = rxa_wrr_next(rx_adapter, poll_q, &cw,
673 rx_poll, max_wt, gcd, prev);
679 rxa_mtoip(struct rte_mbuf *m, struct rte_ipv4_hdr **ipv4_hdr,
680 struct rte_ipv6_hdr **ipv6_hdr)
682 struct rte_ether_hdr *eth_hdr =
683 rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
684 struct rte_vlan_hdr *vlan_hdr;
689 switch (eth_hdr->ether_type) {
690 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
691 *ipv4_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
694 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
695 *ipv6_hdr = (struct rte_ipv6_hdr *)(eth_hdr + 1);
698 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
699 vlan_hdr = (struct rte_vlan_hdr *)(eth_hdr + 1);
700 switch (vlan_hdr->eth_proto) {
701 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
702 *ipv4_hdr = (struct rte_ipv4_hdr *)(vlan_hdr + 1);
704 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
705 *ipv6_hdr = (struct rte_ipv6_hdr *)(vlan_hdr + 1);
717 /* Calculate RSS hash for IPv4/6 */
718 static inline uint32_t
719 rxa_do_softrss(struct rte_mbuf *m, const uint8_t *rss_key_be)
723 struct rte_ipv4_tuple ipv4_tuple;
724 struct rte_ipv6_tuple ipv6_tuple;
725 struct rte_ipv4_hdr *ipv4_hdr;
726 struct rte_ipv6_hdr *ipv6_hdr;
728 rxa_mtoip(m, &ipv4_hdr, &ipv6_hdr);
731 ipv4_tuple.src_addr = rte_be_to_cpu_32(ipv4_hdr->src_addr);
732 ipv4_tuple.dst_addr = rte_be_to_cpu_32(ipv4_hdr->dst_addr);
734 input_len = RTE_THASH_V4_L3_LEN;
735 } else if (ipv6_hdr) {
736 rte_thash_load_v6_addrs(ipv6_hdr,
737 (union rte_thash_tuple *)&ipv6_tuple);
739 input_len = RTE_THASH_V6_L3_LEN;
743 return rte_softrss_be(tuple, input_len, rss_key_be);
747 rxa_enq_blocked(struct rte_event_eth_rx_adapter *rx_adapter)
749 return !!rx_adapter->enq_block_count;
753 rxa_enq_block_start_ts(struct rte_event_eth_rx_adapter *rx_adapter)
755 if (rx_adapter->rx_enq_block_start_ts)
758 rx_adapter->enq_block_count++;
759 if (rx_adapter->enq_block_count < BLOCK_CNT_THRESHOLD)
762 rx_adapter->rx_enq_block_start_ts = rte_get_tsc_cycles();
766 rxa_enq_block_end_ts(struct rte_event_eth_rx_adapter *rx_adapter,
767 struct rte_event_eth_rx_adapter_stats *stats)
769 if (unlikely(!stats->rx_enq_start_ts))
770 stats->rx_enq_start_ts = rte_get_tsc_cycles();
772 if (likely(!rxa_enq_blocked(rx_adapter)))
775 rx_adapter->enq_block_count = 0;
776 if (rx_adapter->rx_enq_block_start_ts) {
777 stats->rx_enq_end_ts = rte_get_tsc_cycles();
778 stats->rx_enq_block_cycles += stats->rx_enq_end_ts -
779 rx_adapter->rx_enq_block_start_ts;
780 rx_adapter->rx_enq_block_start_ts = 0;
784 /* Enqueue buffered events to event device */
785 static inline uint16_t
786 rxa_flush_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter,
787 struct rte_eth_event_enqueue_buffer *buf)
789 struct rte_event_eth_rx_adapter_stats *stats = &rx_adapter->stats;
790 uint16_t count = buf->last ? buf->last - buf->head : buf->count;
795 uint16_t n = rte_event_enqueue_new_burst(rx_adapter->eventdev_id,
796 rx_adapter->event_port_id,
797 &buf->events[buf->head],
800 stats->rx_enq_retry++;
804 if (buf->last && n == count) {
807 n1 = rte_event_enqueue_new_burst(rx_adapter->eventdev_id,
808 rx_adapter->event_port_id,
813 stats->rx_enq_retry++;
821 n ? rxa_enq_block_end_ts(rx_adapter, stats) :
822 rxa_enq_block_start_ts(rx_adapter);
825 stats->rx_enq_count += n;
831 rxa_init_vector(struct rte_event_eth_rx_adapter *rx_adapter,
832 struct eth_rx_vector_data *vec)
834 vec->vector_ev->nb_elem = 0;
835 vec->vector_ev->port = vec->port;
836 vec->vector_ev->queue = vec->queue;
837 vec->vector_ev->attr_valid = true;
838 TAILQ_INSERT_TAIL(&rx_adapter->vector_list, vec, next);
841 static inline uint16_t
842 rxa_create_event_vector(struct rte_event_eth_rx_adapter *rx_adapter,
843 struct eth_rx_queue_info *queue_info,
844 struct rte_eth_event_enqueue_buffer *buf,
845 struct rte_mbuf **mbufs, uint16_t num)
847 struct rte_event *ev = &buf->events[buf->count];
848 struct eth_rx_vector_data *vec;
849 uint16_t filled, space, sz;
852 vec = &queue_info->vector_data;
854 if (vec->vector_ev == NULL) {
855 if (rte_mempool_get(vec->vector_pool,
856 (void **)&vec->vector_ev) < 0) {
857 rte_pktmbuf_free_bulk(mbufs, num);
860 rxa_init_vector(rx_adapter, vec);
863 if (vec->vector_ev->nb_elem == vec->max_vector_count) {
865 ev->event = vec->event;
866 ev->vec = vec->vector_ev;
869 vec->vector_ev = NULL;
870 TAILQ_REMOVE(&rx_adapter->vector_list, vec, next);
871 if (rte_mempool_get(vec->vector_pool,
872 (void **)&vec->vector_ev) < 0) {
873 rte_pktmbuf_free_bulk(mbufs, num);
876 rxa_init_vector(rx_adapter, vec);
879 space = vec->max_vector_count - vec->vector_ev->nb_elem;
880 sz = num > space ? space : num;
881 memcpy(vec->vector_ev->mbufs + vec->vector_ev->nb_elem, mbufs,
882 sizeof(void *) * sz);
883 vec->vector_ev->nb_elem += sz;
886 vec->ts = rte_rdtsc();
889 if (vec->vector_ev->nb_elem == vec->max_vector_count) {
890 ev->event = vec->event;
891 ev->vec = vec->vector_ev;
894 vec->vector_ev = NULL;
895 TAILQ_REMOVE(&rx_adapter->vector_list, vec, next);
902 rxa_buffer_mbufs(struct rte_event_eth_rx_adapter *rx_adapter,
904 uint16_t rx_queue_id,
905 struct rte_mbuf **mbufs,
907 struct rte_eth_event_enqueue_buffer *buf)
910 struct eth_device_info *dev_info =
911 &rx_adapter->eth_devices[eth_dev_id];
912 struct eth_rx_queue_info *eth_rx_queue_info =
913 &dev_info->rx_queue[rx_queue_id];
914 uint16_t new_tail = buf->tail;
915 uint64_t event = eth_rx_queue_info->event;
916 uint32_t flow_id_mask = eth_rx_queue_info->flow_id_mask;
917 struct rte_mbuf *m = mbufs[0];
923 uint64_t ts, ts_mask;
925 if (!eth_rx_queue_info->ena_vector) {
926 ts = m->ol_flags & event_eth_rx_timestamp_dynflag ?
927 0 : rte_get_tsc_cycles();
929 /* 0xffff ffff ffff ffff if PKT_RX_TIMESTAMP is set,
932 ts_mask = (uint64_t)(!(m->ol_flags &
933 event_eth_rx_timestamp_dynflag)) - 1ULL;
935 /* 0xffff ffff if PKT_RX_RSS_HASH is set, otherwise 0 */
936 rss_mask = ~(((m->ol_flags & PKT_RX_RSS_HASH) != 0) - 1);
937 do_rss = !rss_mask && !eth_rx_queue_info->flow_id_mask;
938 for (i = 0; i < num; i++) {
939 struct rte_event *ev;
942 *rxa_timestamp_dynfield(m) = ts |
943 (*rxa_timestamp_dynfield(m) & ts_mask);
945 ev = &buf->events[new_tail];
947 rss = do_rss ? rxa_do_softrss(m, rx_adapter->rss_key_be)
950 ev->flow_id = (rss & ~flow_id_mask) |
951 (ev->flow_id & flow_id_mask);
956 num = rxa_create_event_vector(rx_adapter, eth_rx_queue_info,
960 if (num && dev_info->cb_fn) {
963 nb_cb = dev_info->cb_fn(eth_dev_id, rx_queue_id,
965 (buf->events_size & ~buf->last_mask),
966 buf->count >= BATCH_SIZE ?
967 buf->count - BATCH_SIZE : 0,
968 &buf->events[buf->tail],
972 if (unlikely(nb_cb > num))
973 RTE_EDEV_LOG_ERR("Rx CB returned %d (> %d) events",
978 rx_adapter->stats.rx_dropped += dropped;
986 rxa_pkt_buf_available(struct rte_eth_event_enqueue_buffer *buf)
988 uint32_t nb_req = buf->tail + BATCH_SIZE;
991 if (nb_req <= buf->events_size)
994 if (buf->head >= BATCH_SIZE) {
996 buf->last = buf->tail;
1002 return nb_req <= buf->head;
1005 /* Enqueue packets from <port, q> to event buffer */
1006 static inline uint32_t
1007 rxa_eth_rx(struct rte_event_eth_rx_adapter *rx_adapter,
1013 struct rte_eth_event_enqueue_buffer *buf)
1015 struct rte_mbuf *mbufs[BATCH_SIZE];
1016 struct rte_event_eth_rx_adapter_stats *stats =
1023 /* Don't do a batch dequeue from the rx queue if there isn't
1024 * enough space in the enqueue buffer.
1026 while (rxa_pkt_buf_available(buf)) {
1027 if (buf->count >= BATCH_SIZE)
1028 rxa_flush_event_buffer(rx_adapter, buf);
1030 stats->rx_poll_count++;
1031 n = rte_eth_rx_burst(port_id, queue_id, mbufs, BATCH_SIZE);
1037 rxa_buffer_mbufs(rx_adapter, port_id, queue_id, mbufs, n, buf);
1039 if (rx_count + nb_rx > max_rx)
1044 rxa_flush_event_buffer(rx_adapter, buf);
1050 rxa_intr_ring_enqueue(struct rte_event_eth_rx_adapter *rx_adapter,
1056 union queue_data qd;
1057 struct eth_device_info *dev_info;
1058 struct eth_rx_queue_info *queue_info;
1065 dev_info = &rx_adapter->eth_devices[port_id];
1066 queue_info = &dev_info->rx_queue[queue];
1067 rte_spinlock_lock(&rx_adapter->intr_ring_lock);
1068 if (rxa_shared_intr(dev_info, queue))
1069 intr_enabled = &dev_info->shared_intr_enabled;
1071 intr_enabled = &queue_info->intr_enabled;
1073 if (*intr_enabled) {
1075 err = rte_ring_enqueue(rx_adapter->intr_ring, data);
1076 /* Entry should always be available.
1077 * The ring size equals the maximum number of interrupt
1078 * vectors supported (an interrupt vector is shared in
1079 * case of shared interrupts)
1082 RTE_EDEV_LOG_ERR("Failed to enqueue interrupt"
1083 " to ring: %s", strerror(-err));
1085 rte_eth_dev_rx_intr_disable(port_id, queue);
1087 rte_spinlock_unlock(&rx_adapter->intr_ring_lock);
1091 rxa_intr_ring_check_avail(struct rte_event_eth_rx_adapter *rx_adapter,
1092 uint32_t num_intr_vec)
1094 if (rx_adapter->num_intr_vec + num_intr_vec >
1095 RTE_EVENT_ETH_INTR_RING_SIZE) {
1096 RTE_EDEV_LOG_ERR("Exceeded intr ring slots current"
1097 " %d needed %d limit %d", rx_adapter->num_intr_vec,
1098 num_intr_vec, RTE_EVENT_ETH_INTR_RING_SIZE);
1105 /* Delete entries for (dev, queue) from the interrupt ring */
1107 rxa_intr_ring_del_entries(struct rte_event_eth_rx_adapter *rx_adapter,
1108 struct eth_device_info *dev_info,
1109 uint16_t rx_queue_id)
1112 union queue_data qd;
1114 rte_spinlock_lock(&rx_adapter->intr_ring_lock);
1116 n = rte_ring_count(rx_adapter->intr_ring);
1117 for (i = 0; i < n; i++) {
1118 rte_ring_dequeue(rx_adapter->intr_ring, &qd.ptr);
1119 if (!rxa_shared_intr(dev_info, rx_queue_id)) {
1120 if (qd.port == dev_info->dev->data->port_id &&
1121 qd.queue == rx_queue_id)
1124 if (qd.port == dev_info->dev->data->port_id)
1127 rte_ring_enqueue(rx_adapter->intr_ring, qd.ptr);
1130 rte_spinlock_unlock(&rx_adapter->intr_ring_lock);
1133 /* pthread callback handling interrupt mode receive queues
1134 * After receiving an Rx interrupt, it enqueues the port id and queue id of the
1135 * interrupting queue to the adapter's ring buffer for interrupt events.
1136 * These events are picked up by rxa_intr_ring_dequeue() which is invoked from
1137 * the adapter service function.
1140 rxa_intr_thread(void *arg)
1142 struct rte_event_eth_rx_adapter *rx_adapter = arg;
1143 struct rte_epoll_event *epoll_events = rx_adapter->epoll_events;
1147 n = rte_epoll_wait(rx_adapter->epd, epoll_events,
1148 RTE_EVENT_ETH_INTR_RING_SIZE, -1);
1149 if (unlikely(n < 0))
1150 RTE_EDEV_LOG_ERR("rte_epoll_wait returned error %d",
1152 for (i = 0; i < n; i++) {
1153 rxa_intr_ring_enqueue(rx_adapter,
1154 epoll_events[i].epdata.data);
1161 /* Dequeue <port, q> from interrupt ring and enqueue received
1164 static inline uint32_t
1165 rxa_intr_ring_dequeue(struct rte_event_eth_rx_adapter *rx_adapter)
1170 struct rte_eth_event_enqueue_buffer *buf;
1171 rte_spinlock_t *ring_lock;
1172 uint8_t max_done = 0;
1174 if (rx_adapter->num_rx_intr == 0)
1177 if (rte_ring_count(rx_adapter->intr_ring) == 0
1178 && !rx_adapter->qd_valid)
1181 buf = &rx_adapter->event_enqueue_buffer;
1182 ring_lock = &rx_adapter->intr_ring_lock;
1184 if (buf->count >= BATCH_SIZE)
1185 rxa_flush_event_buffer(rx_adapter, buf);
1187 while (rxa_pkt_buf_available(buf)) {
1188 struct eth_device_info *dev_info;
1191 union queue_data qd = rx_adapter->qd;
1194 if (!rx_adapter->qd_valid) {
1195 struct eth_rx_queue_info *queue_info;
1197 rte_spinlock_lock(ring_lock);
1198 err = rte_ring_dequeue(rx_adapter->intr_ring, &qd.ptr);
1200 rte_spinlock_unlock(ring_lock);
1206 rx_adapter->qd = qd;
1207 rx_adapter->qd_valid = 1;
1208 dev_info = &rx_adapter->eth_devices[port];
1209 if (rxa_shared_intr(dev_info, queue))
1210 dev_info->shared_intr_enabled = 1;
1212 queue_info = &dev_info->rx_queue[queue];
1213 queue_info->intr_enabled = 1;
1215 rte_eth_dev_rx_intr_enable(port, queue);
1216 rte_spinlock_unlock(ring_lock);
1221 dev_info = &rx_adapter->eth_devices[port];
1224 if (rxa_shared_intr(dev_info, queue)) {
1228 nb_queues = dev_info->dev->data->nb_rx_queues;
1230 for (i = dev_info->next_q_idx; i < nb_queues; i++) {
1231 uint8_t enq_buffer_full;
1233 if (!rxa_intr_queue(dev_info, i))
1235 n = rxa_eth_rx(rx_adapter, port, i, nb_rx,
1236 rx_adapter->max_nb_rx,
1240 enq_buffer_full = !rxq_empty && n == 0;
1241 max_done = nb_rx > rx_adapter->max_nb_rx;
1243 if (enq_buffer_full || max_done) {
1244 dev_info->next_q_idx = i;
1249 rx_adapter->qd_valid = 0;
1251 /* Reinitialize for next interrupt */
1252 dev_info->next_q_idx = dev_info->multi_intr_cap ?
1253 RTE_MAX_RXTX_INTR_VEC_ID - 1 :
1256 n = rxa_eth_rx(rx_adapter, port, queue, nb_rx,
1257 rx_adapter->max_nb_rx,
1259 rx_adapter->qd_valid = !rxq_empty;
1261 if (nb_rx > rx_adapter->max_nb_rx)
1267 rx_adapter->stats.rx_intr_packets += nb_rx;
1272 * Polls receive queues added to the event adapter and enqueues received
1273 * packets to the event device.
1275 * The receive code enqueues initially to a temporary buffer, the
1276 * temporary buffer is drained anytime it holds >= BATCH_SIZE packets
1278 * If there isn't space available in the temporary buffer, packets from the
1279 * Rx queue aren't dequeued from the eth device, this back pressures the
1280 * eth device, in virtual device environments this back pressure is relayed to
1281 * the hypervisor's switching layer where adjustments can be made to deal with
1284 static inline uint32_t
1285 rxa_poll(struct rte_event_eth_rx_adapter *rx_adapter)
1289 struct rte_eth_event_enqueue_buffer *buf = NULL;
1293 wrr_pos = rx_adapter->wrr_pos;
1294 max_nb_rx = rx_adapter->max_nb_rx;
1296 /* Iterate through a WRR sequence */
1297 for (num_queue = 0; num_queue < rx_adapter->wrr_len; num_queue++) {
1298 unsigned int poll_idx = rx_adapter->wrr_sched[wrr_pos];
1299 uint16_t qid = rx_adapter->eth_rx_poll[poll_idx].eth_rx_qid;
1300 uint16_t d = rx_adapter->eth_rx_poll[poll_idx].eth_dev_id;
1302 buf = rxa_event_buf_get(rx_adapter, d, qid);
1304 /* Don't do a batch dequeue from the rx queue if there isn't
1305 * enough space in the enqueue buffer.
1307 if (buf->count >= BATCH_SIZE)
1308 rxa_flush_event_buffer(rx_adapter, buf);
1309 if (!rxa_pkt_buf_available(buf)) {
1310 if (rx_adapter->use_queue_event_buf)
1311 goto poll_next_entry;
1313 rx_adapter->wrr_pos = wrr_pos;
1318 nb_rx += rxa_eth_rx(rx_adapter, d, qid, nb_rx, max_nb_rx,
1320 if (nb_rx > max_nb_rx) {
1321 rx_adapter->wrr_pos =
1322 (wrr_pos + 1) % rx_adapter->wrr_len;
1327 if (++wrr_pos == rx_adapter->wrr_len)
1334 rxa_vector_expire(struct eth_rx_vector_data *vec, void *arg)
1336 struct rte_event_eth_rx_adapter *rx_adapter = arg;
1337 struct rte_eth_event_enqueue_buffer *buf = NULL;
1338 struct rte_event *ev;
1340 buf = rxa_event_buf_get(rx_adapter, vec->port, vec->queue);
1343 rxa_flush_event_buffer(rx_adapter, buf);
1345 if (vec->vector_ev->nb_elem == 0)
1347 ev = &buf->events[buf->count];
1350 ev->event = vec->event;
1351 ev->vec = vec->vector_ev;
1354 vec->vector_ev = NULL;
1359 rxa_service_func(void *args)
1361 struct rte_event_eth_rx_adapter *rx_adapter = args;
1362 struct rte_event_eth_rx_adapter_stats *stats;
1364 if (rte_spinlock_trylock(&rx_adapter->rx_lock) == 0)
1366 if (!rx_adapter->rxa_started) {
1367 rte_spinlock_unlock(&rx_adapter->rx_lock);
1371 if (rx_adapter->ena_vector) {
1372 if ((rte_rdtsc() - rx_adapter->prev_expiry_ts) >=
1373 rx_adapter->vector_tmo_ticks) {
1374 struct eth_rx_vector_data *vec;
1376 TAILQ_FOREACH(vec, &rx_adapter->vector_list, next) {
1377 uint64_t elapsed_time = rte_rdtsc() - vec->ts;
1379 if (elapsed_time >= vec->vector_timeout_ticks) {
1380 rxa_vector_expire(vec, rx_adapter);
1381 TAILQ_REMOVE(&rx_adapter->vector_list,
1385 rx_adapter->prev_expiry_ts = rte_rdtsc();
1389 stats = &rx_adapter->stats;
1390 stats->rx_packets += rxa_intr_ring_dequeue(rx_adapter);
1391 stats->rx_packets += rxa_poll(rx_adapter);
1392 rte_spinlock_unlock(&rx_adapter->rx_lock);
1397 rte_event_eth_rx_adapter_init(void)
1399 const char *name = RXA_ADAPTER_ARRAY;
1400 const struct rte_memzone *mz;
1403 sz = sizeof(*event_eth_rx_adapter) *
1404 RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE;
1405 sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
1407 mz = rte_memzone_lookup(name);
1409 mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0,
1410 RTE_CACHE_LINE_SIZE);
1412 RTE_EDEV_LOG_ERR("failed to reserve memzone err = %"
1418 event_eth_rx_adapter = mz->addr;
1423 rxa_memzone_lookup(void)
1425 const struct rte_memzone *mz;
1427 if (event_eth_rx_adapter == NULL) {
1428 mz = rte_memzone_lookup(RXA_ADAPTER_ARRAY);
1431 event_eth_rx_adapter = mz->addr;
1437 static inline struct rte_event_eth_rx_adapter *
1438 rxa_id_to_adapter(uint8_t id)
1440 return event_eth_rx_adapter ?
1441 event_eth_rx_adapter[id] : NULL;
1445 rxa_default_conf_cb(uint8_t id, uint8_t dev_id,
1446 struct rte_event_eth_rx_adapter_conf *conf, void *arg)
1449 struct rte_eventdev *dev;
1450 struct rte_event_dev_config dev_conf;
1453 struct rte_event_port_conf *port_conf = arg;
1454 struct rte_event_eth_rx_adapter *rx_adapter = rxa_id_to_adapter(id);
1456 dev = &rte_eventdevs[rx_adapter->eventdev_id];
1457 dev_conf = dev->data->dev_conf;
1459 started = dev->data->dev_started;
1461 rte_event_dev_stop(dev_id);
1462 port_id = dev_conf.nb_event_ports;
1463 dev_conf.nb_event_ports += 1;
1464 ret = rte_event_dev_configure(dev_id, &dev_conf);
1466 RTE_EDEV_LOG_ERR("failed to configure event dev %u\n",
1469 if (rte_event_dev_start(dev_id))
1475 ret = rte_event_port_setup(dev_id, port_id, port_conf);
1477 RTE_EDEV_LOG_ERR("failed to setup event port %u\n",
1482 conf->event_port_id = port_id;
1483 conf->max_nb_rx = 128;
1485 ret = rte_event_dev_start(dev_id);
1486 rx_adapter->default_cb_arg = 1;
1491 rxa_epoll_create1(void)
1495 fd = epoll_create1(EPOLL_CLOEXEC);
1496 return fd < 0 ? -errno : fd;
1503 rxa_init_epd(struct rte_event_eth_rx_adapter *rx_adapter)
1505 if (rx_adapter->epd != INIT_FD)
1508 rx_adapter->epd = rxa_epoll_create1();
1509 if (rx_adapter->epd < 0) {
1510 int err = rx_adapter->epd;
1511 rx_adapter->epd = INIT_FD;
1512 RTE_EDEV_LOG_ERR("epoll_create1() failed, err %d", err);
1520 rxa_create_intr_thread(struct rte_event_eth_rx_adapter *rx_adapter)
1523 char thread_name[RTE_MAX_THREAD_NAME_LEN];
1525 if (rx_adapter->intr_ring)
1528 rx_adapter->intr_ring = rte_ring_create("intr_ring",
1529 RTE_EVENT_ETH_INTR_RING_SIZE,
1530 rte_socket_id(), 0);
1531 if (!rx_adapter->intr_ring)
1534 rx_adapter->epoll_events = rte_zmalloc_socket(rx_adapter->mem_name,
1535 RTE_EVENT_ETH_INTR_RING_SIZE *
1536 sizeof(struct rte_epoll_event),
1537 RTE_CACHE_LINE_SIZE,
1538 rx_adapter->socket_id);
1539 if (!rx_adapter->epoll_events) {
1544 rte_spinlock_init(&rx_adapter->intr_ring_lock);
1546 snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN,
1547 "rx-intr-thread-%d", rx_adapter->id);
1549 err = rte_ctrl_thread_create(&rx_adapter->rx_intr_thread, thread_name,
1550 NULL, rxa_intr_thread, rx_adapter);
1554 RTE_EDEV_LOG_ERR("Failed to create interrupt thread err = %d\n", err);
1555 rte_free(rx_adapter->epoll_events);
1557 rte_ring_free(rx_adapter->intr_ring);
1558 rx_adapter->intr_ring = NULL;
1559 rx_adapter->epoll_events = NULL;
1564 rxa_destroy_intr_thread(struct rte_event_eth_rx_adapter *rx_adapter)
1568 err = pthread_cancel(rx_adapter->rx_intr_thread);
1570 RTE_EDEV_LOG_ERR("Can't cancel interrupt thread err = %d\n",
1573 err = pthread_join(rx_adapter->rx_intr_thread, NULL);
1575 RTE_EDEV_LOG_ERR("Can't join interrupt thread err = %d\n", err);
1577 rte_free(rx_adapter->epoll_events);
1578 rte_ring_free(rx_adapter->intr_ring);
1579 rx_adapter->intr_ring = NULL;
1580 rx_adapter->epoll_events = NULL;
1585 rxa_free_intr_resources(struct rte_event_eth_rx_adapter *rx_adapter)
1589 if (rx_adapter->num_rx_intr == 0)
1592 ret = rxa_destroy_intr_thread(rx_adapter);
1596 close(rx_adapter->epd);
1597 rx_adapter->epd = INIT_FD;
1603 rxa_disable_intr(struct rte_event_eth_rx_adapter *rx_adapter,
1604 struct eth_device_info *dev_info,
1605 uint16_t rx_queue_id)
1608 uint16_t eth_dev_id = dev_info->dev->data->port_id;
1609 int sintr = rxa_shared_intr(dev_info, rx_queue_id);
1611 err = rte_eth_dev_rx_intr_disable(eth_dev_id, rx_queue_id);
1613 RTE_EDEV_LOG_ERR("Could not disable interrupt for Rx queue %u",
1618 err = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
1623 RTE_EDEV_LOG_ERR("Interrupt event deletion failed %d", err);
1626 dev_info->rx_queue[rx_queue_id].intr_enabled = 0;
1628 dev_info->shared_intr_enabled = 0;
1633 rxa_del_intr_queue(struct rte_event_eth_rx_adapter *rx_adapter,
1634 struct eth_device_info *dev_info,
1641 if (dev_info->nb_rx_intr == 0)
1645 if (rx_queue_id == -1) {
1646 s = dev_info->nb_shared_intr;
1647 for (i = 0; i < dev_info->nb_rx_intr; i++) {
1651 q = dev_info->intr_queue[i];
1652 sintr = rxa_shared_intr(dev_info, q);
1655 if (!sintr || s == 0) {
1657 err = rxa_disable_intr(rx_adapter, dev_info,
1661 rxa_intr_ring_del_entries(rx_adapter, dev_info,
1666 if (!rxa_intr_queue(dev_info, rx_queue_id))
1668 if (!rxa_shared_intr(dev_info, rx_queue_id) ||
1669 dev_info->nb_shared_intr == 1) {
1670 err = rxa_disable_intr(rx_adapter, dev_info,
1674 rxa_intr_ring_del_entries(rx_adapter, dev_info,
1678 for (i = 0; i < dev_info->nb_rx_intr; i++) {
1679 if (dev_info->intr_queue[i] == rx_queue_id) {
1680 for (; i < dev_info->nb_rx_intr - 1; i++)
1681 dev_info->intr_queue[i] =
1682 dev_info->intr_queue[i + 1];
1692 rxa_config_intr(struct rte_event_eth_rx_adapter *rx_adapter,
1693 struct eth_device_info *dev_info,
1694 uint16_t rx_queue_id)
1697 uint16_t eth_dev_id = dev_info->dev->data->port_id;
1698 union queue_data qd;
1700 uint16_t *intr_queue;
1701 int sintr = rxa_shared_intr(dev_info, rx_queue_id);
1703 if (rxa_intr_queue(dev_info, rx_queue_id))
1706 intr_queue = dev_info->intr_queue;
1707 if (dev_info->intr_queue == NULL) {
1709 dev_info->dev->data->nb_rx_queues * sizeof(uint16_t);
1710 dev_info->intr_queue =
1712 rx_adapter->mem_name,
1715 rx_adapter->socket_id);
1716 if (dev_info->intr_queue == NULL)
1720 init_fd = rx_adapter->epd;
1721 err = rxa_init_epd(rx_adapter);
1723 goto err_free_queue;
1725 qd.port = eth_dev_id;
1726 qd.queue = rx_queue_id;
1728 err = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
1733 RTE_EDEV_LOG_ERR("Failed to add interrupt event for"
1734 " Rx Queue %u err %d", rx_queue_id, err);
1738 err = rte_eth_dev_rx_intr_enable(eth_dev_id, rx_queue_id);
1740 RTE_EDEV_LOG_ERR("Could not enable interrupt for"
1741 " Rx Queue %u err %d", rx_queue_id, err);
1746 err = rxa_create_intr_thread(rx_adapter);
1749 dev_info->shared_intr_enabled = 1;
1751 dev_info->rx_queue[rx_queue_id].intr_enabled = 1;
1756 err = rte_eth_dev_rx_intr_disable(eth_dev_id, rx_queue_id);
1758 RTE_EDEV_LOG_ERR("Could not disable interrupt for"
1759 " Rx Queue %u err %d", rx_queue_id, err);
1761 err1 = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
1766 RTE_EDEV_LOG_ERR("Could not delete event for"
1767 " Rx Queue %u err %d", rx_queue_id, err1);
1770 if (init_fd == INIT_FD) {
1771 close(rx_adapter->epd);
1772 rx_adapter->epd = -1;
1775 if (intr_queue == NULL)
1776 rte_free(dev_info->intr_queue);
1782 rxa_add_intr_queue(struct rte_event_eth_rx_adapter *rx_adapter,
1783 struct eth_device_info *dev_info,
1789 int shared_done = (dev_info->nb_shared_intr > 0);
1791 if (rx_queue_id != -1) {
1792 if (rxa_shared_intr(dev_info, rx_queue_id) && shared_done)
1794 return rxa_config_intr(rx_adapter, dev_info, rx_queue_id);
1798 for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++) {
1800 if (rxa_shared_intr(dev_info, i) && shared_done)
1803 err = rxa_config_intr(rx_adapter, dev_info, i);
1805 shared_done = err == 0 && rxa_shared_intr(dev_info, i);
1808 dev_info->shared_intr_enabled = 1;
1817 shared_done = (dev_info->nb_shared_intr > 0);
1818 for (j = 0; j < i; j++) {
1819 if (rxa_intr_queue(dev_info, j))
1821 if (rxa_shared_intr(dev_info, j) && si != j)
1823 err = rxa_disable_intr(rx_adapter, dev_info, j);
1834 rxa_init_service(struct rte_event_eth_rx_adapter *rx_adapter, uint8_t id)
1837 struct rte_service_spec service;
1838 struct rte_event_eth_rx_adapter_conf rx_adapter_conf;
1840 if (rx_adapter->service_inited)
1843 memset(&service, 0, sizeof(service));
1844 snprintf(service.name, ETH_RX_ADAPTER_SERVICE_NAME_LEN,
1845 "rte_event_eth_rx_adapter_%d", id);
1846 service.socket_id = rx_adapter->socket_id;
1847 service.callback = rxa_service_func;
1848 service.callback_userdata = rx_adapter;
1849 /* Service function handles locking for queue add/del updates */
1850 service.capabilities = RTE_SERVICE_CAP_MT_SAFE;
1851 ret = rte_service_component_register(&service, &rx_adapter->service_id);
1853 RTE_EDEV_LOG_ERR("failed to register service %s err = %" PRId32,
1858 ret = rx_adapter->conf_cb(id, rx_adapter->eventdev_id,
1859 &rx_adapter_conf, rx_adapter->conf_arg);
1861 RTE_EDEV_LOG_ERR("configuration callback failed err = %" PRId32,
1865 rx_adapter->event_port_id = rx_adapter_conf.event_port_id;
1866 rx_adapter->max_nb_rx = rx_adapter_conf.max_nb_rx;
1867 rx_adapter->service_inited = 1;
1868 rx_adapter->epd = INIT_FD;
1872 rte_service_component_unregister(rx_adapter->service_id);
1877 rxa_update_queue(struct rte_event_eth_rx_adapter *rx_adapter,
1878 struct eth_device_info *dev_info,
1879 int32_t rx_queue_id,
1882 struct eth_rx_queue_info *queue_info;
1886 if (dev_info->rx_queue == NULL)
1889 if (rx_queue_id == -1) {
1890 for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
1891 rxa_update_queue(rx_adapter, dev_info, i, add);
1893 queue_info = &dev_info->rx_queue[rx_queue_id];
1894 enabled = queue_info->queue_enabled;
1896 rx_adapter->nb_queues += !enabled;
1897 dev_info->nb_dev_queues += !enabled;
1899 rx_adapter->nb_queues -= enabled;
1900 dev_info->nb_dev_queues -= enabled;
1902 queue_info->queue_enabled = !!add;
1907 rxa_set_vector_data(struct eth_rx_queue_info *queue_info, uint16_t vector_count,
1908 uint64_t vector_ns, struct rte_mempool *mp, uint32_t qid,
1911 #define NSEC2TICK(__ns, __freq) (((__ns) * (__freq)) / 1E9)
1912 struct eth_rx_vector_data *vector_data;
1915 vector_data = &queue_info->vector_data;
1916 vector_data->max_vector_count = vector_count;
1917 vector_data->port = port_id;
1918 vector_data->queue = qid;
1919 vector_data->vector_pool = mp;
1920 vector_data->vector_timeout_ticks =
1921 NSEC2TICK(vector_ns, rte_get_timer_hz());
1922 vector_data->ts = 0;
1923 flow_id = queue_info->event & 0xFFFFF;
1925 flow_id == 0 ? (qid & 0xFFF) | (port_id & 0xFF) << 12 : flow_id;
1926 vector_data->event = (queue_info->event & ~0xFFFFF) | flow_id;
1930 rxa_sw_del(struct rte_event_eth_rx_adapter *rx_adapter,
1931 struct eth_device_info *dev_info,
1932 int32_t rx_queue_id)
1934 struct eth_rx_vector_data *vec;
1940 if (rx_adapter->nb_queues == 0)
1943 if (rx_queue_id == -1) {
1944 uint16_t nb_rx_queues;
1947 nb_rx_queues = dev_info->dev->data->nb_rx_queues;
1948 for (i = 0; i < nb_rx_queues; i++)
1949 rxa_sw_del(rx_adapter, dev_info, i);
1953 /* Push all the partial event vectors to event device. */
1954 TAILQ_FOREACH(vec, &rx_adapter->vector_list, next) {
1955 if (vec->queue != rx_queue_id)
1957 rxa_vector_expire(vec, rx_adapter);
1958 TAILQ_REMOVE(&rx_adapter->vector_list, vec, next);
1961 pollq = rxa_polled_queue(dev_info, rx_queue_id);
1962 intrq = rxa_intr_queue(dev_info, rx_queue_id);
1963 sintrq = rxa_shared_intr(dev_info, rx_queue_id);
1964 rxa_update_queue(rx_adapter, dev_info, rx_queue_id, 0);
1965 rx_adapter->num_rx_polled -= pollq;
1966 dev_info->nb_rx_poll -= pollq;
1967 rx_adapter->num_rx_intr -= intrq;
1968 dev_info->nb_rx_intr -= intrq;
1969 dev_info->nb_shared_intr -= intrq && sintrq;
1970 if (rx_adapter->use_queue_event_buf) {
1971 struct rte_eth_event_enqueue_buffer *event_buf =
1972 dev_info->rx_queue[rx_queue_id].event_buf;
1973 rte_free(event_buf->events);
1974 rte_free(event_buf);
1975 dev_info->rx_queue[rx_queue_id].event_buf = NULL;
1980 rxa_add_queue(struct rte_event_eth_rx_adapter *rx_adapter,
1981 struct eth_device_info *dev_info,
1982 int32_t rx_queue_id,
1983 const struct rte_event_eth_rx_adapter_queue_conf *conf)
1985 struct eth_rx_queue_info *queue_info;
1986 const struct rte_event *ev = &conf->ev;
1990 struct rte_event *qi_ev;
1991 struct rte_eth_event_enqueue_buffer *new_rx_buf = NULL;
1992 uint16_t eth_dev_id = dev_info->dev->data->port_id;
1995 if (rx_queue_id == -1) {
1996 uint16_t nb_rx_queues;
1999 nb_rx_queues = dev_info->dev->data->nb_rx_queues;
2000 for (i = 0; i < nb_rx_queues; i++) {
2001 ret = rxa_add_queue(rx_adapter, dev_info, i, conf);
2008 pollq = rxa_polled_queue(dev_info, rx_queue_id);
2009 intrq = rxa_intr_queue(dev_info, rx_queue_id);
2010 sintrq = rxa_shared_intr(dev_info, rx_queue_id);
2012 queue_info = &dev_info->rx_queue[rx_queue_id];
2013 queue_info->wt = conf->servicing_weight;
2015 qi_ev = (struct rte_event *)&queue_info->event;
2016 qi_ev->event = ev->event;
2017 qi_ev->op = RTE_EVENT_OP_NEW;
2018 qi_ev->event_type = RTE_EVENT_TYPE_ETH_RX_ADAPTER;
2019 qi_ev->sub_event_type = 0;
2021 if (conf->rx_queue_flags &
2022 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID) {
2023 queue_info->flow_id_mask = ~0;
2027 if (conf->rx_queue_flags &
2028 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR) {
2029 queue_info->ena_vector = 1;
2030 qi_ev->event_type = RTE_EVENT_TYPE_ETH_RX_ADAPTER_VECTOR;
2031 rxa_set_vector_data(queue_info, conf->vector_sz,
2032 conf->vector_timeout_ns, conf->vector_mp,
2033 rx_queue_id, dev_info->dev->data->port_id);
2034 rx_adapter->ena_vector = 1;
2035 rx_adapter->vector_tmo_ticks =
2036 rx_adapter->vector_tmo_ticks ?
2037 RTE_MIN(queue_info->vector_data
2038 .vector_timeout_ticks >>
2040 rx_adapter->vector_tmo_ticks) :
2041 queue_info->vector_data.vector_timeout_ticks >>
2045 rxa_update_queue(rx_adapter, dev_info, rx_queue_id, 1);
2046 if (rxa_polled_queue(dev_info, rx_queue_id)) {
2047 rx_adapter->num_rx_polled += !pollq;
2048 dev_info->nb_rx_poll += !pollq;
2049 rx_adapter->num_rx_intr -= intrq;
2050 dev_info->nb_rx_intr -= intrq;
2051 dev_info->nb_shared_intr -= intrq && sintrq;
2054 if (rxa_intr_queue(dev_info, rx_queue_id)) {
2055 rx_adapter->num_rx_polled -= pollq;
2056 dev_info->nb_rx_poll -= pollq;
2057 rx_adapter->num_rx_intr += !intrq;
2058 dev_info->nb_rx_intr += !intrq;
2059 dev_info->nb_shared_intr += !intrq && sintrq;
2060 if (dev_info->nb_shared_intr == 1) {
2061 if (dev_info->multi_intr_cap)
2062 dev_info->next_q_idx =
2063 RTE_MAX_RXTX_INTR_VEC_ID - 1;
2065 dev_info->next_q_idx = 0;
2069 if (!rx_adapter->use_queue_event_buf)
2072 new_rx_buf = rte_zmalloc_socket("rx_buffer_meta",
2073 sizeof(*new_rx_buf), 0,
2074 rte_eth_dev_socket_id(eth_dev_id));
2075 if (new_rx_buf == NULL) {
2076 RTE_EDEV_LOG_ERR("Failed to allocate event buffer meta for "
2077 "dev_id: %d queue_id: %d",
2078 eth_dev_id, rx_queue_id);
2082 new_rx_buf->events_size = RTE_ALIGN(conf->event_buf_size, BATCH_SIZE);
2083 new_rx_buf->events_size += (2 * BATCH_SIZE);
2084 new_rx_buf->events = rte_zmalloc_socket("rx_buffer",
2085 sizeof(struct rte_event) *
2086 new_rx_buf->events_size, 0,
2087 rte_eth_dev_socket_id(eth_dev_id));
2088 if (new_rx_buf->events == NULL) {
2089 rte_free(new_rx_buf);
2090 RTE_EDEV_LOG_ERR("Failed to allocate event buffer for "
2091 "dev_id: %d queue_id: %d",
2092 eth_dev_id, rx_queue_id);
2096 queue_info->event_buf = new_rx_buf;
2101 static int rxa_sw_add(struct rte_event_eth_rx_adapter *rx_adapter,
2102 uint16_t eth_dev_id,
2104 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
2106 struct eth_device_info *dev_info = &rx_adapter->eth_devices[eth_dev_id];
2107 struct rte_event_eth_rx_adapter_queue_conf temp_conf;
2109 struct eth_rx_poll_entry *rx_poll;
2110 struct eth_rx_queue_info *rx_queue;
2112 uint16_t nb_rx_queues;
2113 uint32_t nb_rx_poll, nb_wrr;
2114 uint32_t nb_rx_intr;
2118 if (queue_conf->servicing_weight == 0) {
2119 struct rte_eth_dev_data *data = dev_info->dev->data;
2121 temp_conf = *queue_conf;
2122 if (!data->dev_conf.intr_conf.rxq) {
2123 /* If Rx interrupts are disabled set wt = 1 */
2124 temp_conf.servicing_weight = 1;
2126 queue_conf = &temp_conf;
2128 if (queue_conf->servicing_weight == 0 &&
2129 rx_adapter->use_queue_event_buf) {
2131 RTE_EDEV_LOG_ERR("Use of queue level event buffer "
2132 "not supported for interrupt queues "
2133 "dev_id: %d queue_id: %d",
2134 eth_dev_id, rx_queue_id);
2139 nb_rx_queues = dev_info->dev->data->nb_rx_queues;
2140 rx_queue = dev_info->rx_queue;
2141 wt = queue_conf->servicing_weight;
2143 if (dev_info->rx_queue == NULL) {
2144 dev_info->rx_queue =
2145 rte_zmalloc_socket(rx_adapter->mem_name,
2147 sizeof(struct eth_rx_queue_info), 0,
2148 rx_adapter->socket_id);
2149 if (dev_info->rx_queue == NULL)
2155 rxa_calc_nb_post_add(rx_adapter, dev_info, rx_queue_id,
2156 queue_conf->servicing_weight,
2157 &nb_rx_poll, &nb_rx_intr, &nb_wrr);
2159 if (dev_info->dev->intr_handle)
2160 dev_info->multi_intr_cap =
2161 rte_intr_cap_multiple(dev_info->dev->intr_handle);
2163 ret = rxa_alloc_poll_arrays(rx_adapter, nb_rx_poll, nb_wrr,
2166 goto err_free_rxqueue;
2169 num_intr_vec = rxa_nb_intr_vect(dev_info, rx_queue_id, 1);
2171 ret = rxa_intr_ring_check_avail(rx_adapter, num_intr_vec);
2173 goto err_free_rxqueue;
2175 ret = rxa_add_intr_queue(rx_adapter, dev_info, rx_queue_id);
2177 goto err_free_rxqueue;
2181 if (rx_adapter->num_rx_intr > nb_rx_intr) {
2182 num_intr_vec = rxa_nb_intr_vect(dev_info,
2184 /* interrupt based queues are being converted to
2185 * poll mode queues, delete the interrupt configuration
2188 ret = rxa_del_intr_queue(rx_adapter,
2189 dev_info, rx_queue_id);
2191 goto err_free_rxqueue;
2195 if (nb_rx_intr == 0) {
2196 ret = rxa_free_intr_resources(rx_adapter);
2198 goto err_free_rxqueue;
2204 if (rx_queue_id == -1) {
2205 for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
2206 dev_info->intr_queue[i] = i;
2208 if (!rxa_intr_queue(dev_info, rx_queue_id))
2209 dev_info->intr_queue[nb_rx_intr - 1] =
2216 ret = rxa_add_queue(rx_adapter, dev_info, rx_queue_id, queue_conf);
2218 goto err_free_rxqueue;
2219 rxa_calc_wrr_sequence(rx_adapter, rx_poll, rx_wrr);
2221 rte_free(rx_adapter->eth_rx_poll);
2222 rte_free(rx_adapter->wrr_sched);
2224 rx_adapter->eth_rx_poll = rx_poll;
2225 rx_adapter->wrr_sched = rx_wrr;
2226 rx_adapter->wrr_len = nb_wrr;
2227 rx_adapter->num_intr_vec += num_intr_vec;
2231 if (rx_queue == NULL) {
2232 rte_free(dev_info->rx_queue);
2233 dev_info->rx_queue = NULL;
2243 rxa_ctrl(uint8_t id, int start)
2245 struct rte_event_eth_rx_adapter *rx_adapter;
2246 struct rte_eventdev *dev;
2247 struct eth_device_info *dev_info;
2249 int use_service = 0;
2252 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2253 rx_adapter = rxa_id_to_adapter(id);
2254 if (rx_adapter == NULL)
2257 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2259 RTE_ETH_FOREACH_DEV(i) {
2260 dev_info = &rx_adapter->eth_devices[i];
2261 /* if start check for num dev queues */
2262 if (start && !dev_info->nb_dev_queues)
2264 /* if stop check if dev has been started */
2265 if (stop && !dev_info->dev_rx_started)
2267 use_service |= !dev_info->internal_event_port;
2268 dev_info->dev_rx_started = start;
2269 if (dev_info->internal_event_port == 0)
2271 start ? (*dev->dev_ops->eth_rx_adapter_start)(dev,
2272 &rte_eth_devices[i]) :
2273 (*dev->dev_ops->eth_rx_adapter_stop)(dev,
2274 &rte_eth_devices[i]);
2278 rte_spinlock_lock(&rx_adapter->rx_lock);
2279 rx_adapter->rxa_started = start;
2280 rte_service_runstate_set(rx_adapter->service_id, start);
2281 rte_spinlock_unlock(&rx_adapter->rx_lock);
2288 rxa_create(uint8_t id, uint8_t dev_id,
2289 struct rte_event_eth_rx_adapter_params *rxa_params,
2290 rte_event_eth_rx_adapter_conf_cb conf_cb,
2293 struct rte_event_eth_rx_adapter *rx_adapter;
2294 struct rte_eth_event_enqueue_buffer *buf;
2295 struct rte_event *events;
2299 char mem_name[ETH_RX_ADAPTER_SERVICE_NAME_LEN];
2300 const uint8_t default_rss_key[] = {
2301 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
2302 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
2303 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
2304 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
2305 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
2308 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2309 RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
2311 if (conf_cb == NULL)
2314 if (event_eth_rx_adapter == NULL) {
2315 ret = rte_event_eth_rx_adapter_init();
2320 rx_adapter = rxa_id_to_adapter(id);
2321 if (rx_adapter != NULL) {
2322 RTE_EDEV_LOG_ERR("Eth Rx adapter exists id = %" PRIu8, id);
2326 socket_id = rte_event_dev_socket_id(dev_id);
2327 snprintf(mem_name, ETH_RX_ADAPTER_MEM_NAME_LEN,
2328 "rte_event_eth_rx_adapter_%d",
2331 rx_adapter = rte_zmalloc_socket(mem_name, sizeof(*rx_adapter),
2332 RTE_CACHE_LINE_SIZE, socket_id);
2333 if (rx_adapter == NULL) {
2334 RTE_EDEV_LOG_ERR("failed to get mem for rx adapter");
2338 rx_adapter->eventdev_id = dev_id;
2339 rx_adapter->socket_id = socket_id;
2340 rx_adapter->conf_cb = conf_cb;
2341 rx_adapter->conf_arg = conf_arg;
2342 rx_adapter->id = id;
2343 TAILQ_INIT(&rx_adapter->vector_list);
2344 strcpy(rx_adapter->mem_name, mem_name);
2345 rx_adapter->eth_devices = rte_zmalloc_socket(rx_adapter->mem_name,
2347 sizeof(struct eth_device_info), 0,
2349 rte_convert_rss_key((const uint32_t *)default_rss_key,
2350 (uint32_t *)rx_adapter->rss_key_be,
2351 RTE_DIM(default_rss_key));
2353 if (rx_adapter->eth_devices == NULL) {
2354 RTE_EDEV_LOG_ERR("failed to get mem for eth devices\n");
2355 rte_free(rx_adapter);
2359 rte_spinlock_init(&rx_adapter->rx_lock);
2361 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
2362 rx_adapter->eth_devices[i].dev = &rte_eth_devices[i];
2364 /* Rx adapter event buffer allocation */
2365 rx_adapter->use_queue_event_buf = rxa_params->use_queue_event_buf;
2367 if (!rx_adapter->use_queue_event_buf) {
2368 buf = &rx_adapter->event_enqueue_buffer;
2369 buf->events_size = rxa_params->event_buf_size;
2371 events = rte_zmalloc_socket(rx_adapter->mem_name,
2372 buf->events_size * sizeof(*events),
2374 if (events == NULL) {
2375 RTE_EDEV_LOG_ERR("Failed to allocate memory "
2376 "for adapter event buffer");
2377 rte_free(rx_adapter->eth_devices);
2378 rte_free(rx_adapter);
2382 rx_adapter->event_enqueue_buffer.events = events;
2385 event_eth_rx_adapter[id] = rx_adapter;
2387 if (conf_cb == rxa_default_conf_cb)
2388 rx_adapter->default_cb_arg = 1;
2390 if (rte_mbuf_dyn_rx_timestamp_register(
2391 &event_eth_rx_timestamp_dynfield_offset,
2392 &event_eth_rx_timestamp_dynflag) != 0) {
2393 RTE_EDEV_LOG_ERR("Error registering timestamp field in mbuf\n");
2397 rte_eventdev_trace_eth_rx_adapter_create(id, dev_id, conf_cb,
2403 rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id,
2404 rte_event_eth_rx_adapter_conf_cb conf_cb,
2407 struct rte_event_eth_rx_adapter_params rxa_params = {0};
2409 /* use default values for adapter params */
2410 rxa_params.event_buf_size = ETH_EVENT_BUFFER_SIZE;
2411 rxa_params.use_queue_event_buf = false;
2413 return rxa_create(id, dev_id, &rxa_params, conf_cb, conf_arg);
2417 rte_event_eth_rx_adapter_create_with_params(uint8_t id, uint8_t dev_id,
2418 struct rte_event_port_conf *port_config,
2419 struct rte_event_eth_rx_adapter_params *rxa_params)
2421 struct rte_event_port_conf *pc;
2423 struct rte_event_eth_rx_adapter_params temp_params = {0};
2425 if (port_config == NULL)
2428 if (rxa_params == NULL) {
2429 /* use default values if rxa_params is NULL */
2430 rxa_params = &temp_params;
2431 rxa_params->event_buf_size = ETH_EVENT_BUFFER_SIZE;
2432 rxa_params->use_queue_event_buf = false;
2433 } else if ((!rxa_params->use_queue_event_buf &&
2434 rxa_params->event_buf_size == 0) ||
2435 (rxa_params->use_queue_event_buf &&
2436 rxa_params->event_buf_size != 0)) {
2437 RTE_EDEV_LOG_ERR("Invalid adapter params\n");
2439 } else if (!rxa_params->use_queue_event_buf) {
2440 /* adjust event buff size with BATCH_SIZE used for fetching
2441 * packets from NIC rx queues to get full buffer utilization
2442 * and prevent unnecessary rollovers.
2445 rxa_params->event_buf_size =
2446 RTE_ALIGN(rxa_params->event_buf_size, BATCH_SIZE);
2447 rxa_params->event_buf_size += (BATCH_SIZE + BATCH_SIZE);
2450 pc = rte_malloc(NULL, sizeof(*pc), 0);
2456 ret = rxa_create(id, dev_id, rxa_params, rxa_default_conf_cb, pc);
2464 rte_event_eth_rx_adapter_create(uint8_t id, uint8_t dev_id,
2465 struct rte_event_port_conf *port_config)
2467 struct rte_event_port_conf *pc;
2470 if (port_config == NULL)
2473 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2475 pc = rte_malloc(NULL, sizeof(*pc), 0);
2480 ret = rte_event_eth_rx_adapter_create_ext(id, dev_id,
2481 rxa_default_conf_cb,
2489 rte_event_eth_rx_adapter_free(uint8_t id)
2491 struct rte_event_eth_rx_adapter *rx_adapter;
2493 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2495 rx_adapter = rxa_id_to_adapter(id);
2496 if (rx_adapter == NULL)
2499 if (rx_adapter->nb_queues) {
2500 RTE_EDEV_LOG_ERR("%" PRIu16 " Rx queues not deleted",
2501 rx_adapter->nb_queues);
2505 if (rx_adapter->default_cb_arg)
2506 rte_free(rx_adapter->conf_arg);
2507 rte_free(rx_adapter->eth_devices);
2508 if (!rx_adapter->use_queue_event_buf)
2509 rte_free(rx_adapter->event_enqueue_buffer.events);
2510 rte_free(rx_adapter);
2511 event_eth_rx_adapter[id] = NULL;
2513 rte_eventdev_trace_eth_rx_adapter_free(id);
2518 rte_event_eth_rx_adapter_queue_add(uint8_t id,
2519 uint16_t eth_dev_id,
2520 int32_t rx_queue_id,
2521 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
2525 struct rte_event_eth_rx_adapter *rx_adapter;
2526 struct rte_eventdev *dev;
2527 struct eth_device_info *dev_info;
2528 struct rte_event_eth_rx_adapter_vector_limits limits;
2530 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2531 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
2533 rx_adapter = rxa_id_to_adapter(id);
2534 if ((rx_adapter == NULL) || (queue_conf == NULL))
2537 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2538 ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
2542 RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
2543 "eth port %" PRIu16, id, eth_dev_id);
2547 if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) == 0
2548 && (queue_conf->rx_queue_flags &
2549 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID)) {
2550 RTE_EDEV_LOG_ERR("Flow ID override is not supported,"
2551 " eth port: %" PRIu16 " adapter id: %" PRIu8,
2556 if (queue_conf->rx_queue_flags &
2557 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR) {
2559 if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) == 0) {
2560 RTE_EDEV_LOG_ERR("Event vectorization is not supported,"
2561 " eth port: %" PRIu16
2562 " adapter id: %" PRIu8,
2567 ret = rte_event_eth_rx_adapter_vector_limits_get(
2568 rx_adapter->eventdev_id, eth_dev_id, &limits);
2570 RTE_EDEV_LOG_ERR("Failed to get event device vector limits,"
2571 " eth port: %" PRIu16
2572 " adapter id: %" PRIu8,
2576 if (queue_conf->vector_sz < limits.min_sz ||
2577 queue_conf->vector_sz > limits.max_sz ||
2578 queue_conf->vector_timeout_ns < limits.min_timeout_ns ||
2579 queue_conf->vector_timeout_ns > limits.max_timeout_ns ||
2580 queue_conf->vector_mp == NULL) {
2581 RTE_EDEV_LOG_ERR("Invalid event vector configuration,"
2582 " eth port: %" PRIu16
2583 " adapter id: %" PRIu8,
2587 if (queue_conf->vector_mp->elt_size <
2588 (sizeof(struct rte_event_vector) +
2589 (sizeof(uintptr_t) * queue_conf->vector_sz))) {
2590 RTE_EDEV_LOG_ERR("Invalid event vector configuration,"
2591 " eth port: %" PRIu16
2592 " adapter id: %" PRIu8,
2598 if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ) == 0 &&
2599 (rx_queue_id != -1)) {
2600 RTE_EDEV_LOG_ERR("Rx queues can only be connected to single "
2601 "event queue, eth port: %" PRIu16 " adapter id: %"
2602 PRIu8, eth_dev_id, id);
2606 if (rx_queue_id != -1 && (uint16_t)rx_queue_id >=
2607 rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
2608 RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16,
2609 (uint16_t)rx_queue_id);
2613 if ((rx_adapter->use_queue_event_buf &&
2614 queue_conf->event_buf_size == 0) ||
2615 (!rx_adapter->use_queue_event_buf &&
2616 queue_conf->event_buf_size != 0)) {
2617 RTE_EDEV_LOG_ERR("Invalid Event buffer size for the queue");
2621 dev_info = &rx_adapter->eth_devices[eth_dev_id];
2623 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
2624 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->eth_rx_adapter_queue_add,
2626 if (dev_info->rx_queue == NULL) {
2627 dev_info->rx_queue =
2628 rte_zmalloc_socket(rx_adapter->mem_name,
2629 dev_info->dev->data->nb_rx_queues *
2630 sizeof(struct eth_rx_queue_info), 0,
2631 rx_adapter->socket_id);
2632 if (dev_info->rx_queue == NULL)
2636 ret = (*dev->dev_ops->eth_rx_adapter_queue_add)(dev,
2637 &rte_eth_devices[eth_dev_id],
2638 rx_queue_id, queue_conf);
2640 dev_info->internal_event_port = 1;
2641 rxa_update_queue(rx_adapter,
2642 &rx_adapter->eth_devices[eth_dev_id],
2647 rte_spinlock_lock(&rx_adapter->rx_lock);
2648 dev_info->internal_event_port = 0;
2649 ret = rxa_init_service(rx_adapter, id);
2651 uint32_t service_id = rx_adapter->service_id;
2652 ret = rxa_sw_add(rx_adapter, eth_dev_id, rx_queue_id,
2654 rte_service_component_runstate_set(service_id,
2655 rxa_sw_adapter_queue_count(rx_adapter));
2657 rte_spinlock_unlock(&rx_adapter->rx_lock);
2660 rte_eventdev_trace_eth_rx_adapter_queue_add(id, eth_dev_id,
2661 rx_queue_id, queue_conf, ret);
2669 rxa_sw_vector_limits(struct rte_event_eth_rx_adapter_vector_limits *limits)
2671 limits->max_sz = MAX_VECTOR_SIZE;
2672 limits->min_sz = MIN_VECTOR_SIZE;
2673 limits->max_timeout_ns = MAX_VECTOR_NS;
2674 limits->min_timeout_ns = MIN_VECTOR_NS;
2680 rte_event_eth_rx_adapter_queue_del(uint8_t id, uint16_t eth_dev_id,
2681 int32_t rx_queue_id)
2684 struct rte_eventdev *dev;
2685 struct rte_event_eth_rx_adapter *rx_adapter;
2686 struct eth_device_info *dev_info;
2688 uint32_t nb_rx_poll = 0;
2689 uint32_t nb_wrr = 0;
2690 uint32_t nb_rx_intr;
2691 struct eth_rx_poll_entry *rx_poll = NULL;
2692 uint32_t *rx_wrr = NULL;
2695 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2696 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
2698 rx_adapter = rxa_id_to_adapter(id);
2699 if (rx_adapter == NULL)
2702 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2703 ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
2709 if (rx_queue_id != -1 && (uint16_t)rx_queue_id >=
2710 rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
2711 RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16,
2712 (uint16_t)rx_queue_id);
2716 dev_info = &rx_adapter->eth_devices[eth_dev_id];
2718 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
2719 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->eth_rx_adapter_queue_del,
2721 ret = (*dev->dev_ops->eth_rx_adapter_queue_del)(dev,
2722 &rte_eth_devices[eth_dev_id],
2725 rxa_update_queue(rx_adapter,
2726 &rx_adapter->eth_devices[eth_dev_id],
2729 if (dev_info->nb_dev_queues == 0) {
2730 rte_free(dev_info->rx_queue);
2731 dev_info->rx_queue = NULL;
2735 rxa_calc_nb_post_del(rx_adapter, dev_info, rx_queue_id,
2736 &nb_rx_poll, &nb_rx_intr, &nb_wrr);
2738 ret = rxa_alloc_poll_arrays(rx_adapter, nb_rx_poll, nb_wrr,
2743 rte_spinlock_lock(&rx_adapter->rx_lock);
2746 if (rx_adapter->num_rx_intr > nb_rx_intr) {
2748 num_intr_vec = rxa_nb_intr_vect(dev_info,
2750 ret = rxa_del_intr_queue(rx_adapter, dev_info,
2756 if (nb_rx_intr == 0) {
2757 ret = rxa_free_intr_resources(rx_adapter);
2762 rxa_sw_del(rx_adapter, dev_info, rx_queue_id);
2763 rxa_calc_wrr_sequence(rx_adapter, rx_poll, rx_wrr);
2765 rte_free(rx_adapter->eth_rx_poll);
2766 rte_free(rx_adapter->wrr_sched);
2768 if (nb_rx_intr == 0) {
2769 rte_free(dev_info->intr_queue);
2770 dev_info->intr_queue = NULL;
2773 rx_adapter->eth_rx_poll = rx_poll;
2774 rx_adapter->wrr_sched = rx_wrr;
2775 rx_adapter->wrr_len = nb_wrr;
2776 rx_adapter->num_intr_vec += num_intr_vec;
2778 if (dev_info->nb_dev_queues == 0) {
2779 rte_free(dev_info->rx_queue);
2780 dev_info->rx_queue = NULL;
2783 rte_spinlock_unlock(&rx_adapter->rx_lock);
2790 rte_service_component_runstate_set(rx_adapter->service_id,
2791 rxa_sw_adapter_queue_count(rx_adapter));
2794 rte_eventdev_trace_eth_rx_adapter_queue_del(id, eth_dev_id,
2800 rte_event_eth_rx_adapter_vector_limits_get(
2801 uint8_t dev_id, uint16_t eth_port_id,
2802 struct rte_event_eth_rx_adapter_vector_limits *limits)
2804 struct rte_eventdev *dev;
2808 RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
2809 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_port_id, -EINVAL);
2814 dev = &rte_eventdevs[dev_id];
2816 ret = rte_event_eth_rx_adapter_caps_get(dev_id, eth_port_id, &cap);
2818 RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
2819 "eth port %" PRIu16,
2820 dev_id, eth_port_id);
2824 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
2825 RTE_FUNC_PTR_OR_ERR_RET(
2826 *dev->dev_ops->eth_rx_adapter_vector_limits_get,
2828 ret = dev->dev_ops->eth_rx_adapter_vector_limits_get(
2829 dev, &rte_eth_devices[eth_port_id], limits);
2831 ret = rxa_sw_vector_limits(limits);
2838 rte_event_eth_rx_adapter_start(uint8_t id)
2840 rte_eventdev_trace_eth_rx_adapter_start(id);
2841 return rxa_ctrl(id, 1);
2845 rte_event_eth_rx_adapter_stop(uint8_t id)
2847 rte_eventdev_trace_eth_rx_adapter_stop(id);
2848 return rxa_ctrl(id, 0);
2852 rte_event_eth_rx_adapter_stats_get(uint8_t id,
2853 struct rte_event_eth_rx_adapter_stats *stats)
2855 struct rte_event_eth_rx_adapter *rx_adapter;
2856 struct rte_eth_event_enqueue_buffer *buf;
2857 struct rte_event_eth_rx_adapter_stats dev_stats_sum = { 0 };
2858 struct rte_event_eth_rx_adapter_stats dev_stats;
2859 struct rte_eventdev *dev;
2860 struct eth_device_info *dev_info;
2864 if (rxa_memzone_lookup())
2867 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2869 rx_adapter = rxa_id_to_adapter(id);
2870 if (rx_adapter == NULL || stats == NULL)
2873 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2874 memset(stats, 0, sizeof(*stats));
2875 RTE_ETH_FOREACH_DEV(i) {
2876 dev_info = &rx_adapter->eth_devices[i];
2877 if (dev_info->internal_event_port == 0 ||
2878 dev->dev_ops->eth_rx_adapter_stats_get == NULL)
2880 ret = (*dev->dev_ops->eth_rx_adapter_stats_get)(dev,
2881 &rte_eth_devices[i],
2885 dev_stats_sum.rx_packets += dev_stats.rx_packets;
2886 dev_stats_sum.rx_enq_count += dev_stats.rx_enq_count;
2889 if (rx_adapter->service_inited)
2890 *stats = rx_adapter->stats;
2892 stats->rx_packets += dev_stats_sum.rx_packets;
2893 stats->rx_enq_count += dev_stats_sum.rx_enq_count;
2895 if (!rx_adapter->use_queue_event_buf) {
2896 buf = &rx_adapter->event_enqueue_buffer;
2897 stats->rx_event_buf_count = buf->count;
2898 stats->rx_event_buf_size = buf->events_size;
2900 stats->rx_event_buf_count = 0;
2901 stats->rx_event_buf_size = 0;
2908 rte_event_eth_rx_adapter_stats_reset(uint8_t id)
2910 struct rte_event_eth_rx_adapter *rx_adapter;
2911 struct rte_eventdev *dev;
2912 struct eth_device_info *dev_info;
2915 if (rxa_memzone_lookup())
2918 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2920 rx_adapter = rxa_id_to_adapter(id);
2921 if (rx_adapter == NULL)
2924 dev = &rte_eventdevs[rx_adapter->eventdev_id];
2925 RTE_ETH_FOREACH_DEV(i) {
2926 dev_info = &rx_adapter->eth_devices[i];
2927 if (dev_info->internal_event_port == 0 ||
2928 dev->dev_ops->eth_rx_adapter_stats_reset == NULL)
2930 (*dev->dev_ops->eth_rx_adapter_stats_reset)(dev,
2931 &rte_eth_devices[i]);
2934 memset(&rx_adapter->stats, 0, sizeof(rx_adapter->stats));
2939 rte_event_eth_rx_adapter_service_id_get(uint8_t id, uint32_t *service_id)
2941 struct rte_event_eth_rx_adapter *rx_adapter;
2943 if (rxa_memzone_lookup())
2946 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2948 rx_adapter = rxa_id_to_adapter(id);
2949 if (rx_adapter == NULL || service_id == NULL)
2952 if (rx_adapter->service_inited)
2953 *service_id = rx_adapter->service_id;
2955 return rx_adapter->service_inited ? 0 : -ESRCH;
2959 rte_event_eth_rx_adapter_cb_register(uint8_t id,
2960 uint16_t eth_dev_id,
2961 rte_event_eth_rx_adapter_cb_fn cb_fn,
2964 struct rte_event_eth_rx_adapter *rx_adapter;
2965 struct eth_device_info *dev_info;
2969 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
2970 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
2972 rx_adapter = rxa_id_to_adapter(id);
2973 if (rx_adapter == NULL)
2976 dev_info = &rx_adapter->eth_devices[eth_dev_id];
2977 if (dev_info->rx_queue == NULL)
2980 ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
2984 RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
2985 "eth port %" PRIu16, id, eth_dev_id);
2989 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
2990 RTE_EDEV_LOG_ERR("Rx callback not supported for eth port %"
2991 PRIu16, eth_dev_id);
2995 rte_spinlock_lock(&rx_adapter->rx_lock);
2996 dev_info->cb_fn = cb_fn;
2997 dev_info->cb_arg = cb_arg;
2998 rte_spinlock_unlock(&rx_adapter->rx_lock);
3004 rte_event_eth_rx_adapter_queue_conf_get(uint8_t id,
3005 uint16_t eth_dev_id,
3006 uint16_t rx_queue_id,
3007 struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
3009 struct rte_eventdev *dev;
3010 struct rte_event_eth_rx_adapter *rx_adapter;
3011 struct eth_device_info *dev_info;
3012 struct eth_rx_queue_info *queue_info;
3013 struct rte_event *qi_ev;
3016 if (rxa_memzone_lookup())
3019 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
3020 RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
3022 if (rx_queue_id >= rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
3023 RTE_EDEV_LOG_ERR("Invalid rx queue_id %u", rx_queue_id);
3027 if (queue_conf == NULL) {
3028 RTE_EDEV_LOG_ERR("Rx queue conf struct cannot be NULL");
3032 rx_adapter = rxa_id_to_adapter(id);
3033 if (rx_adapter == NULL)
3036 dev_info = &rx_adapter->eth_devices[eth_dev_id];
3037 if (dev_info->rx_queue == NULL ||
3038 !dev_info->rx_queue[rx_queue_id].queue_enabled) {
3039 RTE_EDEV_LOG_ERR("Rx queue %u not added", rx_queue_id);
3043 queue_info = &dev_info->rx_queue[rx_queue_id];
3044 qi_ev = (struct rte_event *)&queue_info->event;
3046 memset(queue_conf, 0, sizeof(*queue_conf));
3047 queue_conf->rx_queue_flags = 0;
3048 if (queue_info->flow_id_mask != 0)
3049 queue_conf->rx_queue_flags |=
3050 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID;
3051 queue_conf->servicing_weight = queue_info->wt;
3053 memcpy(&queue_conf->ev, qi_ev, sizeof(*qi_ev));
3055 dev = &rte_eventdevs[rx_adapter->eventdev_id];
3056 if (dev->dev_ops->eth_rx_adapter_queue_conf_get != NULL) {
3057 ret = (*dev->dev_ops->eth_rx_adapter_queue_conf_get)(dev,
3058 &rte_eth_devices[eth_dev_id],
3067 #define RXA_ADD_DICT(stats, s) rte_tel_data_add_dict_u64(d, #s, stats.s)
3070 handle_rxa_stats(const char *cmd __rte_unused,
3072 struct rte_tel_data *d)
3074 uint8_t rx_adapter_id;
3075 struct rte_event_eth_rx_adapter_stats rx_adptr_stats;
3077 if (params == NULL || strlen(params) == 0 || !isdigit(*params))
3080 /* Get Rx adapter ID from parameter string */
3081 rx_adapter_id = atoi(params);
3082 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(rx_adapter_id, -EINVAL);
3084 /* Get Rx adapter stats */
3085 if (rte_event_eth_rx_adapter_stats_get(rx_adapter_id,
3087 RTE_EDEV_LOG_ERR("Failed to get Rx adapter stats\n");
3091 rte_tel_data_start_dict(d);
3092 rte_tel_data_add_dict_u64(d, "rx_adapter_id", rx_adapter_id);
3093 RXA_ADD_DICT(rx_adptr_stats, rx_packets);
3094 RXA_ADD_DICT(rx_adptr_stats, rx_poll_count);
3095 RXA_ADD_DICT(rx_adptr_stats, rx_dropped);
3096 RXA_ADD_DICT(rx_adptr_stats, rx_enq_retry);
3097 RXA_ADD_DICT(rx_adptr_stats, rx_event_buf_count);
3098 RXA_ADD_DICT(rx_adptr_stats, rx_event_buf_size);
3099 RXA_ADD_DICT(rx_adptr_stats, rx_enq_count);
3100 RXA_ADD_DICT(rx_adptr_stats, rx_enq_start_ts);
3101 RXA_ADD_DICT(rx_adptr_stats, rx_enq_block_cycles);
3102 RXA_ADD_DICT(rx_adptr_stats, rx_enq_end_ts);
3103 RXA_ADD_DICT(rx_adptr_stats, rx_intr_packets);
3109 handle_rxa_stats_reset(const char *cmd __rte_unused,
3111 struct rte_tel_data *d __rte_unused)
3113 uint8_t rx_adapter_id;
3115 if (params == NULL || strlen(params) == 0 || ~isdigit(*params))
3118 /* Get Rx adapter ID from parameter string */
3119 rx_adapter_id = atoi(params);
3120 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(rx_adapter_id, -EINVAL);
3122 /* Reset Rx adapter stats */
3123 if (rte_event_eth_rx_adapter_stats_reset(rx_adapter_id)) {
3124 RTE_EDEV_LOG_ERR("Failed to reset Rx adapter stats\n");
3132 handle_rxa_get_queue_conf(const char *cmd __rte_unused,
3134 struct rte_tel_data *d)
3136 uint8_t rx_adapter_id;
3137 uint16_t rx_queue_id;
3139 char *token, *l_params;
3140 struct rte_event_eth_rx_adapter_queue_conf queue_conf;
3142 if (params == NULL || strlen(params) == 0 || !isdigit(*params))
3145 /* Get Rx adapter ID from parameter string */
3146 l_params = strdup(params);
3147 token = strtok(l_params, ",");
3148 rx_adapter_id = strtoul(token, NULL, 10);
3149 RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(rx_adapter_id, -EINVAL);
3151 token = strtok(NULL, ",");
3152 if (token == NULL || strlen(token) == 0 || !isdigit(*token))
3155 /* Get device ID from parameter string */
3156 eth_dev_id = strtoul(token, NULL, 10);
3157 RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(eth_dev_id, -EINVAL);
3159 token = strtok(NULL, ",");
3160 if (token == NULL || strlen(token) == 0 || !isdigit(*token))
3163 /* Get Rx queue ID from parameter string */
3164 rx_queue_id = strtoul(token, NULL, 10);
3165 if (rx_queue_id >= rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
3166 RTE_EDEV_LOG_ERR("Invalid rx queue_id %u", rx_queue_id);
3170 token = strtok(NULL, "\0");
3172 RTE_EDEV_LOG_ERR("Extra parameters passed to eventdev"
3173 " telemetry command, igrnoring");
3175 if (rte_event_eth_rx_adapter_queue_conf_get(rx_adapter_id, eth_dev_id,
3176 rx_queue_id, &queue_conf)) {
3177 RTE_EDEV_LOG_ERR("Failed to get Rx adapter queue config");
3181 rte_tel_data_start_dict(d);
3182 rte_tel_data_add_dict_u64(d, "rx_adapter_id", rx_adapter_id);
3183 rte_tel_data_add_dict_u64(d, "eth_dev_id", eth_dev_id);
3184 rte_tel_data_add_dict_u64(d, "rx_queue_id", rx_queue_id);
3185 RXA_ADD_DICT(queue_conf, rx_queue_flags);
3186 RXA_ADD_DICT(queue_conf, servicing_weight);
3187 RXA_ADD_DICT(queue_conf.ev, queue_id);
3188 RXA_ADD_DICT(queue_conf.ev, sched_type);
3189 RXA_ADD_DICT(queue_conf.ev, priority);
3190 RXA_ADD_DICT(queue_conf.ev, flow_id);
3195 RTE_INIT(rxa_init_telemetry)
3197 rte_telemetry_register_cmd("/eventdev/rxa_stats",
3199 "Returns Rx adapter stats. Parameter: rxa_id");
3201 rte_telemetry_register_cmd("/eventdev/rxa_stats_reset",
3202 handle_rxa_stats_reset,
3203 "Reset Rx adapter stats. Parameter: rxa_id");
3205 rte_telemetry_register_cmd("/eventdev/rxa_queue_conf",
3206 handle_rxa_get_queue_conf,
3207 "Returns Rx queue config. Parameter: rxa_id, dev_id, queue_id");