4 * Copyright(c) 2015 EZchip Semiconductor Ltd. All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of EZchip Semiconductor nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <rte_eal_memconfig.h>
38 #include <rte_ethdev.h>
39 #include <rte_malloc.h>
40 #include <rte_cycles.h>
42 #include <arch/mpipe_xaui_def.h>
43 #include <arch/mpipe_gbe_def.h>
45 #include <gxio/mpipe.h>
47 #ifdef RTE_LIBRTE_MPIPE_PMD_DEBUG
48 #define PMD_DEBUG_RX(...) RTE_LOG(DEBUG, PMD, __VA_ARGS__)
49 #define PMD_DEBUG_TX(...) RTE_LOG(DEBUG, PMD, __VA_ARGS__)
51 #define PMD_DEBUG_RX(...)
52 #define PMD_DEBUG_TX(...)
55 #define MPIPE_MAX_CHANNELS 128
56 #define MPIPE_TX_MAX_QUEUES 128
57 #define MPIPE_RX_MAX_QUEUES 16
58 #define MPIPE_TX_DESCS 512
59 #define MPIPE_RX_BUCKETS 256
60 #define MPIPE_RX_STACK_SIZE 65536
61 #define MPIPE_RX_IP_ALIGN 2
62 #define MPIPE_BSM_ALIGN 128
64 #define MPIPE_LINK_UPDATE_TIMEOUT 10 /* s */
65 #define MPIPE_LINK_UPDATE_INTERVAL 100000 /* us */
67 struct mpipe_channel_config {
72 gxio_mpipe_rules_stacks_t stacks;
75 struct mpipe_context {
77 gxio_mpipe_context_t context;
78 struct mpipe_channel_config channels[MPIPE_MAX_CHANNELS];
81 /* Per-core local data. */
83 int mbuf_push_debt[RTE_MAX_ETHPORTS]; /* Buffer push debt. */
84 } __rte_cache_aligned;
86 #define MPIPE_BUF_DEBT_THRESHOLD 32
87 static __thread struct mpipe_local mpipe_local;
88 static struct mpipe_context mpipe_contexts[GXIO_MPIPE_INSTANCE_MAX];
89 static int mpipe_instances;
90 static const char *drivername = "MPIPE PMD";
92 /* Per queue statistics. */
93 struct mpipe_queue_stats {
94 uint64_t packets, bytes, errors, nomem;
97 /* Common tx/rx queue fields. */
99 struct mpipe_dev_priv *priv; /* "priv" data of its device. */
100 uint16_t nb_desc; /* Number of tx descriptors. */
101 uint16_t port_id; /* Device index. */
102 uint16_t stat_idx; /* Queue stats index. */
103 uint8_t queue_idx; /* Queue index. */
104 uint8_t link_status; /* 0 = link down. */
105 struct mpipe_queue_stats stats; /* Stat data for the queue. */
108 /* Transmit queue description. */
109 struct mpipe_tx_queue {
110 struct mpipe_queue q; /* Common stuff. */
113 /* Receive queue description. */
114 struct mpipe_rx_queue {
115 struct mpipe_queue q; /* Common stuff. */
116 gxio_mpipe_iqueue_t iqueue; /* mPIPE iqueue. */
117 gxio_mpipe_idesc_t *next_desc; /* Next idesc to process. */
118 int avail_descs; /* Number of available descs. */
119 void *rx_ring_mem; /* DMA ring memory. */
122 struct mpipe_dev_priv {
123 gxio_mpipe_context_t *context; /* mPIPE context. */
124 gxio_mpipe_link_t link; /* mPIPE link for the device. */
125 gxio_mpipe_equeue_t equeue; /* mPIPE equeue. */
126 unsigned equeue_size; /* mPIPE equeue desc count. */
127 int instance; /* mPIPE instance. */
128 int ering; /* mPIPE eDMA ring. */
129 int stack; /* mPIPE buffer stack. */
130 int channel; /* Device channel. */
131 int port_id; /* DPDK port index. */
132 struct rte_eth_dev *eth_dev; /* DPDK device. */
133 struct rte_mbuf **tx_comps; /* TX completion array. */
134 struct rte_mempool *rx_mpool; /* mpool used by the rx queues. */
135 unsigned rx_offset; /* Receive head room. */
136 unsigned rx_size_code; /* mPIPE rx buffer size code. */
137 unsigned rx_buffers; /* receive buffers on stack. */
138 int is_xaui:1, /* Is this an xgbe or gbe? */
139 initialized:1, /* Initialized port? */
140 running:1; /* Running port? */
141 struct ether_addr mac_addr; /* MAC address. */
142 unsigned nb_rx_queues; /* Configured tx queues. */
143 unsigned nb_tx_queues; /* Configured rx queues. */
144 int first_bucket; /* mPIPE bucket start index. */
145 int first_ring; /* mPIPE notif ring start index. */
146 int notif_group; /* mPIPE notif group. */
147 rte_atomic32_t dp_count __rte_cache_aligned; /* DP Entry count. */
148 int tx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
149 int rx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
152 #define mpipe_priv(dev) \
153 ((struct mpipe_dev_priv*)(dev)->data->dev_private)
155 #define mpipe_name(priv) \
156 ((priv)->eth_dev->data->name)
158 #define mpipe_rx_queue(priv, n) \
159 ((struct mpipe_rx_queue *)(priv)->eth_dev->data->rx_queues[n])
161 #define mpipe_tx_queue(priv, n) \
162 ((struct mpipe_tx_queue *)(priv)->eth_dev->data->tx_queues[n])
165 mpipe_xmit_flush(struct mpipe_dev_priv *priv);
168 mpipe_recv_flush(struct mpipe_dev_priv *priv);
170 static int mpipe_equeue_sizes[] = {
171 [GXIO_MPIPE_EQUEUE_ENTRY_512] = 512,
172 [GXIO_MPIPE_EQUEUE_ENTRY_2K] = 2048,
173 [GXIO_MPIPE_EQUEUE_ENTRY_8K] = 8192,
174 [GXIO_MPIPE_EQUEUE_ENTRY_64K] = 65536,
177 static int mpipe_iqueue_sizes[] = {
178 [GXIO_MPIPE_IQUEUE_ENTRY_128] = 128,
179 [GXIO_MPIPE_IQUEUE_ENTRY_512] = 512,
180 [GXIO_MPIPE_IQUEUE_ENTRY_2K] = 2048,
181 [GXIO_MPIPE_IQUEUE_ENTRY_64K] = 65536,
184 static int mpipe_buffer_sizes[] = {
185 [GXIO_MPIPE_BUFFER_SIZE_128] = 128,
186 [GXIO_MPIPE_BUFFER_SIZE_256] = 256,
187 [GXIO_MPIPE_BUFFER_SIZE_512] = 512,
188 [GXIO_MPIPE_BUFFER_SIZE_1024] = 1024,
189 [GXIO_MPIPE_BUFFER_SIZE_1664] = 1664,
190 [GXIO_MPIPE_BUFFER_SIZE_4096] = 4096,
191 [GXIO_MPIPE_BUFFER_SIZE_10368] = 10368,
192 [GXIO_MPIPE_BUFFER_SIZE_16384] = 16384,
195 static gxio_mpipe_context_t *
196 mpipe_context(int instance)
198 if (instance < 0 || instance >= mpipe_instances)
200 return &mpipe_contexts[instance].context;
203 static int mpipe_channel_config(int instance, int channel,
204 struct mpipe_channel_config *config)
206 struct mpipe_channel_config *data;
207 struct mpipe_context *context;
208 gxio_mpipe_rules_t rules;
211 if (instance < 0 || instance >= mpipe_instances ||
212 channel < 0 || channel >= MPIPE_MAX_CHANNELS)
215 context = &mpipe_contexts[instance];
217 rte_spinlock_lock(&context->lock);
219 gxio_mpipe_rules_init(&rules, &context->context);
221 for (idx = 0; idx < MPIPE_MAX_CHANNELS; idx++) {
222 data = (channel == idx) ? config : &context->channels[idx];
227 rc = gxio_mpipe_rules_begin(&rules, data->first_bucket,
228 data->num_buckets, &data->stacks);
233 rc = gxio_mpipe_rules_add_channel(&rules, idx);
238 rc = gxio_mpipe_rules_set_headroom(&rules, data->head_room);
244 rc = gxio_mpipe_rules_commit(&rules);
246 memcpy(&context->channels[channel], config, sizeof(*config));
250 rte_spinlock_unlock(&context->lock);
256 mpipe_get_size_index(int *array, int count, int size,
261 for (i = 0; i < count && array[i] < size; i++) {
267 return i < count ? (int)i : -ENOENT;
269 return last >= 0 ? last : -ENOENT;
273 mpipe_calc_size(int *array, int count, int size)
275 int index = mpipe_get_size_index(array, count, size, 1);
276 return index < 0 ? index : array[index];
279 static int mpipe_equeue_size(int size)
282 result = mpipe_calc_size(mpipe_equeue_sizes,
283 RTE_DIM(mpipe_equeue_sizes), size);
287 static int mpipe_iqueue_size(int size)
290 result = mpipe_calc_size(mpipe_iqueue_sizes,
291 RTE_DIM(mpipe_iqueue_sizes), size);
295 static int mpipe_buffer_size_index(int size)
298 result = mpipe_get_size_index(mpipe_buffer_sizes,
299 RTE_DIM(mpipe_buffer_sizes), size, 0);
304 mpipe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
305 struct rte_eth_link *link)
307 struct rte_eth_link *dst = link;
308 struct rte_eth_link *src = &(dev->data->dev_link);
310 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
311 *(uint64_t *)src) == 0)
318 mpipe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
319 struct rte_eth_link *link)
321 struct rte_eth_link *dst = &(dev->data->dev_link);
322 struct rte_eth_link *src = link;
324 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
325 *(uint64_t *)src) == 0)
332 mpipe_infos_get(struct rte_eth_dev *dev __rte_unused,
333 struct rte_eth_dev_info *dev_info)
335 dev_info->min_rx_bufsize = 128;
336 dev_info->max_rx_pktlen = 1518;
337 dev_info->max_tx_queues = MPIPE_TX_MAX_QUEUES;
338 dev_info->max_rx_queues = MPIPE_RX_MAX_QUEUES;
339 dev_info->max_mac_addrs = 1;
340 dev_info->rx_offload_capa = 0;
341 dev_info->tx_offload_capa = 0;
345 mpipe_configure(struct rte_eth_dev *dev)
347 struct mpipe_dev_priv *priv = mpipe_priv(dev);
349 if (dev->data->nb_tx_queues > MPIPE_TX_MAX_QUEUES) {
350 RTE_LOG(ERR, PMD, "%s: Too many tx queues: %d > %d\n",
351 mpipe_name(priv), dev->data->nb_tx_queues,
352 MPIPE_TX_MAX_QUEUES);
355 priv->nb_tx_queues = dev->data->nb_tx_queues;
357 if (dev->data->nb_rx_queues > MPIPE_RX_MAX_QUEUES) {
358 RTE_LOG(ERR, PMD, "%s: Too many rx queues: %d > %d\n",
359 mpipe_name(priv), dev->data->nb_rx_queues,
360 MPIPE_RX_MAX_QUEUES);
362 priv->nb_rx_queues = dev->data->nb_rx_queues;
368 mpipe_link_compare(struct rte_eth_link *link1,
369 struct rte_eth_link *link2)
371 return (*(uint64_t *)link1 == *(uint64_t *)link2)
376 mpipe_link_update(struct rte_eth_dev *dev, int wait_to_complete)
378 struct mpipe_dev_priv *priv = mpipe_priv(dev);
379 struct rte_eth_link old, new;
380 int64_t state, speed;
383 memset(&old, 0, sizeof(old));
384 memset(&new, 0, sizeof(new));
385 mpipe_dev_atomic_read_link_status(dev, &old);
387 for (count = 0, rc = 0; count < MPIPE_LINK_UPDATE_TIMEOUT; count++) {
388 if (!priv->initialized)
391 state = gxio_mpipe_link_get_attr(&priv->link,
392 GXIO_MPIPE_LINK_CURRENT_STATE);
396 speed = state & GXIO_MPIPE_LINK_SPEED_MASK;
398 if (speed == GXIO_MPIPE_LINK_1G) {
399 new.link_speed = ETH_LINK_SPEED_1000;
400 new.link_duplex = ETH_LINK_FULL_DUPLEX;
402 } else if (speed == GXIO_MPIPE_LINK_10G) {
403 new.link_speed = ETH_LINK_SPEED_10000;
404 new.link_duplex = ETH_LINK_FULL_DUPLEX;
408 rc = mpipe_link_compare(&old, &new);
409 if (rc == 0 || !wait_to_complete)
412 rte_delay_us(MPIPE_LINK_UPDATE_INTERVAL);
415 mpipe_dev_atomic_write_link_status(dev, &new);
420 mpipe_set_link(struct rte_eth_dev *dev, int up)
422 struct mpipe_dev_priv *priv = mpipe_priv(dev);
425 rc = gxio_mpipe_link_set_attr(&priv->link,
426 GXIO_MPIPE_LINK_DESIRED_STATE,
427 up ? GXIO_MPIPE_LINK_ANYSPEED : 0);
429 RTE_LOG(ERR, PMD, "%s: Failed to set link %s.\n",
430 mpipe_name(priv), up ? "up" : "down");
432 mpipe_link_update(dev, 0);
439 mpipe_set_link_up(struct rte_eth_dev *dev)
441 return mpipe_set_link(dev, 1);
445 mpipe_set_link_down(struct rte_eth_dev *dev)
447 return mpipe_set_link(dev, 0);
451 mpipe_dp_enter(struct mpipe_dev_priv *priv)
453 __insn_mtspr(SPR_DSTREAM_PF, 0);
454 rte_atomic32_inc(&priv->dp_count);
458 mpipe_dp_exit(struct mpipe_dev_priv *priv)
460 rte_atomic32_dec(&priv->dp_count);
464 mpipe_dp_wait(struct mpipe_dev_priv *priv)
466 while (rte_atomic32_read(&priv->dp_count) != 0) {
472 mpipe_mbuf_stack_index(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
474 return (mbuf->port < RTE_MAX_ETHPORTS) ?
475 mpipe_priv(&rte_eth_devices[mbuf->port])->stack :
479 static inline struct rte_mbuf *
480 mpipe_recv_mbuf(struct mpipe_dev_priv *priv, gxio_mpipe_idesc_t *idesc,
483 void *va = gxio_mpipe_idesc_get_va(idesc);
484 uint16_t size = gxio_mpipe_idesc_get_xfer_size(idesc);
485 struct rte_mbuf *mbuf = RTE_PTR_SUB(va, priv->rx_offset);
487 rte_pktmbuf_reset(mbuf);
488 mbuf->data_off = (uintptr_t)va - (uintptr_t)mbuf->buf_addr;
489 mbuf->port = in_port;
490 mbuf->data_len = size;
491 mbuf->pkt_len = size;
492 mbuf->hash.rss = gxio_mpipe_idesc_get_flow_hash(idesc);
494 PMD_DEBUG_RX("%s: RX mbuf %p, buffer %p, buf_addr %p, size %d\n",
495 mpipe_name(priv), mbuf, va, mbuf->buf_addr, size);
501 mpipe_recv_push(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
503 const int offset = RTE_PKTMBUF_HEADROOM + MPIPE_RX_IP_ALIGN;
504 void *buf_addr = RTE_PTR_ADD(mbuf->buf_addr, offset);
506 gxio_mpipe_push_buffer(priv->context, priv->stack, buf_addr);
507 PMD_DEBUG_RX("%s: Pushed mbuf %p, buffer %p into stack %d\n",
508 mpipe_name(priv), mbuf, buf_addr, priv->stack);
512 mpipe_recv_fill_stack(struct mpipe_dev_priv *priv, int count)
514 struct rte_mbuf *mbuf;
517 for (i = 0; i < count; i++) {
518 mbuf = __rte_mbuf_raw_alloc(priv->rx_mpool);
521 mpipe_recv_push(priv, mbuf);
524 priv->rx_buffers += count;
525 PMD_DEBUG_RX("%s: Filled %d/%d buffers\n", mpipe_name(priv), i, count);
529 mpipe_recv_flush_stack(struct mpipe_dev_priv *priv)
531 const int offset = priv->rx_offset & ~RTE_MEMPOOL_ALIGN_MASK;
532 uint8_t in_port = priv->port_id;
533 struct rte_mbuf *mbuf;
537 for (count = 0; count < priv->rx_buffers; count++) {
538 va = gxio_mpipe_pop_buffer(priv->context, priv->stack);
541 mbuf = RTE_PTR_SUB(va, offset);
543 PMD_DEBUG_RX("%s: Flushing mbuf %p, va %p\n",
544 mpipe_name(priv), mbuf, va);
546 mbuf->data_off = (uintptr_t)va - (uintptr_t)mbuf->buf_addr;
549 mbuf->port = in_port;
550 mbuf->packet_type = 0;
554 __rte_mbuf_raw_free(mbuf);
557 PMD_DEBUG_RX("%s: Returned %d/%d buffers\n",
558 mpipe_name(priv), count, priv->rx_buffers);
559 priv->rx_buffers -= count;
563 mpipe_register_segment(struct mpipe_dev_priv *priv, const struct rte_memseg *ms)
565 size_t size = ms->hugepage_sz;
569 for (addr = ms->addr, end = addr + ms->len; addr < end; addr += size) {
570 rc = gxio_mpipe_register_page(priv->context, priv->stack, addr,
577 RTE_LOG(ERR, PMD, "%s: Could not register memseg @%p, %d.\n",
578 mpipe_name(priv), ms->addr, rc);
580 RTE_LOG(DEBUG, PMD, "%s: Registered segment %p - %p\n",
581 mpipe_name(priv), ms->addr,
582 RTE_PTR_ADD(ms->addr, ms->len - 1));
587 mpipe_recv_init(struct mpipe_dev_priv *priv)
589 const struct rte_memseg *seg = rte_eal_get_physmem_layout();
594 if (!priv->rx_mpool) {
595 RTE_LOG(ERR, PMD, "%s: No buffer pool.\n",
600 /* Allocate one NotifRing for each queue. */
601 rc = gxio_mpipe_alloc_notif_rings(priv->context, MPIPE_RX_MAX_QUEUES,
604 RTE_LOG(ERR, PMD, "%s: Failed to allocate notif rings.\n",
608 priv->first_ring = rc;
610 /* Allocate a NotifGroup. */
611 rc = gxio_mpipe_alloc_notif_groups(priv->context, 1, 0, 0);
613 RTE_LOG(ERR, PMD, "%s: Failed to allocate rx group.\n",
617 priv->notif_group = rc;
619 /* Allocate required buckets. */
620 rc = gxio_mpipe_alloc_buckets(priv->context, MPIPE_RX_BUCKETS, 0, 0);
622 RTE_LOG(ERR, PMD, "%s: Failed to allocate buckets.\n",
626 priv->first_bucket = rc;
628 rc = gxio_mpipe_alloc_buffer_stacks(priv->context, 1, 0, 0);
630 RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer stack.\n",
636 while (seg && seg->addr)
637 mpipe_register_segment(priv, seg++);
639 stack_size = gxio_mpipe_calc_buffer_stack_bytes(MPIPE_RX_STACK_SIZE);
640 stack_mem = rte_zmalloc(NULL, stack_size, 65536);
642 RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer memory.\n",
646 RTE_LOG(DEBUG, PMD, "%s: Buffer stack memory %p - %p.\n",
647 mpipe_name(priv), stack_mem,
648 RTE_PTR_ADD(stack_mem, stack_size - 1));
651 rc = gxio_mpipe_init_buffer_stack(priv->context, priv->stack,
652 priv->rx_size_code, stack_mem,
655 RTE_LOG(ERR, PMD, "%s: Failed to initialize buffer stack.\n",
664 mpipe_xmit_init(struct mpipe_dev_priv *priv)
670 /* Allocate eDMA ring. */
671 rc = gxio_mpipe_alloc_edma_rings(priv->context, 1, 0, 0);
673 RTE_LOG(ERR, PMD, "%s: Failed to alloc tx ring.\n",
679 rc = mpipe_equeue_size(MPIPE_TX_DESCS);
681 RTE_LOG(ERR, PMD, "%s: Cannot allocate %d equeue descs.\n",
682 mpipe_name(priv), (int)MPIPE_TX_DESCS);
685 priv->equeue_size = rc;
687 /* Initialize completion array. */
688 ring_size = sizeof(priv->tx_comps[0]) * priv->equeue_size;
689 priv->tx_comps = rte_zmalloc(NULL, ring_size, RTE_CACHE_LINE_SIZE);
690 if (!priv->tx_comps) {
691 RTE_LOG(ERR, PMD, "%s: Failed to allocate egress comps.\n",
696 /* Allocate eDMA ring memory. */
697 ring_size = sizeof(gxio_mpipe_edesc_t) * priv->equeue_size;
698 ring_mem = rte_zmalloc(NULL, ring_size, ring_size);
700 RTE_LOG(ERR, PMD, "%s: Failed to allocate egress descs.\n",
704 RTE_LOG(DEBUG, PMD, "%s: eDMA ring memory %p - %p.\n",
705 mpipe_name(priv), ring_mem,
706 RTE_PTR_ADD(ring_mem, ring_size - 1));
709 /* Initialize eDMA ring. */
710 rc = gxio_mpipe_equeue_init(&priv->equeue, priv->context, priv->ering,
711 priv->channel, ring_mem, ring_size, 0);
713 RTE_LOG(ERR, PMD, "%s: Failed to init equeue\n",
722 mpipe_link_init(struct mpipe_dev_priv *priv)
727 rc = gxio_mpipe_link_open(&priv->link, priv->context,
728 mpipe_name(priv), GXIO_MPIPE_LINK_AUTO_NONE);
730 RTE_LOG(ERR, PMD, "%s: Failed to open link.\n",
735 /* Get the channel index. */
736 rc = gxio_mpipe_link_channel(&priv->link);
738 RTE_LOG(ERR, PMD, "%s: Bad channel\n",
748 mpipe_init(struct mpipe_dev_priv *priv)
752 if (priv->initialized)
755 rc = mpipe_link_init(priv);
757 RTE_LOG(ERR, PMD, "%s: Failed to init link.\n",
762 rc = mpipe_recv_init(priv);
764 RTE_LOG(ERR, PMD, "%s: Failed to init rx.\n",
769 rc = mpipe_xmit_init(priv);
771 RTE_LOG(ERR, PMD, "%s: Failed to init tx.\n",
777 priv->initialized = 1;
783 mpipe_start(struct rte_eth_dev *dev)
785 struct mpipe_dev_priv *priv = mpipe_priv(dev);
786 struct mpipe_channel_config config;
787 struct mpipe_rx_queue *rx_queue;
788 struct rte_eth_link eth_link;
789 unsigned queue, buffers = 0;
794 memset(ð_link, 0, sizeof(eth_link));
795 mpipe_dev_atomic_write_link_status(dev, ð_link);
797 rc = mpipe_init(priv);
801 /* Initialize NotifRings. */
802 for (queue = 0; queue < priv->nb_rx_queues; queue++) {
803 rx_queue = mpipe_rx_queue(priv, queue);
804 ring_size = rx_queue->q.nb_desc * sizeof(gxio_mpipe_idesc_t);
806 ring_mem = rte_malloc(NULL, ring_size, ring_size);
808 RTE_LOG(ERR, PMD, "%s: Failed to alloc rx descs.\n",
812 RTE_LOG(DEBUG, PMD, "%s: iDMA ring %d memory %p - %p.\n",
813 mpipe_name(priv), queue, ring_mem,
814 RTE_PTR_ADD(ring_mem, ring_size - 1));
817 rc = gxio_mpipe_iqueue_init(&rx_queue->iqueue, priv->context,
818 priv->first_ring + queue, ring_mem,
821 RTE_LOG(ERR, PMD, "%s: Failed to init rx queue.\n",
826 rx_queue->rx_ring_mem = ring_mem;
827 buffers += rx_queue->q.nb_desc;
830 /* Initialize ingress NotifGroup and buckets. */
831 rc = gxio_mpipe_init_notif_group_and_buckets(priv->context,
832 priv->notif_group, priv->first_ring, priv->nb_rx_queues,
833 priv->first_bucket, MPIPE_RX_BUCKETS,
834 GXIO_MPIPE_BUCKET_STATIC_FLOW_AFFINITY);
836 RTE_LOG(ERR, PMD, "%s: Failed to init group and buckets.\n",
841 /* Configure the classifier to deliver packets from this port. */
843 config.first_bucket = priv->first_bucket;
844 config.num_buckets = MPIPE_RX_BUCKETS;
845 memset(&config.stacks, 0xff, sizeof(config.stacks));
846 config.stacks.stacks[priv->rx_size_code] = priv->stack;
847 config.head_room = priv->rx_offset & RTE_MEMPOOL_ALIGN_MASK;
849 rc = mpipe_channel_config(priv->instance, priv->channel,
852 RTE_LOG(ERR, PMD, "%s: Failed to setup classifier.\n",
857 /* Fill empty buffers into the buffer stack. */
858 mpipe_recv_fill_stack(priv, buffers);
860 /* Bring up the link. */
861 mpipe_set_link_up(dev);
863 /* Start xmit/recv on queues. */
864 for (queue = 0; queue < priv->nb_tx_queues; queue++)
865 mpipe_tx_queue(priv, queue)->q.link_status = 1;
866 for (queue = 0; queue < priv->nb_rx_queues; queue++)
867 mpipe_rx_queue(priv, queue)->q.link_status = 1;
874 mpipe_stop(struct rte_eth_dev *dev)
876 struct mpipe_dev_priv *priv = mpipe_priv(dev);
877 struct mpipe_channel_config config;
881 for (queue = 0; queue < priv->nb_tx_queues; queue++)
882 mpipe_tx_queue(priv, queue)->q.link_status = 0;
883 for (queue = 0; queue < priv->nb_rx_queues; queue++)
884 mpipe_rx_queue(priv, queue)->q.link_status = 0;
886 /* Make sure the link_status writes land. */
890 * Wait for link_status change to register with straggling datapath
895 /* Bring down the link. */
896 mpipe_set_link_down(dev);
898 /* Remove classifier rules. */
899 memset(&config, 0, sizeof(config));
900 rc = mpipe_channel_config(priv->instance, priv->channel,
903 RTE_LOG(ERR, PMD, "%s: Failed to stop classifier.\n",
907 /* Flush completed xmit packets. */
908 mpipe_xmit_flush(priv);
910 /* Flush buffer stacks. */
911 mpipe_recv_flush(priv);
917 mpipe_close(struct rte_eth_dev *dev)
919 struct mpipe_dev_priv *priv = mpipe_priv(dev);
925 mpipe_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
927 struct mpipe_dev_priv *priv = mpipe_priv(dev);
928 struct mpipe_tx_queue *tx_queue;
929 struct mpipe_rx_queue *rx_queue;
933 memset(stats, 0, sizeof(*stats));
935 for (i = 0; i < priv->nb_tx_queues; i++) {
936 tx_queue = mpipe_tx_queue(priv, i);
938 stats->opackets += tx_queue->q.stats.packets;
939 stats->obytes += tx_queue->q.stats.bytes;
940 stats->oerrors += tx_queue->q.stats.errors;
942 idx = tx_queue->q.stat_idx;
943 if (idx != (uint16_t)-1) {
944 stats->q_opackets[idx] += tx_queue->q.stats.packets;
945 stats->q_obytes[idx] += tx_queue->q.stats.bytes;
946 stats->q_errors[idx] += tx_queue->q.stats.errors;
950 for (i = 0; i < priv->nb_rx_queues; i++) {
951 rx_queue = mpipe_rx_queue(priv, i);
953 stats->ipackets += rx_queue->q.stats.packets;
954 stats->ibytes += rx_queue->q.stats.bytes;
955 stats->ierrors += rx_queue->q.stats.errors;
956 stats->rx_nombuf += rx_queue->q.stats.nomem;
958 idx = rx_queue->q.stat_idx;
959 if (idx != (uint16_t)-1) {
960 stats->q_ipackets[idx] += rx_queue->q.stats.packets;
961 stats->q_ibytes[idx] += rx_queue->q.stats.bytes;
962 stats->q_errors[idx] += rx_queue->q.stats.errors;
968 mpipe_stats_reset(struct rte_eth_dev *dev)
970 struct mpipe_dev_priv *priv = mpipe_priv(dev);
971 struct mpipe_tx_queue *tx_queue;
972 struct mpipe_rx_queue *rx_queue;
975 for (i = 0; i < priv->nb_tx_queues; i++) {
976 tx_queue = mpipe_tx_queue(priv, i);
977 memset(&tx_queue->q.stats, 0, sizeof(tx_queue->q.stats));
980 for (i = 0; i < priv->nb_rx_queues; i++) {
981 rx_queue = mpipe_rx_queue(priv, i);
982 memset(&rx_queue->q.stats, 0, sizeof(rx_queue->q.stats));
987 mpipe_queue_stats_mapping_set(struct rte_eth_dev *dev, uint16_t queue_id,
988 uint8_t stat_idx, uint8_t is_rx)
990 struct mpipe_dev_priv *priv = mpipe_priv(dev);
993 priv->rx_stat_mapping[stat_idx] = queue_id;
995 priv->tx_stat_mapping[stat_idx] = queue_id;
1002 mpipe_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
1003 uint16_t nb_desc, unsigned int socket_id __rte_unused,
1004 const struct rte_eth_txconf *tx_conf __rte_unused)
1006 struct mpipe_tx_queue *tx_queue = dev->data->tx_queues[queue_idx];
1007 struct mpipe_dev_priv *priv = mpipe_priv(dev);
1010 tx_queue = rte_realloc(tx_queue, sizeof(*tx_queue),
1011 RTE_CACHE_LINE_SIZE);
1013 RTE_LOG(ERR, PMD, "%s: Failed to allocate TX queue.\n",
1018 memset(&tx_queue->q, 0, sizeof(tx_queue->q));
1019 tx_queue->q.priv = priv;
1020 tx_queue->q.queue_idx = queue_idx;
1021 tx_queue->q.port_id = dev->data->port_id;
1022 tx_queue->q.nb_desc = nb_desc;
1024 tx_queue->q.stat_idx = -1;
1025 for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) {
1026 if (priv->tx_stat_mapping[idx] == queue_idx)
1027 tx_queue->q.stat_idx = idx;
1030 dev->data->tx_queues[queue_idx] = tx_queue;
1036 mpipe_tx_queue_release(void *_txq)
1042 mpipe_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
1043 uint16_t nb_desc, unsigned int socket_id __rte_unused,
1044 const struct rte_eth_rxconf *rx_conf __rte_unused,
1045 struct rte_mempool *mp)
1047 struct mpipe_rx_queue *rx_queue = dev->data->rx_queues[queue_idx];
1048 struct mpipe_dev_priv *priv = mpipe_priv(dev);
1052 rc = mpipe_iqueue_size(nb_desc);
1054 RTE_LOG(ERR, PMD, "%s: Cannot allocate %d iqueue descs.\n",
1055 mpipe_name(priv), (int)nb_desc);
1059 if (rc != nb_desc) {
1060 RTE_LOG(WARNING, PMD, "%s: Extending RX descs from %d to %d.\n",
1061 mpipe_name(priv), (int)nb_desc, rc);
1065 size = sizeof(*rx_queue);
1066 rx_queue = rte_realloc(rx_queue, size, RTE_CACHE_LINE_SIZE);
1068 RTE_LOG(ERR, PMD, "%s: Failed to allocate RX queue.\n",
1073 memset(&rx_queue->q, 0, sizeof(rx_queue->q));
1074 rx_queue->q.priv = priv;
1075 rx_queue->q.nb_desc = nb_desc;
1076 rx_queue->q.port_id = dev->data->port_id;
1077 rx_queue->q.queue_idx = queue_idx;
1079 if (!priv->rx_mpool) {
1080 int size = (rte_pktmbuf_data_room_size(mp) -
1081 RTE_PKTMBUF_HEADROOM -
1084 priv->rx_offset = (sizeof(struct rte_mbuf) +
1085 rte_pktmbuf_priv_size(mp) +
1086 RTE_PKTMBUF_HEADROOM +
1089 RTE_LOG(ERR, PMD, "%s: Bad buffer size %d.\n",
1091 rte_pktmbuf_data_room_size(mp));
1095 priv->rx_size_code = mpipe_buffer_size_index(size);
1096 priv->rx_mpool = mp;
1099 if (priv->rx_mpool != mp) {
1100 RTE_LOG(WARNING, PMD, "%s: Ignoring multiple buffer pools.\n",
1104 rx_queue->q.stat_idx = -1;
1105 for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) {
1106 if (priv->rx_stat_mapping[idx] == queue_idx)
1107 rx_queue->q.stat_idx = idx;
1110 dev->data->rx_queues[queue_idx] = rx_queue;
1116 mpipe_rx_queue_release(void *_rxq)
1121 #define MPIPE_XGBE_ENA_HASH_MULTI \
1122 (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_MULTI_SHIFT)
1123 #define MPIPE_XGBE_ENA_HASH_UNI \
1124 (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_UNI_SHIFT)
1125 #define MPIPE_XGBE_COPY_ALL \
1126 (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__COPY_ALL_SHIFT)
1127 #define MPIPE_GBE_ENA_MULTI_HASH \
1128 (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__MULTI_HASH_ENA_SHIFT)
1129 #define MPIPE_GBE_ENA_UNI_HASH \
1130 (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__UNI_HASH_ENA_SHIFT)
1131 #define MPIPE_GBE_COPY_ALL \
1132 (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__COPY_ALL_SHIFT)
1135 mpipe_promiscuous_enable(struct rte_eth_dev *dev)
1137 struct mpipe_dev_priv *priv = mpipe_priv(dev);
1141 if (priv->is_xaui) {
1142 addr = MPIPE_XAUI_RECEIVE_CONFIGURATION;
1143 reg = gxio_mpipe_link_mac_rd(&priv->link, addr);
1144 reg &= ~MPIPE_XGBE_ENA_HASH_MULTI;
1145 reg &= ~MPIPE_XGBE_ENA_HASH_UNI;
1146 reg |= MPIPE_XGBE_COPY_ALL;
1147 gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1149 addr = MPIPE_GBE_NETWORK_CONFIGURATION;
1150 reg = gxio_mpipe_link_mac_rd(&priv->link, addr);
1151 reg &= ~MPIPE_GBE_ENA_MULTI_HASH;
1152 reg &= ~MPIPE_GBE_ENA_UNI_HASH;
1153 reg |= MPIPE_GBE_COPY_ALL;
1154 gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1159 mpipe_promiscuous_disable(struct rte_eth_dev *dev)
1161 struct mpipe_dev_priv *priv = mpipe_priv(dev);
1165 if (priv->is_xaui) {
1166 addr = MPIPE_XAUI_RECEIVE_CONFIGURATION;
1167 reg = gxio_mpipe_link_mac_rd(&priv->link, addr);
1168 reg |= MPIPE_XGBE_ENA_HASH_MULTI;
1169 reg |= MPIPE_XGBE_ENA_HASH_UNI;
1170 reg &= ~MPIPE_XGBE_COPY_ALL;
1171 gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1173 addr = MPIPE_GBE_NETWORK_CONFIGURATION;
1174 reg = gxio_mpipe_link_mac_rd(&priv->link, addr);
1175 reg |= MPIPE_GBE_ENA_MULTI_HASH;
1176 reg |= MPIPE_GBE_ENA_UNI_HASH;
1177 reg &= ~MPIPE_GBE_COPY_ALL;
1178 gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1182 static struct eth_dev_ops mpipe_dev_ops = {
1183 .dev_infos_get = mpipe_infos_get,
1184 .dev_configure = mpipe_configure,
1185 .dev_start = mpipe_start,
1186 .dev_stop = mpipe_stop,
1187 .dev_close = mpipe_close,
1188 .stats_get = mpipe_stats_get,
1189 .stats_reset = mpipe_stats_reset,
1190 .queue_stats_mapping_set = mpipe_queue_stats_mapping_set,
1191 .tx_queue_setup = mpipe_tx_queue_setup,
1192 .rx_queue_setup = mpipe_rx_queue_setup,
1193 .tx_queue_release = mpipe_tx_queue_release,
1194 .rx_queue_release = mpipe_rx_queue_release,
1195 .link_update = mpipe_link_update,
1196 .dev_set_link_up = mpipe_set_link_up,
1197 .dev_set_link_down = mpipe_set_link_down,
1198 .promiscuous_enable = mpipe_promiscuous_enable,
1199 .promiscuous_disable = mpipe_promiscuous_disable,
1203 mpipe_xmit_null(struct mpipe_dev_priv *priv, int64_t start, int64_t end)
1205 gxio_mpipe_edesc_t null_desc = { { .bound = 1, .ns = 1 } };
1206 gxio_mpipe_equeue_t *equeue = &priv->equeue;
1209 for (slot = start; slot < end; slot++) {
1210 gxio_mpipe_equeue_put_at(equeue, null_desc, slot);
1215 mpipe_xmit_flush(struct mpipe_dev_priv *priv)
1217 gxio_mpipe_equeue_t *equeue = &priv->equeue;
1220 /* Post a dummy descriptor and wait for its return. */
1221 slot = gxio_mpipe_equeue_reserve(equeue, 1);
1223 RTE_LOG(ERR, PMD, "%s: Failed to reserve stop slot.\n",
1228 mpipe_xmit_null(priv, slot, slot + 1);
1230 while (!gxio_mpipe_equeue_is_complete(equeue, slot, 1)) {
1234 for (slot = 0; slot < priv->equeue_size; slot++) {
1235 if (priv->tx_comps[slot])
1236 rte_pktmbuf_free_seg(priv->tx_comps[slot]);
1241 mpipe_recv_flush(struct mpipe_dev_priv *priv)
1243 uint8_t in_port = priv->port_id;
1244 struct mpipe_rx_queue *rx_queue;
1245 gxio_mpipe_iqueue_t *iqueue;
1246 gxio_mpipe_idesc_t idesc;
1247 struct rte_mbuf *mbuf;
1252 mpipe_recv_flush_stack(priv);
1254 /* Flush packets sitting in recv queues. */
1255 for (queue = 0; queue < priv->nb_rx_queues; queue++) {
1256 rx_queue = mpipe_rx_queue(priv, queue);
1257 iqueue = &rx_queue->iqueue;
1258 while (gxio_mpipe_iqueue_try_get(iqueue, &idesc) >= 0) {
1259 mbuf = mpipe_recv_mbuf(priv, &idesc, in_port);
1260 rte_pktmbuf_free(mbuf);
1263 rte_free(rx_queue->rx_ring_mem);
1265 } while (retries++ < 10 && priv->rx_buffers);
1267 if (priv->rx_buffers) {
1268 RTE_LOG(ERR, PMD, "%s: Leaked %d receive buffers.\n",
1269 mpipe_name(priv), priv->rx_buffers);
1271 PMD_DEBUG_RX("%s: Returned all receive buffers.\n",
1276 static inline uint16_t
1277 mpipe_do_xmit(struct mpipe_tx_queue *tx_queue, struct rte_mbuf **tx_pkts,
1280 struct mpipe_dev_priv *priv = tx_queue->q.priv;
1281 gxio_mpipe_equeue_t *equeue = &priv->equeue;
1282 unsigned nb_bytes = 0;
1283 unsigned nb_sent = 0;
1287 PMD_DEBUG_TX("Trying to transmit %d packets on %s:%d.\n",
1288 nb_pkts, mpipe_name(tx_queue->q.priv),
1289 tx_queue->q.queue_idx);
1291 /* Optimistic assumption that we need exactly one slot per packet. */
1292 nb_slots = RTE_MIN(nb_pkts, MPIPE_TX_DESCS / 2);
1295 struct rte_mbuf *mbuf = NULL, *pkt = NULL;
1298 /* Reserve eDMA ring slots. */
1299 slot = gxio_mpipe_equeue_try_reserve_fast(equeue, nb_slots);
1300 if (unlikely(slot < 0)) {
1304 for (i = 0; i < nb_slots; i++) {
1305 unsigned idx = (slot + i) & (priv->equeue_size - 1);
1306 rte_prefetch0(priv->tx_comps[idx]);
1309 /* Fill up slots with descriptor and completion info. */
1310 for (i = 0; i < nb_slots; i++) {
1311 unsigned idx = (slot + i) & (priv->equeue_size - 1);
1312 gxio_mpipe_edesc_t desc;
1313 struct rte_mbuf *next;
1315 /* Starting on a new packet? */
1316 if (likely(!mbuf)) {
1317 int room = nb_slots - i;
1319 pkt = mbuf = tx_pkts[nb_sent];
1321 /* Bail out if we run out of descs. */
1322 if (unlikely(pkt->nb_segs > room))
1328 /* We have a segment to send. */
1331 if (priv->tx_comps[idx])
1332 rte_pktmbuf_free_seg(priv->tx_comps[idx]);
1334 port_id = (mbuf->port < RTE_MAX_ETHPORTS) ?
1335 mbuf->port : priv->port_id;
1336 desc = (gxio_mpipe_edesc_t) { {
1337 .va = rte_pktmbuf_mtod(mbuf, uintptr_t),
1338 .xfer_size = rte_pktmbuf_data_len(mbuf),
1339 .bound = next ? 0 : 1,
1340 .stack_idx = mpipe_mbuf_stack_index(priv, mbuf),
1342 if (mpipe_local.mbuf_push_debt[port_id] > 0) {
1343 mpipe_local.mbuf_push_debt[port_id]--;
1345 priv->tx_comps[idx] = NULL;
1347 priv->tx_comps[idx] = mbuf;
1349 nb_bytes += mbuf->data_len;
1350 gxio_mpipe_equeue_put_at(equeue, desc, slot + i);
1352 PMD_DEBUG_TX("%s:%d: Sending packet %p, len %d\n",
1354 tx_queue->q.queue_idx,
1355 rte_pktmbuf_mtod(mbuf, void *),
1356 rte_pktmbuf_data_len(mbuf));
1361 if (unlikely(nb_sent < nb_pkts)) {
1363 /* Fill remaining slots with null descriptors. */
1364 mpipe_xmit_null(priv, slot + i, slot + nb_slots);
1367 * Calculate exact number of descriptors needed for
1368 * the next go around.
1371 for (i = nb_sent; i < nb_pkts; i++) {
1372 nb_slots += tx_pkts[i]->nb_segs;
1375 nb_slots = RTE_MIN(nb_slots, MPIPE_TX_DESCS / 2);
1377 } while (nb_sent < nb_pkts);
1379 tx_queue->q.stats.packets += nb_sent;
1380 tx_queue->q.stats.bytes += nb_bytes;
1385 static inline uint16_t
1386 mpipe_do_recv(struct mpipe_rx_queue *rx_queue, struct rte_mbuf **rx_pkts,
1389 struct mpipe_dev_priv *priv = rx_queue->q.priv;
1390 gxio_mpipe_iqueue_t *iqueue = &rx_queue->iqueue;
1391 gxio_mpipe_idesc_t *first_idesc, *idesc, *last_idesc;
1392 uint8_t in_port = rx_queue->q.port_id;
1393 const unsigned look_ahead = 8;
1394 int room = nb_pkts, rc = 0;
1395 unsigned nb_packets = 0;
1396 unsigned nb_dropped = 0;
1397 unsigned nb_nomem = 0;
1398 unsigned nb_bytes = 0;
1399 unsigned nb_descs, i;
1401 while (room && !rc) {
1402 if (rx_queue->avail_descs < room) {
1403 rc = gxio_mpipe_iqueue_try_peek(iqueue,
1404 &rx_queue->next_desc);
1405 rx_queue->avail_descs = rc < 0 ? 0 : rc;
1408 if (unlikely(!rx_queue->avail_descs)) {
1412 nb_descs = RTE_MIN(room, rx_queue->avail_descs);
1414 first_idesc = rx_queue->next_desc;
1415 last_idesc = first_idesc + nb_descs;
1417 rx_queue->next_desc += nb_descs;
1418 rx_queue->avail_descs -= nb_descs;
1420 for (i = 1; i < look_ahead; i++) {
1421 rte_prefetch0(first_idesc + i);
1424 PMD_DEBUG_RX("%s:%d: Trying to receive %d packets\n",
1425 mpipe_name(rx_queue->q.priv),
1426 rx_queue->q.queue_idx,
1429 for (idesc = first_idesc; idesc < last_idesc; idesc++) {
1430 struct rte_mbuf *mbuf;
1432 PMD_DEBUG_RX("%s:%d: processing idesc %d/%d\n",
1434 rx_queue->q.queue_idx,
1435 nb_packets, nb_descs);
1437 rte_prefetch0(idesc + look_ahead);
1439 PMD_DEBUG_RX("%s:%d: idesc %p, %s%s%s%s%s%s%s%s%s%s"
1440 "size: %d, bkt: %d, chan: %d, ring: %d, sqn: %lu, va: %lu\n",
1442 rx_queue->q.queue_idx,
1444 idesc->me ? "me, " : "",
1445 idesc->tr ? "tr, " : "",
1446 idesc->ce ? "ce, " : "",
1447 idesc->ct ? "ct, " : "",
1448 idesc->cs ? "cs, " : "",
1449 idesc->nr ? "nr, " : "",
1450 idesc->sq ? "sq, " : "",
1451 idesc->ts ? "ts, " : "",
1452 idesc->ps ? "ps, " : "",
1453 idesc->be ? "be, " : "",
1458 (unsigned long)idesc->packet_sqn,
1459 (unsigned long)idesc->va);
1461 if (unlikely(gxio_mpipe_idesc_has_error(idesc))) {
1463 gxio_mpipe_iqueue_drop(iqueue, idesc);
1464 PMD_DEBUG_RX("%s:%d: Descriptor error\n",
1465 mpipe_name(rx_queue->q.priv),
1466 rx_queue->q.queue_idx);
1470 if (mpipe_local.mbuf_push_debt[in_port] <
1471 MPIPE_BUF_DEBT_THRESHOLD)
1472 mpipe_local.mbuf_push_debt[in_port]++;
1474 mbuf = __rte_mbuf_raw_alloc(priv->rx_mpool);
1475 if (unlikely(!mbuf)) {
1477 gxio_mpipe_iqueue_drop(iqueue, idesc);
1478 PMD_DEBUG_RX("%s:%d: alloc failure\n",
1479 mpipe_name(rx_queue->q.priv),
1480 rx_queue->q.queue_idx);
1484 mpipe_recv_push(priv, mbuf);
1487 /* Get and setup the mbuf for the received packet. */
1488 mbuf = mpipe_recv_mbuf(priv, idesc, in_port);
1490 /* Update results and statistics counters. */
1491 rx_pkts[nb_packets] = mbuf;
1492 nb_bytes += mbuf->pkt_len;
1497 * We release the ring in bursts, but do not track and release
1498 * buckets. This therefore breaks dynamic flow affinity, but
1499 * we always operate in static affinity mode, and so we're OK
1500 * with this optimization.
1502 gxio_mpipe_iqueue_advance(iqueue, nb_descs);
1503 gxio_mpipe_credit(iqueue->context, iqueue->ring, -1, nb_descs);
1506 * Go around once more if we haven't yet peeked the queue, and
1507 * if we have more room to receive.
1509 room = nb_pkts - nb_packets;
1512 rx_queue->q.stats.packets += nb_packets;
1513 rx_queue->q.stats.bytes += nb_bytes;
1514 rx_queue->q.stats.errors += nb_dropped;
1515 rx_queue->q.stats.nomem += nb_nomem;
1517 PMD_DEBUG_RX("%s:%d: RX: %d/%d pkts/bytes, %d/%d drops/nomem\n",
1518 mpipe_name(rx_queue->q.priv), rx_queue->q.queue_idx,
1519 nb_packets, nb_bytes, nb_dropped, nb_nomem);
1525 mpipe_recv_pkts(void *_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1527 struct mpipe_rx_queue *rx_queue = _rxq;
1528 uint16_t result = 0;
1531 mpipe_dp_enter(rx_queue->q.priv);
1532 if (likely(rx_queue->q.link_status))
1533 result = mpipe_do_recv(rx_queue, rx_pkts, nb_pkts);
1534 mpipe_dp_exit(rx_queue->q.priv);
1541 mpipe_xmit_pkts(void *_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1543 struct mpipe_tx_queue *tx_queue = _txq;
1544 uint16_t result = 0;
1547 mpipe_dp_enter(tx_queue->q.priv);
1548 if (likely(tx_queue->q.link_status))
1549 result = mpipe_do_xmit(tx_queue, tx_pkts, nb_pkts);
1550 mpipe_dp_exit(tx_queue->q.priv);
1557 mpipe_link_mac(const char *ifname, uint8_t *mac)
1560 char name[GXIO_MPIPE_LINK_NAME_LEN];
1562 for (idx = 0, rc = 0; !rc; idx++) {
1563 rc = gxio_mpipe_link_enumerate_mac(idx, name, mac);
1564 if (!rc && !strncmp(name, ifname, GXIO_MPIPE_LINK_NAME_LEN))
1571 rte_pmd_mpipe_devinit(const char *ifname,
1572 const char *params __rte_unused)
1574 gxio_mpipe_context_t *context;
1575 struct rte_eth_dev *eth_dev;
1576 struct mpipe_dev_priv *priv;
1580 /* Get the mPIPE instance that the device belongs to. */
1581 instance = gxio_mpipe_link_instance(ifname);
1582 context = mpipe_context(instance);
1584 RTE_LOG(ERR, PMD, "%s: No device for link.\n", ifname);
1588 priv = rte_zmalloc(NULL, sizeof(*priv), 0);
1590 RTE_LOG(ERR, PMD, "%s: Failed to allocate priv.\n", ifname);
1594 memset(&priv->tx_stat_mapping, 0xff, sizeof(priv->tx_stat_mapping));
1595 memset(&priv->rx_stat_mapping, 0xff, sizeof(priv->rx_stat_mapping));
1596 priv->context = context;
1597 priv->instance = instance;
1598 priv->is_xaui = (strncmp(ifname, "xgbe", 4) == 0);
1601 mac = priv->mac_addr.addr_bytes;
1602 rc = mpipe_link_mac(ifname, mac);
1604 RTE_LOG(ERR, PMD, "%s: Failed to enumerate link.\n", ifname);
1609 eth_dev = rte_eth_dev_allocate(ifname, RTE_ETH_DEV_VIRTUAL);
1611 RTE_LOG(ERR, PMD, "%s: Failed to allocate device.\n", ifname);
1616 RTE_LOG(INFO, PMD, "%s: Initialized mpipe device"
1617 "(mac %02x:%02x:%02x:%02x:%02x:%02x).\n",
1618 ifname, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
1620 priv->eth_dev = eth_dev;
1621 priv->port_id = eth_dev->data->port_id;
1622 eth_dev->data->dev_private = priv;
1623 eth_dev->data->mac_addrs = &priv->mac_addr;
1625 eth_dev->data->dev_flags = 0;
1626 eth_dev->data->kdrv = RTE_KDRV_NONE;
1627 eth_dev->driver = NULL;
1628 eth_dev->data->drv_name = drivername;
1629 eth_dev->data->numa_node = instance;
1631 eth_dev->dev_ops = &mpipe_dev_ops;
1632 eth_dev->rx_pkt_burst = &mpipe_recv_pkts;
1633 eth_dev->tx_pkt_burst = &mpipe_xmit_pkts;
1638 static struct rte_driver pmd_mpipe_xgbe_drv = {
1641 .init = rte_pmd_mpipe_devinit,
1644 static struct rte_driver pmd_mpipe_gbe_drv = {
1647 .init = rte_pmd_mpipe_devinit,
1650 PMD_REGISTER_DRIVER(pmd_mpipe_xgbe_drv);
1651 PMD_REGISTER_DRIVER(pmd_mpipe_gbe_drv);
1653 static void __attribute__((constructor, used))
1654 mpipe_init_contexts(void)
1656 struct mpipe_context *context;
1659 for (instance = 0; instance < GXIO_MPIPE_INSTANCE_MAX; instance++) {
1660 context = &mpipe_contexts[instance];
1662 rte_spinlock_init(&context->lock);
1663 rc = gxio_mpipe_init(&context->context, instance);
1668 mpipe_instances = instance;