X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fark%2Fark_ethdev_rx.c;h=0fbb2603db4ac1c088a775547b65ea49b6aa929c;hb=6baa15684c5a2ea3f4d7a6f4cfc7f30d86a51fea;hp=f5d812a5594af37594d99bc713c36ac91eaf77cf;hpb=76f5f48c51ccbb5ab696af6ae22a7c1acbe63a07;p=dpdk.git diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c index f5d812a559..0fbb2603db 100644 --- a/drivers/net/ark/ark_ethdev_rx.c +++ b/drivers/net/ark/ark_ethdev_rx.c @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright (c) 2015-2017 Atomic Rules LLC - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of copyright holder nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Atomic Rules LLC */ #include @@ -41,7 +12,7 @@ #define ARK_RX_META_SIZE 32 #define ARK_RX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_RX_META_SIZE) -#define ARK_RX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) +#define ARK_RX_MPU_CHUNK (64U) /* Forward declarations */ struct ark_rx_queue; @@ -61,12 +32,18 @@ struct ark_rx_queue { struct rte_mbuf **reserve_q; /* array of physical addresses of the mbuf data pointer */ /* This point is a virtual address */ - phys_addr_t *paddress_q; + rte_iova_t *paddress_q; struct rte_mempool *mb_pool; struct ark_udm_t *udm; struct ark_mpu_t *mpu; + rx_user_meta_hook_fn rx_user_meta_hook; + void *ext_user_data; + + uint32_t dataroom; + uint32_t headroom; + uint32_t queue_size; uint32_t queue_mask; @@ -79,35 +56,33 @@ struct ark_rx_queue { /* The queue Index is used within the dpdk device structures */ uint16_t queue_index; - uint32_t pad1; + uint32_t unused; - /* separate cache line */ - /* second cache line - fields only used in slow path */ - MARKER cacheline1 __rte_cache_min_aligned; + /* next cache line - fields written by device */ + RTE_MARKER cacheline1 __rte_cache_min_aligned; volatile uint32_t prod_index; /* step 2 filled by FPGA */ } __rte_cache_aligned; - /* ************************************************************************* */ static int eth_ark_rx_hw_setup(struct rte_eth_dev *dev, struct ark_rx_queue *queue, uint16_t rx_queue_id __rte_unused, uint16_t rx_queue_idx) { - phys_addr_t queue_base; - phys_addr_t phys_addr_q_base; - phys_addr_t phys_addr_prod_index; + rte_iova_t queue_base; + rte_iova_t phys_addr_q_base; + rte_iova_t phys_addr_prod_index; - queue_base = rte_malloc_virt2phy(queue); + queue_base = rte_malloc_virt2iova(queue); phys_addr_prod_index = queue_base + offsetof(struct ark_rx_queue, prod_index); - phys_addr_q_base = rte_malloc_virt2phy(queue->paddress_q); + phys_addr_q_base = rte_malloc_virt2iova(queue->paddress_q); /* Verify HW */ - if (ark_mpu_verify(queue->mpu, sizeof(phys_addr_t))) { - PMD_DRV_LOG(ERR, "Illegal configuration rx queue\n"); + if (ark_mpu_verify(queue->mpu, sizeof(rte_iova_t))) { + ARK_PMD_LOG(ERR, "Illegal configuration rx queue\n"); return -1; } @@ -130,8 +105,10 @@ static inline void eth_ark_rx_update_cons_index(struct ark_rx_queue *queue, uint32_t cons_index) { queue->cons_index = cons_index; - eth_ark_rx_seed_mbufs(queue); - ark_mpu_set_producer(queue->mpu, queue->seed_index); + if ((cons_index + queue->queue_size - queue->seed_index) >= ARK_RX_MPU_CHUNK) { + eth_ark_rx_seed_mbufs(queue); + ark_mpu_set_producer(queue->mpu, queue->seed_index); + } } /* ************************************************************************* */ @@ -144,15 +121,13 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, struct rte_mempool *mb_pool) { static int warning1; /* = 0 */ - struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private; + struct ark_adapter *ark = dev->data->dev_private; struct ark_rx_queue *queue; uint32_t i; int status; - /* Future works: divide the Q's evenly with multi-ports */ - int port = dev->data->port_id; - int qidx = port + queue_idx; + int qidx = queue_idx; /* We may already be setup, free memory prior to re-allocation */ if (dev->data->rx_queues[queue_idx] != NULL) { @@ -162,19 +137,19 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, if (rx_conf != NULL && warning1 == 0) { warning1 = 1; - PMD_DRV_LOG(INFO, + ARK_PMD_LOG(NOTICE, "Arkville ignores rte_eth_rxconf argument.\n"); } if (RTE_PKTMBUF_HEADROOM < ARK_RX_META_SIZE) { - PMD_DRV_LOG(ERR, + ARK_PMD_LOG(ERR, "Error: DPDK Arkville requires head room > %d bytes (%s)\n", ARK_RX_META_SIZE, __func__); return -1; /* ERROR CODE */ } if (!rte_is_power_of_2(nb_desc)) { - PMD_DRV_LOG(ERR, + ARK_PMD_LOG(ERR, "DPDK Arkville configuration queue size must be power of two %u (%s)\n", nb_desc, __func__); return -1; /* ERROR CODE */ @@ -186,30 +161,35 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, 64, socket_id); if (queue == 0) { - PMD_DRV_LOG(ERR, "Failed to allocate memory in %s\n", __func__); + ARK_PMD_LOG(ERR, "Failed to allocate memory in %s\n", __func__); return -ENOMEM; } /* NOTE zmalloc is used, no need to 0 indexes, etc. */ queue->mb_pool = mb_pool; + queue->dataroom = rte_pktmbuf_data_room_size(mb_pool) - + RTE_PKTMBUF_HEADROOM; + queue->headroom = RTE_PKTMBUF_HEADROOM; queue->phys_qid = qidx; queue->queue_index = queue_idx; queue->queue_size = nb_desc; queue->queue_mask = nb_desc - 1; + queue->rx_user_meta_hook = ark->user_ext.rx_user_meta_hook; + queue->ext_user_data = ark->user_data[dev->data->port_id]; queue->reserve_q = rte_zmalloc_socket("Ark_rx_queue mbuf", nb_desc * sizeof(struct rte_mbuf *), - 64, + 512, socket_id); queue->paddress_q = rte_zmalloc_socket("Ark_rx_queue paddr", - nb_desc * sizeof(phys_addr_t), - 64, + nb_desc * sizeof(rte_iova_t), + 512, socket_id); if (queue->reserve_q == 0 || queue->paddress_q == 0) { - PMD_DRV_LOG(ERR, + ARK_PMD_LOG(ERR, "Failed to allocate queue memory in %s\n", __func__); rte_free(queue->reserve_q); @@ -222,23 +202,37 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, queue->udm = RTE_PTR_ADD(ark->udm.v, qidx * ARK_UDM_QOFFSET); queue->mpu = RTE_PTR_ADD(ark->mpurx.v, qidx * ARK_MPU_QOFFSET); + /* Configure UDM per queue */ + ark_udm_stop(queue->udm, 0); + ark_udm_configure(queue->udm, + RTE_PKTMBUF_HEADROOM, + queue->dataroom, + ARK_RX_WRITE_TIME_NS); + ark_udm_stats_reset(queue->udm); + ark_udm_stop(queue->udm, 0); + /* populate mbuf reserve */ status = eth_ark_rx_seed_mbufs(queue); + if (queue->seed_index != nb_desc) { + ARK_PMD_LOG(ERR, "Failed to allocate %u mbufs for RX queue %d\n", + nb_desc, qidx); + status = -1; + } /* MPU Setup */ if (status == 0) status = eth_ark_rx_hw_setup(dev, queue, qidx, queue_idx); if (unlikely(status != 0)) { - struct rte_mbuf *mbuf; + struct rte_mbuf **mbuf; - PMD_DRV_LOG(ERR, "Failed to initialize RX queue %d %s\n", + ARK_PMD_LOG(ERR, "Failed to initialize RX queue %d %s\n", qidx, __func__); /* Free the mbufs allocated */ - for (i = 0, mbuf = queue->reserve_q[0]; - i < nb_desc; ++i, mbuf++) { - rte_pktmbuf_free(mbuf); + for (i = 0, mbuf = queue->reserve_q; + i < queue->seed_index; ++i, mbuf++) { + rte_pktmbuf_free(*mbuf); } rte_free(queue->reserve_q); rte_free(queue->paddress_q); @@ -249,15 +243,6 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, return 0; } -/* ************************************************************************* */ -uint16_t -eth_ark_recv_pkts_noop(void *rx_queue __rte_unused, - struct rte_mbuf **rx_pkts __rte_unused, - uint16_t nb_pkts __rte_unused) -{ - return 0; -} - /* ************************************************************************* */ uint16_t eth_ark_recv_pkts(void *rx_queue, @@ -267,8 +252,11 @@ eth_ark_recv_pkts(void *rx_queue, struct ark_rx_queue *queue; register uint32_t cons_index, prod_index; uint16_t nb; + uint16_t i; struct rte_mbuf *mbuf; + struct rte_mbuf **pmbuf; struct ark_rx_meta *meta; + rx_user_meta_hook_fn rx_user_meta_hook; queue = (struct ark_rx_queue *)rx_queue; if (unlikely(queue == 0)) @@ -277,6 +265,8 @@ eth_ark_recv_pkts(void *rx_queue, return 0; prod_index = queue->prod_index; cons_index = queue->cons_index; + if (prod_index == cons_index) + return 0; nb = 0; while (prod_index != cons_index) { @@ -288,16 +278,14 @@ eth_ark_recv_pkts(void *rx_queue, /* META DATA embedded in headroom */ meta = RTE_PTR_ADD(mbuf->buf_addr, ARK_RX_META_OFFSET); - mbuf->port = meta->port; mbuf->pkt_len = meta->pkt_len; mbuf->data_len = meta->pkt_len; - mbuf->timestamp = meta->timestamp; - mbuf->udata64 = meta->user_data; - if (ARK_RX_DEBUG) { /* debug sanity checks */ + if (ARK_DEBUG_CORE) { /* debug sanity checks */ + if ((meta->pkt_len > (1024 * 16)) || (meta->pkt_len == 0)) { - PMD_RX_LOG(DEBUG, "RX: Bad Meta Q: %u" + ARK_PMD_LOG(DEBUG, "RX: Bad Meta Q: %u" " cons: %" PRIU32 " prod: %" PRIU32 " seed_index %" PRIU32 @@ -308,7 +296,7 @@ eth_ark_recv_pkts(void *rx_queue, queue->seed_index); - PMD_RX_LOG(DEBUG, " : UDM" + ARK_PMD_LOG(DEBUG, " : UDM" " prod: %" PRIU32 " len: %u\n", queue->udm->rt_cfg.prod_idx, @@ -321,11 +309,9 @@ eth_ark_recv_pkts(void *rx_queue, mbuf->pkt_len = 63; meta->pkt_len = 63; } - /* seqn is only set under debug */ - mbuf->seqn = cons_index; } - if (unlikely(meta->pkt_len > ARK_RX_MAX_NOCHAIN)) + if (unlikely(meta->pkt_len > queue->dataroom)) cons_index = eth_ark_rx_jumbo (queue, meta, mbuf, cons_index + 1); else @@ -337,9 +323,14 @@ eth_ark_recv_pkts(void *rx_queue, break; } - if (unlikely(nb != 0)) - /* report next free to FPGA */ - eth_ark_rx_update_cons_index(queue, cons_index); + rx_user_meta_hook = queue->rx_user_meta_hook; + for (pmbuf = rx_pkts, i = 0; rx_user_meta_hook && i < nb; i++) { + mbuf = *pmbuf++; + meta = RTE_PTR_ADD(mbuf->buf_addr, ARK_RX_META_OFFSET); + rx_user_meta_hook(mbuf, meta->user_meta, queue->ext_user_data); + } + + eth_ark_rx_update_cons_index(queue, cons_index); return nb; } @@ -356,20 +347,19 @@ eth_ark_rx_jumbo(struct ark_rx_queue *queue, uint16_t remaining; uint16_t data_len; - uint8_t segments; + uint16_t segments; /* first buf populated by called */ mbuf_prev = mbuf0; segments = 1; - data_len = RTE_MIN(meta->pkt_len, RTE_MBUF_DEFAULT_DATAROOM); + data_len = RTE_MIN(meta->pkt_len, queue->dataroom); remaining = meta->pkt_len - data_len; mbuf0->data_len = data_len; /* HW guarantees that the data does not exceed prod_index! */ while (remaining != 0) { data_len = RTE_MIN(remaining, - RTE_MBUF_DEFAULT_DATAROOM + - RTE_PKTMBUF_HEADROOM); + queue->dataroom); remaining -= data_len; segments += 1; @@ -378,9 +368,6 @@ eth_ark_rx_jumbo(struct ark_rx_queue *queue, mbuf_prev->next = mbuf; mbuf_prev = mbuf; mbuf->data_len = data_len; - mbuf->data_off = 0; - if (ARK_RX_DEBUG) - mbuf->seqn = cons_index; /* for debug only */ cons_index += 1; } @@ -408,11 +395,11 @@ eth_ark_rx_queue_drain(struct ark_rx_queue *queue) } uint32_t -eth_ark_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_id) +eth_ark_dev_rx_queue_count(void *rx_queue) { struct ark_rx_queue *queue; - queue = dev->data->rx_queues[queue_id]; + queue = rx_queue; return (queue->prod_index - queue->cons_index); /* mod arith */ } @@ -460,7 +447,8 @@ eth_ark_rx_stop_queue(struct rte_eth_dev *dev, uint16_t queue_id) static inline int eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue) { - uint32_t limit = queue->cons_index + queue->queue_size; + uint32_t limit = (queue->cons_index & ~(ARK_RX_MPU_CHUNK - 1)) + + queue->queue_size; uint32_t seed_index = queue->seed_index; uint32_t count = 0; @@ -475,10 +463,17 @@ eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue) struct rte_mbuf **mbufs = &queue->reserve_q[seed_m]; int status = rte_pktmbuf_alloc_bulk(queue->mb_pool, mbufs, nb); - if (unlikely(status != 0)) + if (unlikely(status != 0)) { + ARK_PMD_LOG(NOTICE, + "Could not allocate %u mbufs from pool" + " for RX queue %u;" + " %u free buffers remaining in queue\n", + nb, queue->queue_index, + queue->seed_index - queue->cons_index); return -1; + } - if (ARK_RX_DEBUG) { /* DEBUG */ + if (ARK_DEBUG_CORE) { /* DEBUG */ while (count != nb) { struct rte_mbuf *mbuf_init = queue->reserve_q[seed_m + count]; @@ -499,22 +494,22 @@ eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue) case 0: while (count != nb) { queue->paddress_q[seed_m++] = - (*mbufs++)->buf_physaddr; + (*mbufs++)->buf_iova; count++; /* FALLTHROUGH */ case 3: queue->paddress_q[seed_m++] = - (*mbufs++)->buf_physaddr; + (*mbufs++)->buf_iova; count++; /* FALLTHROUGH */ case 2: queue->paddress_q[seed_m++] = - (*mbufs++)->buf_physaddr; + (*mbufs++)->buf_iova; count++; /* FALLTHROUGH */ case 1: queue->paddress_q[seed_m++] = - (*mbufs++)->buf_physaddr; + (*mbufs++)->buf_iova; count++; /* FALLTHROUGH */ @@ -601,14 +596,14 @@ eth_rx_queue_stats_reset(void *vqueue) void eth_ark_udm_force_close(struct rte_eth_dev *dev) { - struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private; + struct ark_adapter *ark = dev->data->dev_private; struct ark_rx_queue *queue; uint32_t index; uint16_t i; if (!ark_udm_is_flushed(ark->udm.v)) { /* restart the MPUs */ - PMD_DRV_LOG(ERR, "ARK: %s UDM not flushed\n", __func__); + ARK_PMD_LOG(NOTICE, "UDM not flushed -- forcing flush\n"); for (i = 0; i < dev->data->nb_rx_queues; i++) { queue = (struct ark_rx_queue *)dev->data->rx_queues[i]; if (queue == 0) @@ -616,14 +611,14 @@ eth_ark_udm_force_close(struct rte_eth_dev *dev) ark_mpu_start(queue->mpu); /* Add some buffers */ - index = 100000 + queue->seed_index; + index = ARK_RX_MPU_CHUNK + queue->seed_index; ark_mpu_set_producer(queue->mpu, index); } /* Wait to allow data to pass */ usleep(100); - PMD_DEBUG_LOG(DEBUG, "UDM forced flush attempt, stopped = %d\n", - ark_udm_is_flushed(ark->udm.v)); + ARK_PMD_LOG(NOTICE, "UDM forced flush attempt, stopped = %d\n", + ark_udm_is_flushed(ark->udm.v)); } ark_udm_reset(ark->udm.v); } @@ -633,8 +628,8 @@ ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue) { if (queue == NULL) return; - PMD_DEBUG_LOG(DEBUG, "RX QUEUE %d -- %s", queue->phys_qid, name); - PMD_DEBUG_LOG(DEBUG, ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 "\n", + ARK_PMD_LOG(DEBUG, "RX QUEUE %d -- %s", queue->phys_qid, name); + ARK_PMD_LOG(DEBUG, ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 "\n", "queue_size", queue->queue_size, "seed_index", queue->seed_index, "prod_index", queue->prod_index, @@ -658,15 +653,15 @@ dump_mbuf_data(struct rte_mbuf *mbuf, uint16_t lo, uint16_t hi) { uint16_t i, j; - PMD_DRV_LOG(INFO, " MBUF: %p len %d, off: %d, seq: %" PRIU32 "\n", mbuf, - mbuf->pkt_len, mbuf->data_off, mbuf->seqn); + ARK_PMD_LOG(DEBUG, " MBUF: %p len %d, off: %d\n", + mbuf, mbuf->pkt_len, mbuf->data_off); for (i = lo; i < hi; i += 16) { uint8_t *dp = RTE_PTR_ADD(mbuf->buf_addr, i); - PMD_DRV_LOG(INFO, " %6d: ", i); + ARK_PMD_LOG(DEBUG, " %6d: ", i); for (j = 0; j < 16; j++) - PMD_DRV_LOG(INFO, " %02x", dp[j]); + ARK_PMD_LOG(DEBUG, " %02x", dp[j]); - PMD_DRV_LOG(INFO, "\n"); + ARK_PMD_LOG(DEBUG, "\n"); } }