/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
+ * Copyright (c) 2015-2021 Atomic Rules LLC
*/
#include <unistd.h>
/* ************************************************************************* */
struct ark_tx_queue {
- struct ark_tx_meta *meta_q;
+ union ark_tx_meta *meta_q;
struct rte_mbuf **bufs;
/* handles for hw objects */
/* Stats HW tracks bytes and packets, need to count send errors */
uint64_t tx_errors;
+ tx_user_meta_hook_fn tx_user_meta_hook;
+ void *ext_user_data;
+
uint32_t queue_size;
uint32_t queue_mask;
/* 3 indexes to the paired data rings. */
- uint32_t prod_index; /* where to put the next one */
- uint32_t free_index; /* mbuf has been freed */
+ int32_t prod_index; /* where to put the next one */
+ int32_t free_index; /* mbuf has been freed */
/* The queue Id is used to identify the HW Q */
uint16_t phys_qid;
/* The queue Index within the dpdk device structures */
uint16_t queue_index;
- uint32_t pad[1];
-
- /* second cache line - fields only used in slow path */
+ /* next cache line - fields written by device */
RTE_MARKER cacheline1 __rte_cache_min_aligned;
- uint32_t cons_index; /* hw is done, can be freed */
+ volatile int32_t cons_index; /* hw is done, can be freed */
} __rte_cache_aligned;
/* Forward declarations */
-static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue,
- struct rte_mbuf *mbuf);
+static int eth_ark_tx_jumbo(struct ark_tx_queue *queue,
+ struct rte_mbuf *mbuf,
+ uint32_t *user_meta, uint8_t meta_cnt);
static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
static void free_completed_tx(struct ark_tx_queue *queue);
/* ************************************************************************* */
static inline void
-eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
- const struct rte_mbuf *mbuf,
- uint8_t flags)
+eth_ark_tx_desc_fill(struct ark_tx_queue *queue,
+ struct rte_mbuf *mbuf,
+ uint8_t flags,
+ uint32_t *user_meta,
+ uint8_t meta_cnt /* 0 to 5 */
+ )
{
- meta->physaddr = rte_mbuf_data_iova(mbuf);
- meta->user1 = (uint32_t)mbuf->udata64;
+ uint32_t tx_idx;
+ union ark_tx_meta *meta;
+ uint8_t m;
+
+ /* Header */
+ tx_idx = queue->prod_index & queue->queue_mask;
+ meta = &queue->meta_q[tx_idx];
meta->data_len = rte_pktmbuf_data_len(mbuf);
meta->flags = flags;
+ meta->meta_cnt = meta_cnt / 2;
+ meta->user1 = meta_cnt ? (*user_meta++) : 0;
+ queue->prod_index++;
+
+ queue->bufs[tx_idx] = mbuf;
+
+ /* 1 or 2 user meta data entries, user words 1,2 and 3,4 */
+ for (m = 1; m < meta_cnt; m += 2) {
+ tx_idx = queue->prod_index & queue->queue_mask;
+ meta = &queue->meta_q[tx_idx];
+ meta->usermeta0 = *user_meta++;
+ meta->usermeta1 = *user_meta++;
+ queue->prod_index++;
+ }
+
+ tx_idx = queue->prod_index & queue->queue_mask;
+ meta = &queue->meta_q[tx_idx];
+ meta->physaddr = rte_mbuf_data_iova(mbuf);
+ queue->prod_index++;
}
+
/* ************************************************************************* */
uint16_t
eth_ark_xmit_pkts_noop(void *vtxq __rte_unused,
{
struct ark_tx_queue *queue;
struct rte_mbuf *mbuf;
- struct ark_tx_meta *meta;
+ uint32_t user_meta[5];
- uint32_t idx;
- uint32_t prod_index_limit;
int stat;
+ int32_t prod_index_limit;
uint16_t nb;
+ uint8_t user_len = 0;
const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN;
+ tx_user_meta_hook_fn tx_user_meta_hook;
queue = (struct ark_tx_queue *)vtxq;
+ tx_user_meta_hook = queue->tx_user_meta_hook;
/* free any packets after the HW is done with them */
free_completed_tx(queue);
- prod_index_limit = queue->queue_size + queue->free_index;
+ /* leave 4 elements mpu data */
+ prod_index_limit = queue->queue_size + queue->free_index - 4;
for (nb = 0;
- (nb < nb_pkts) && (queue->prod_index != prod_index_limit);
+ (nb < nb_pkts) && (prod_index_limit - queue->prod_index) > 0;
++nb) {
mbuf = tx_pkts[nb];
memset(appended, 0, to_add);
}
+ if (tx_user_meta_hook)
+ tx_user_meta_hook(mbuf, user_meta, &user_len,
+ queue->ext_user_data);
if (unlikely(mbuf->nb_segs != 1)) {
- stat = eth_ark_tx_jumbo(queue, mbuf);
+ stat = eth_ark_tx_jumbo(queue, mbuf,
+ user_meta, user_len);
if (unlikely(stat != 0))
break; /* Queue is full */
} else {
- idx = queue->prod_index & queue->queue_mask;
- queue->bufs[idx] = mbuf;
- meta = &queue->meta_q[idx];
- eth_ark_tx_meta_from_mbuf(meta,
- mbuf,
- ARK_DDM_SOP |
- ARK_DDM_EOP);
- queue->prod_index++;
+ eth_ark_tx_desc_fill(queue, mbuf,
+ ARK_DDM_SOP | ARK_DDM_EOP,
+ user_meta, user_len);
}
}
}
/* ************************************************************************* */
-static uint32_t
-eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf)
+static int
+eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf,
+ uint32_t *user_meta, uint8_t meta_cnt)
{
struct rte_mbuf *next;
- struct ark_tx_meta *meta;
- uint32_t free_queue_space;
- uint32_t idx;
+ int32_t free_queue_space;
uint8_t flags = ARK_DDM_SOP;
free_queue_space = queue->queue_mask -
(queue->prod_index - queue->free_index);
- if (unlikely(free_queue_space < mbuf->nb_segs))
+ /* We need up to 4 mbufs for first header and 2 for subsequent ones */
+ if (unlikely(free_queue_space < (2 + (2 * mbuf->nb_segs))))
return -1;
while (mbuf != NULL) {
next = mbuf->next;
-
- idx = queue->prod_index & queue->queue_mask;
- queue->bufs[idx] = mbuf;
- meta = &queue->meta_q[idx];
-
flags |= (next == NULL) ? ARK_DDM_EOP : 0;
- eth_ark_tx_meta_from_mbuf(meta, mbuf, flags);
- queue->prod_index++;
+
+ eth_ark_tx_desc_fill(queue, mbuf, flags, user_meta, meta_cnt);
flags &= ~ARK_DDM_SOP; /* drop SOP flags */
+ meta_cnt = 0; /* Meta only on SOP */
mbuf = next;
}
return -1;
}
+ /* Each packet requires at least 2 mpu elements - double desc count */
+ nb_desc = 2 * nb_desc;
+
/* Allocate queue struct */
queue = rte_zmalloc_socket("Ark_txqueue",
sizeof(struct ark_tx_queue),
queue->phys_qid = qidx;
queue->queue_index = queue_idx;
dev->data->tx_queues[queue_idx] = queue;
+ queue->tx_user_meta_hook = ark->user_ext.tx_user_meta_hook;
+ queue->ext_user_data = ark->user_data[dev->data->port_id];
queue->meta_q =
rte_zmalloc_socket("Ark_txqueue meta",
- nb_desc * sizeof(struct ark_tx_meta),
+ nb_desc * sizeof(union ark_tx_meta),
64,
socket_id);
queue->bufs =
uint32_t write_interval_ns;
/* Verify HW -- MPU */
- if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta)))
+ if (ark_mpu_verify(queue->mpu, sizeof(union ark_tx_meta)))
return -1;
queue_base = rte_malloc_virt2iova(queue);
free_completed_tx(struct ark_tx_queue *queue)
{
struct rte_mbuf *mbuf;
- struct ark_tx_meta *meta;
- uint32_t top_index;
+ union ark_tx_meta *meta;
+ int32_t top_index;
top_index = queue->cons_index; /* read once */
- while (queue->free_index != top_index) {
+ while ((top_index - queue->free_index) > 0) {
meta = &queue->meta_q[queue->free_index & queue->queue_mask];
- mbuf = queue->bufs[queue->free_index & queue->queue_mask];
-
if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
+ mbuf = queue->bufs[queue->free_index &
+ queue->queue_mask];
/* ref count of the mbuf is checked in this call. */
rte_pktmbuf_free(mbuf);
}
- queue->free_index++;
+ queue->free_index += (meta->meta_cnt + 2);
}
}
git.droids-corp.org / dpdk.git / blobdiff