4 * Copyright (c) 2015-2017 Atomic Rules LLC
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of copyright holder nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include "ark_ethdev_tx.h"
37 #include "ark_global.h"
40 #include "ark_ethdev.h"
43 #define ARK_TX_META_SIZE 32
44 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE)
45 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM)
48 /* ************************************************************************* */
50 struct ark_tx_meta *meta_q;
51 struct rte_mbuf **bufs;
53 /* handles for hw objects */
54 struct ark_mpu_t *mpu;
55 struct ark_ddm_t *ddm;
57 /* Stats HW tracks bytes and packets, need to count send errors */
63 /* 3 indexes to the paired data rings. */
64 uint32_t prod_index; /* where to put the next one */
65 uint32_t free_index; /* mbuf has been freed */
67 /* The queue Id is used to identify the HW Q */
69 /* The queue Index within the dpdk device structures */
74 /* second cache line - fields only used in slow path */
75 MARKER cacheline1 __rte_cache_min_aligned;
76 uint32_t cons_index; /* hw is done, can be freed */
77 } __rte_cache_aligned;
79 /* Forward declarations */
80 static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue,
81 struct rte_mbuf *mbuf);
82 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
83 static void free_completed_tx(struct ark_tx_queue *queue);
86 ark_tx_hw_queue_stop(struct ark_tx_queue *queue)
88 ark_mpu_stop(queue->mpu);
91 /* ************************************************************************* */
93 eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
94 const struct rte_mbuf *mbuf,
97 meta->physaddr = rte_mbuf_data_dma_addr(mbuf);
99 meta->data_len = rte_pktmbuf_data_len(mbuf);
103 /* ************************************************************************* */
105 eth_ark_xmit_pkts_noop(void *vtxq __rte_unused,
106 struct rte_mbuf **tx_pkts __rte_unused,
107 uint16_t nb_pkts __rte_unused)
112 /* ************************************************************************* */
114 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
116 struct ark_tx_queue *queue;
117 struct rte_mbuf *mbuf;
118 struct ark_tx_meta *meta;
121 uint32_t prod_index_limit;
125 queue = (struct ark_tx_queue *)vtxq;
127 /* free any packets after the HW is done with them */
128 free_completed_tx(queue);
130 prod_index_limit = queue->queue_size + queue->free_index;
133 (nb < nb_pkts) && (queue->prod_index != prod_index_limit);
137 if (ARK_TX_PAD_TO_60) {
138 if (unlikely(rte_pktmbuf_pkt_len(mbuf) < 60)) {
139 /* this packet even if it is small can be split,
140 * be sure to add to the end mbuf
143 60 - rte_pktmbuf_pkt_len(mbuf);
145 rte_pktmbuf_append(mbuf, to_add);
148 /* This packet is in error,
149 * we cannot send it so just
150 * count it and delete it.
152 queue->tx_errors += 1;
153 rte_pktmbuf_free(mbuf);
156 memset(appended, 0, to_add);
160 if (unlikely(mbuf->nb_segs != 1)) {
161 stat = eth_ark_tx_jumbo(queue, mbuf);
162 if (unlikely(stat != 0))
163 break; /* Queue is full */
165 idx = queue->prod_index & queue->queue_mask;
166 queue->bufs[idx] = mbuf;
167 meta = &queue->meta_q[idx];
168 eth_ark_tx_meta_from_mbuf(meta,
176 if (ARK_TX_DEBUG && (nb != nb_pkts)) {
177 PMD_TX_LOG(DEBUG, "TX: Failure to send:"
182 " free: %" PRIU32 "\n",
187 ark_mpu_dump(queue->mpu,
192 /* let FPGA know producer index. */
194 ark_mpu_set_producer(queue->mpu, queue->prod_index);
199 /* ************************************************************************* */
201 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf)
203 struct rte_mbuf *next;
204 struct ark_tx_meta *meta;
205 uint32_t free_queue_space;
207 uint8_t flags = ARK_DDM_SOP;
209 free_queue_space = queue->queue_mask -
210 (queue->prod_index - queue->free_index);
211 if (unlikely(free_queue_space < mbuf->nb_segs))
214 while (mbuf != NULL) {
217 idx = queue->prod_index & queue->queue_mask;
218 queue->bufs[idx] = mbuf;
219 meta = &queue->meta_q[idx];
221 flags |= (next == NULL) ? ARK_DDM_EOP : 0;
222 eth_ark_tx_meta_from_mbuf(meta, mbuf, flags);
225 flags &= ~ARK_DDM_SOP; /* drop SOP flags */
232 /* ************************************************************************* */
234 eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
237 unsigned int socket_id,
238 const struct rte_eth_txconf *tx_conf __rte_unused)
240 struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private;
241 struct ark_tx_queue *queue;
244 /* Future: divide the Q's evenly with multi-ports */
245 int port = dev->data->port_id;
246 int qidx = port + queue_idx;
248 if (!rte_is_power_of_2(nb_desc)) {
250 "DPDK Arkville configuration queue size"
251 " must be power of two %u (%s)\n",
256 /* Allocate queue struct */
257 queue = rte_zmalloc_socket("Ark_txqueue",
258 sizeof(struct ark_tx_queue),
262 PMD_DRV_LOG(ERR, "Failed to allocate tx "
263 "queue memory in %s\n",
268 /* we use zmalloc no need to initialize fields */
269 queue->queue_size = nb_desc;
270 queue->queue_mask = nb_desc - 1;
271 queue->phys_qid = qidx;
272 queue->queue_index = queue_idx;
273 dev->data->tx_queues[queue_idx] = queue;
276 rte_zmalloc_socket("Ark_txqueue meta",
277 nb_desc * sizeof(struct ark_tx_meta),
281 rte_zmalloc_socket("Ark_txqueue bufs",
282 nb_desc * sizeof(struct rte_mbuf *),
286 if (queue->meta_q == 0 || queue->bufs == 0) {
287 PMD_DRV_LOG(ERR, "Failed to allocate "
288 "queue memory in %s\n", __func__);
289 rte_free(queue->meta_q);
290 rte_free(queue->bufs);
295 queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET);
296 queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET);
298 status = eth_ark_tx_hw_queue_config(queue);
300 if (unlikely(status != 0)) {
301 rte_free(queue->meta_q);
302 rte_free(queue->bufs);
304 return -1; /* ERROR CODE */
310 /* ************************************************************************* */
312 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)
314 phys_addr_t queue_base, ring_base, cons_index_addr;
315 uint32_t write_interval_ns;
317 /* Verify HW -- MPU */
318 if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta)))
321 queue_base = rte_malloc_virt2phy(queue);
322 ring_base = rte_malloc_virt2phy(queue->meta_q);
324 queue_base + offsetof(struct ark_tx_queue, cons_index);
326 ark_mpu_stop(queue->mpu);
327 ark_mpu_reset(queue->mpu);
329 /* Stop and Reset and configure MPU */
330 ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1);
333 * Adjust the write interval based on queue size --
334 * increase pcie traffic when low mbuf count
335 * Queue sizes less than 128 are not allowed
337 switch (queue->queue_size) {
339 write_interval_ns = 500;
342 write_interval_ns = 500;
345 write_interval_ns = 1000;
348 write_interval_ns = 2000;
352 /* Completion address in UDM */
353 ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns);
358 /* ************************************************************************* */
360 eth_ark_tx_queue_release(void *vtx_queue)
362 struct ark_tx_queue *queue;
364 queue = (struct ark_tx_queue *)vtx_queue;
366 ark_tx_hw_queue_stop(queue);
368 queue->cons_index = queue->prod_index;
369 free_completed_tx(queue);
371 rte_free(queue->meta_q);
372 rte_free(queue->bufs);
376 /* ************************************************************************* */
378 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
380 struct ark_tx_queue *queue;
383 queue = dev->data->tx_queues[queue_id];
385 /* Wait for DDM to send out all packets. */
386 while (queue->cons_index != queue->prod_index) {
392 ark_mpu_stop(queue->mpu);
393 free_completed_tx(queue);
395 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
401 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
403 struct ark_tx_queue *queue;
405 queue = dev->data->tx_queues[queue_id];
406 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED)
409 ark_mpu_start(queue->mpu);
410 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
415 /* ************************************************************************* */
417 free_completed_tx(struct ark_tx_queue *queue)
419 struct rte_mbuf *mbuf;
420 struct ark_tx_meta *meta;
423 top_index = queue->cons_index; /* read once */
424 while (queue->free_index != top_index) {
425 meta = &queue->meta_q[queue->free_index & queue->queue_mask];
426 mbuf = queue->bufs[queue->free_index & queue->queue_mask];
428 if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
429 /* ref count of the mbuf is checked in this call. */
430 rte_pktmbuf_free(mbuf);
436 /* ************************************************************************* */
438 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats)
440 struct ark_tx_queue *queue;
441 struct ark_ddm_t *ddm;
442 uint64_t bytes, pkts;
447 bytes = ark_ddm_queue_byte_count(ddm);
448 pkts = ark_ddm_queue_pkt_count(ddm);
450 stats->q_opackets[queue->queue_index] = pkts;
451 stats->q_obytes[queue->queue_index] = bytes;
452 stats->opackets += pkts;
453 stats->obytes += bytes;
454 stats->oerrors += queue->tx_errors;
458 eth_tx_queue_stats_reset(void *vqueue)
460 struct ark_tx_queue *queue;
461 struct ark_ddm_t *ddm;
466 ark_ddm_queue_reset_stats(ddm);
467 queue->tx_errors = 0;