4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Cavium, Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <rte_ethdev.h>
35 #include <rte_cycles.h>
36 #include <rte_malloc.h>
39 #include "lio_struct.h"
40 #include "lio_ethdev.h"
44 /* Flush iq if available tx_desc fall below LIO_FLUSH_WM */
45 #define LIO_FLUSH_WM(_iq) ((_iq)->max_count / 2)
46 #define LIO_PKT_IN_DONE_CNT_MASK 0x00000000FFFFFFFFULL
49 lio_droq_compute_max_packet_bufs(struct lio_droq *droq)
54 count += droq->buffer_size;
55 } while (count < LIO_MAX_RX_PKTLEN);
59 lio_droq_reset_indices(struct lio_droq *droq)
64 droq->refill_count = 0;
65 rte_atomic64_set(&droq->pkts_pending, 0);
69 lio_droq_destroy_ring_buffers(struct lio_droq *droq)
73 for (i = 0; i < droq->max_count; i++) {
74 if (droq->recv_buf_list[i].buffer) {
75 rte_pktmbuf_free((struct rte_mbuf *)
76 droq->recv_buf_list[i].buffer);
77 droq->recv_buf_list[i].buffer = NULL;
81 lio_droq_reset_indices(droq);
85 lio_recv_buffer_alloc(struct lio_device *lio_dev, int q_no)
87 struct lio_droq *droq = lio_dev->droq[q_no];
88 struct rte_mempool *mpool = droq->mpool;
91 m = rte_pktmbuf_alloc(mpool);
93 lio_dev_err(lio_dev, "Cannot allocate\n");
97 rte_mbuf_refcnt_set(m, 1);
99 m->data_off = RTE_PKTMBUF_HEADROOM;
107 lio_droq_setup_ring_buffers(struct lio_device *lio_dev,
108 struct lio_droq *droq)
110 struct lio_droq_desc *desc_ring = droq->desc_ring;
114 for (i = 0; i < droq->max_count; i++) {
115 buf = lio_recv_buffer_alloc(lio_dev, droq->q_no);
117 lio_dev_err(lio_dev, "buffer alloc failed\n");
118 droq->stats.rx_alloc_failure++;
119 lio_droq_destroy_ring_buffers(droq);
123 droq->recv_buf_list[i].buffer = buf;
124 droq->info_list[i].length = 0;
126 /* map ring buffers into memory */
127 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
128 desc_ring[i].buffer_ptr =
129 lio_map_ring(droq->recv_buf_list[i].buffer);
132 lio_droq_reset_indices(droq);
134 lio_droq_compute_max_packet_bufs(droq);
140 lio_dma_zone_free(struct lio_device *lio_dev, const struct rte_memzone *mz)
142 const struct rte_memzone *mz_tmp;
146 lio_dev_err(lio_dev, "Memzone NULL\n");
150 mz_tmp = rte_memzone_lookup(mz->name);
151 if (mz_tmp == NULL) {
152 lio_dev_err(lio_dev, "Memzone %s Not Found\n", mz->name);
156 ret = rte_memzone_free(mz);
158 lio_dev_err(lio_dev, "Memzone free Failed ret %d\n", ret);
162 * Frees the space for descriptor ring for the droq.
164 * @param lio_dev - pointer to the lio device structure
165 * @param q_no - droq no.
168 lio_delete_droq(struct lio_device *lio_dev, uint32_t q_no)
170 struct lio_droq *droq = lio_dev->droq[q_no];
172 lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no);
174 lio_droq_destroy_ring_buffers(droq);
175 rte_free(droq->recv_buf_list);
176 droq->recv_buf_list = NULL;
177 lio_dma_zone_free(lio_dev, droq->info_mz);
178 lio_dma_zone_free(lio_dev, droq->desc_ring_mz);
180 memset(droq, 0, LIO_DROQ_SIZE);
184 lio_alloc_info_buffer(struct lio_device *lio_dev,
185 struct lio_droq *droq, unsigned int socket_id)
187 droq->info_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev,
188 "info_list", droq->q_no,
194 if (droq->info_mz == NULL)
197 droq->info_list_dma = droq->info_mz->phys_addr;
198 droq->info_alloc_size = droq->info_mz->len;
199 droq->info_base_addr = (size_t)droq->info_mz->addr;
201 return droq->info_mz->addr;
205 * Allocates space for the descriptor ring for the droq and
206 * sets the base addr, num desc etc in Octeon registers.
208 * @param lio_dev - pointer to the lio device structure
209 * @param q_no - droq no.
210 * @param app_ctx - pointer to application context
211 * @return Success: 0 Failure: -1
214 lio_init_droq(struct lio_device *lio_dev, uint32_t q_no,
215 uint32_t num_descs, uint32_t desc_size,
216 struct rte_mempool *mpool, unsigned int socket_id)
218 uint32_t c_refill_threshold;
219 uint32_t desc_ring_size;
220 struct lio_droq *droq;
222 lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no);
224 droq = lio_dev->droq[q_no];
225 droq->lio_dev = lio_dev;
229 c_refill_threshold = LIO_OQ_REFILL_THRESHOLD_CFG(lio_dev);
231 droq->max_count = num_descs;
232 droq->buffer_size = desc_size;
234 desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE;
235 droq->desc_ring_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev,
241 if (droq->desc_ring_mz == NULL) {
243 "Output queue %d ring alloc failed\n", q_no);
247 droq->desc_ring_dma = droq->desc_ring_mz->phys_addr;
248 droq->desc_ring = (struct lio_droq_desc *)droq->desc_ring_mz->addr;
250 lio_dev_dbg(lio_dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
251 q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma);
252 lio_dev_dbg(lio_dev, "droq[%d]: num_desc: %d\n", q_no,
255 droq->info_list = lio_alloc_info_buffer(lio_dev, droq, socket_id);
256 if (droq->info_list == NULL) {
257 lio_dev_err(lio_dev, "Cannot allocate memory for info list.\n");
261 droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list",
263 LIO_DROQ_RECVBUF_SIZE),
266 if (droq->recv_buf_list == NULL) {
268 "Output queue recv buf list alloc failed\n");
272 if (lio_droq_setup_ring_buffers(lio_dev, droq))
275 droq->refill_threshold = c_refill_threshold;
277 rte_spinlock_init(&droq->lock);
279 lio_dev->fn_list.setup_oq_regs(lio_dev, q_no);
281 lio_dev->io_qmask.oq |= (1ULL << q_no);
286 lio_delete_droq(lio_dev, q_no);
292 lio_setup_droq(struct lio_device *lio_dev, int oq_no, int num_descs,
293 int desc_size, struct rte_mempool *mpool, unsigned int socket_id)
295 struct lio_droq *droq;
297 PMD_INIT_FUNC_TRACE();
299 if (lio_dev->droq[oq_no]) {
300 lio_dev_dbg(lio_dev, "Droq %d in use\n", oq_no);
304 /* Allocate the DS for the new droq. */
305 droq = rte_zmalloc_socket("ethdev RX queue", sizeof(*droq),
306 RTE_CACHE_LINE_SIZE, socket_id);
310 lio_dev->droq[oq_no] = droq;
312 /* Initialize the Droq */
313 if (lio_init_droq(lio_dev, oq_no, num_descs, desc_size, mpool,
315 lio_dev_err(lio_dev, "Droq[%u] Initialization Failed\n", oq_no);
316 rte_free(lio_dev->droq[oq_no]);
317 lio_dev->droq[oq_no] = NULL;
323 lio_dev_dbg(lio_dev, "Total number of OQ: %d\n", lio_dev->num_oqs);
325 /* Send credit for octeon output queues. credits are always
326 * sent after the output queue is enabled.
328 rte_write32(lio_dev->droq[oq_no]->max_count,
329 lio_dev->droq[oq_no]->pkts_credit_reg);
335 static inline uint32_t
336 lio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len)
338 uint32_t buf_cnt = 0;
340 while (total_len > (buf_size * buf_cnt))
346 /* If we were not able to refill all buffers, try to move around
347 * the buffers that were not dispatched.
349 static inline uint32_t
350 lio_droq_refill_pullup_descs(struct lio_droq *droq,
351 struct lio_droq_desc *desc_ring)
353 uint32_t refill_index = droq->refill_idx;
354 uint32_t desc_refilled = 0;
356 while (refill_index != droq->read_idx) {
357 if (droq->recv_buf_list[refill_index].buffer) {
358 droq->recv_buf_list[droq->refill_idx].buffer =
359 droq->recv_buf_list[refill_index].buffer;
360 desc_ring[droq->refill_idx].buffer_ptr =
361 desc_ring[refill_index].buffer_ptr;
362 droq->recv_buf_list[refill_index].buffer = NULL;
363 desc_ring[refill_index].buffer_ptr = 0;
365 droq->refill_idx = lio_incr_index(
369 droq->refill_count--;
370 } while (droq->recv_buf_list[droq->refill_idx].buffer);
372 refill_index = lio_incr_index(refill_index, 1,
376 return desc_refilled;
381 * @param lio_dev - pointer to the lio device structure
382 * @param droq - droq in which descriptors require new buffers.
385 * Called during normal DROQ processing in interrupt mode or by the poll
386 * thread to refill the descriptors from which buffers were dispatched
387 * to upper layers. Attempts to allocate new buffers. If that fails, moves
388 * up buffers (that were not dispatched) to form a contiguous ring.
391 * No of descriptors refilled.
394 * This routine is called with droq->lock held.
397 lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq)
399 struct lio_droq_desc *desc_ring;
400 uint32_t desc_refilled = 0;
403 desc_ring = droq->desc_ring;
405 while (droq->refill_count && (desc_refilled < droq->max_count)) {
406 /* If a valid buffer exists (happens if there is no dispatch),
407 * reuse the buffer, else allocate.
409 if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) {
410 buf = lio_recv_buffer_alloc(lio_dev, droq->q_no);
411 /* If a buffer could not be allocated, no point in
415 droq->stats.rx_alloc_failure++;
419 droq->recv_buf_list[droq->refill_idx].buffer = buf;
422 desc_ring[droq->refill_idx].buffer_ptr =
423 lio_map_ring(droq->recv_buf_list[droq->refill_idx].buffer);
424 /* Reset any previous values in the length field. */
425 droq->info_list[droq->refill_idx].length = 0;
427 droq->refill_idx = lio_incr_index(droq->refill_idx, 1,
430 droq->refill_count--;
433 if (droq->refill_count)
434 desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring);
436 /* if droq->refill_count
437 * The refill count would not change in pass two. We only moved buffers
438 * to close the gap in the ring, but we would still have the same no. of
441 return desc_refilled;
445 lio_droq_fast_process_packet(struct lio_device *lio_dev,
446 struct lio_droq *droq,
447 struct rte_mbuf **rx_pkts)
449 struct rte_mbuf *nicbuf = NULL;
450 struct lio_droq_info *info;
451 uint32_t total_len = 0;
452 int data_total_len = 0;
453 uint32_t pkt_len = 0;
457 info = &droq->info_list[droq->read_idx];
458 lio_swap_8B_data((uint64_t *)info, 2);
463 /* Len of resp hdr in included in the received data len. */
464 info->length -= OCTEON_RH_SIZE;
467 total_len += (uint32_t)info->length;
469 if (lio_opcode_slow_path(rh)) {
472 buf_cnt = lio_droq_get_bufcount(droq->buffer_size,
473 (uint32_t)info->length);
474 droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt,
476 droq->refill_count += buf_cnt;
478 if (info->length <= droq->buffer_size) {
479 if (rh->r_dh.has_hash)
480 pkt_len = (uint32_t)(info->length - 8);
482 pkt_len = (uint32_t)info->length;
484 nicbuf = droq->recv_buf_list[droq->read_idx].buffer;
485 droq->recv_buf_list[droq->read_idx].buffer = NULL;
486 droq->read_idx = lio_incr_index(
489 droq->refill_count++;
491 if (likely(nicbuf != NULL)) {
492 nicbuf->data_off = RTE_PKTMBUF_HEADROOM;
495 /* We don't have a way to pass flags yet */
496 nicbuf->ol_flags = 0;
497 if (rh->r_dh.has_hash) {
500 nicbuf->ol_flags |= PKT_RX_RSS_HASH;
501 hash_ptr = rte_pktmbuf_mtod(nicbuf,
503 lio_swap_8B_data(hash_ptr, 1);
504 nicbuf->hash.rss = (uint32_t)*hash_ptr;
505 nicbuf->data_off += 8;
508 nicbuf->pkt_len = pkt_len;
509 nicbuf->data_len = pkt_len;
510 nicbuf->port = lio_dev->port_id;
512 rx_pkts[data_pkts++] = nicbuf;
513 data_total_len += pkt_len;
516 /* Prefetch buffer pointers when on a cache line
519 if ((droq->read_idx & 3) == 0) {
521 &droq->recv_buf_list[droq->read_idx]);
523 &droq->info_list[droq->read_idx]);
526 struct rte_mbuf *first_buf = NULL;
527 struct rte_mbuf *last_buf = NULL;
529 while (pkt_len < info->length) {
532 cpy_len = ((pkt_len + droq->buffer_size) >
534 ? ((uint32_t)info->length -
539 droq->recv_buf_list[droq->read_idx].buffer;
540 droq->recv_buf_list[droq->read_idx].buffer =
543 if (likely(nicbuf != NULL)) {
544 /* Note the first seg */
548 nicbuf->data_off = RTE_PKTMBUF_HEADROOM;
551 nicbuf->port = lio_dev->port_id;
552 /* We don't have a way to pass
555 nicbuf->ol_flags = 0;
556 if ((!pkt_len) && (rh->r_dh.has_hash)) {
561 hash_ptr = rte_pktmbuf_mtod(
563 lio_swap_8B_data(hash_ptr, 1);
566 nicbuf->data_off += 8;
567 nicbuf->pkt_len = cpy_len - 8;
568 nicbuf->data_len = cpy_len - 8;
570 nicbuf->pkt_len = cpy_len;
571 nicbuf->data_len = cpy_len;
575 first_buf->nb_segs++;
578 last_buf->next = nicbuf;
582 PMD_RX_LOG(lio_dev, ERR, "no buf\n");
586 droq->read_idx = lio_incr_index(
589 droq->refill_count++;
591 /* Prefetch buffer pointers when on a
592 * cache line boundary
594 if ((droq->read_idx & 3) == 0) {
595 rte_prefetch0(&droq->recv_buf_list
599 &droq->info_list[droq->read_idx]);
602 rx_pkts[data_pkts++] = first_buf;
603 if (rh->r_dh.has_hash)
604 data_total_len += (pkt_len - 8);
606 data_total_len += pkt_len;
609 /* Inform upper layer about packet checksum verification */
610 struct rte_mbuf *m = rx_pkts[data_pkts - 1];
612 if (rh->r_dh.csum_verified & LIO_IP_CSUM_VERIFIED)
613 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
615 if (rh->r_dh.csum_verified & LIO_L4_CSUM_VERIFIED)
616 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
619 if (droq->refill_count >= droq->refill_threshold) {
620 int desc_refilled = lio_droq_refill(lio_dev, droq);
622 /* Flush the droq descriptor data to memory to be sure
623 * that when we update the credits the data in memory is
627 rte_write32(desc_refilled, droq->pkts_credit_reg);
628 /* make sure mmio write completes */
635 droq->stats.pkts_received++;
636 droq->stats.rx_pkts_received += data_pkts;
637 droq->stats.rx_bytes_received += data_total_len;
638 droq->stats.bytes_received += total_len;
644 lio_droq_fast_process_packets(struct lio_device *lio_dev,
645 struct lio_droq *droq,
646 struct rte_mbuf **rx_pkts,
647 uint32_t pkts_to_process)
649 int ret, data_pkts = 0;
652 for (pkt = 0; pkt < pkts_to_process; pkt++) {
653 ret = lio_droq_fast_process_packet(lio_dev, droq,
654 &rx_pkts[data_pkts]);
656 lio_dev_err(lio_dev, "Port[%d] DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
657 lio_dev->port_id, droq->q_no,
658 droq->read_idx, pkts_to_process);
664 rte_atomic64_sub(&droq->pkts_pending, pkt);
669 static inline uint32_t
670 lio_droq_check_hw_for_pkts(struct lio_droq *droq)
675 pkt_count = rte_read32(droq->pkts_sent_reg);
677 last_count = pkt_count - droq->pkt_count;
678 droq->pkt_count = pkt_count;
681 rte_atomic64_add(&droq->pkts_pending, last_count);
687 lio_dev_recv_pkts(void *rx_queue,
688 struct rte_mbuf **rx_pkts,
691 struct lio_droq *droq = rx_queue;
692 struct lio_device *lio_dev = droq->lio_dev;
693 uint32_t pkts_processed = 0;
694 uint32_t pkt_count = 0;
696 lio_droq_check_hw_for_pkts(droq);
698 pkt_count = rte_atomic64_read(&droq->pkts_pending);
702 if (pkt_count > budget)
706 rte_spinlock_lock(&droq->lock);
707 pkts_processed = lio_droq_fast_process_packets(lio_dev,
711 if (droq->pkt_count) {
712 rte_write32(droq->pkt_count, droq->pkts_sent_reg);
716 /* Release the spin lock */
717 rte_spinlock_unlock(&droq->lock);
719 return pkts_processed;
723 lio_delete_droq_queue(struct lio_device *lio_dev,
726 lio_delete_droq(lio_dev, oq_no);
728 rte_free(lio_dev->droq[oq_no]);
729 lio_dev->droq[oq_no] = NULL;
733 * lio_init_instr_queue()
734 * @param lio_dev - pointer to the lio device structure.
735 * @param txpciq - queue to be initialized.
737 * Called at driver init time for each input queue. iq_conf has the
738 * configuration parameters for the queue.
740 * @return Success: 0 Failure: -1
743 lio_init_instr_queue(struct lio_device *lio_dev,
744 union octeon_txpciq txpciq,
745 uint32_t num_descs, unsigned int socket_id)
747 uint32_t iq_no = (uint32_t)txpciq.s.q_no;
748 struct lio_instr_queue *iq;
752 instr_type = LIO_IQ_INSTR_TYPE(lio_dev);
754 q_size = instr_type * num_descs;
755 iq = lio_dev->instr_queue[iq_no];
756 iq->iq_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev,
757 "instr_queue", iq_no, q_size,
760 if (iq->iq_mz == NULL) {
761 lio_dev_err(lio_dev, "Cannot allocate memory for instr queue %d\n",
766 iq->base_addr_dma = iq->iq_mz->phys_addr;
767 iq->base_addr = (uint8_t *)iq->iq_mz->addr;
769 iq->max_count = num_descs;
771 /* Initialize a list to holds requests that have been posted to Octeon
772 * but has yet to be fetched by octeon
774 iq->request_list = rte_zmalloc_socket("request_list",
775 sizeof(*iq->request_list) *
779 if (iq->request_list == NULL) {
780 lio_dev_err(lio_dev, "Alloc failed for IQ[%d] nr free list\n",
782 lio_dma_zone_free(lio_dev, iq->iq_mz);
786 lio_dev_dbg(lio_dev, "IQ[%d]: base: %p basedma: %lx count: %d\n",
787 iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma,
790 iq->lio_dev = lio_dev;
791 iq->txpciq.txpciq64 = txpciq.txpciq64;
793 iq->host_write_index = 0;
794 iq->lio_read_index = 0;
797 rte_atomic64_set(&iq->instr_pending, 0);
799 /* Initialize the spinlock for this instruction queue */
800 rte_spinlock_init(&iq->lock);
801 rte_spinlock_init(&iq->post_lock);
803 rte_atomic64_clear(&iq->iq_flush_running);
805 lio_dev->io_qmask.iq |= (1ULL << iq_no);
807 /* Set the 32B/64B mode for each input queue */
808 lio_dev->io_qmask.iq64B |= ((instr_type == 64) << iq_no);
809 iq->iqcmd_64B = (instr_type == 64);
811 lio_dev->fn_list.setup_iq_regs(lio_dev, iq_no);
817 lio_setup_instr_queue0(struct lio_device *lio_dev)
819 union octeon_txpciq txpciq;
820 uint32_t num_descs = 0;
823 num_descs = LIO_NUM_DEF_TX_DESCS_CFG(lio_dev);
825 lio_dev->num_iqs = 0;
827 lio_dev->instr_queue[0] = rte_zmalloc(NULL,
828 sizeof(struct lio_instr_queue), 0);
829 if (lio_dev->instr_queue[0] == NULL)
832 lio_dev->instr_queue[0]->q_index = 0;
833 lio_dev->instr_queue[0]->app_ctx = (void *)(size_t)0;
835 txpciq.s.q_no = iq_no;
836 txpciq.s.pkind = lio_dev->pfvf_hsword.pkind;
837 txpciq.s.use_qpg = 0;
839 if (lio_init_instr_queue(lio_dev, txpciq, num_descs, SOCKET_ID_ANY)) {
840 rte_free(lio_dev->instr_queue[0]);
841 lio_dev->instr_queue[0] = NULL;
851 * lio_delete_instr_queue()
852 * @param lio_dev - pointer to the lio device structure.
853 * @param iq_no - queue to be deleted.
855 * Called at driver unload time for each input queue. Deletes all
856 * allocated resources for the input queue.
859 lio_delete_instr_queue(struct lio_device *lio_dev, uint32_t iq_no)
861 struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
863 rte_free(iq->request_list);
864 iq->request_list = NULL;
865 lio_dma_zone_free(lio_dev, iq->iq_mz);
869 lio_free_instr_queue0(struct lio_device *lio_dev)
871 lio_delete_instr_queue(lio_dev, 0);
872 rte_free(lio_dev->instr_queue[0]);
873 lio_dev->instr_queue[0] = NULL;
877 /* Return 0 on success, -1 on failure */
879 lio_setup_iq(struct lio_device *lio_dev, int q_index,
880 union octeon_txpciq txpciq, uint32_t num_descs, void *app_ctx,
881 unsigned int socket_id)
883 uint32_t iq_no = (uint32_t)txpciq.s.q_no;
885 if (lio_dev->instr_queue[iq_no]) {
886 lio_dev_dbg(lio_dev, "IQ is in use. Cannot create the IQ: %d again\n",
888 lio_dev->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64;
889 lio_dev->instr_queue[iq_no]->app_ctx = app_ctx;
893 lio_dev->instr_queue[iq_no] = rte_zmalloc_socket("ethdev TX queue",
894 sizeof(struct lio_instr_queue),
895 RTE_CACHE_LINE_SIZE, socket_id);
896 if (lio_dev->instr_queue[iq_no] == NULL)
899 lio_dev->instr_queue[iq_no]->q_index = q_index;
900 lio_dev->instr_queue[iq_no]->app_ctx = app_ctx;
902 if (lio_init_instr_queue(lio_dev, txpciq, num_descs, socket_id))
906 if (lio_dev->fn_list.enable_io_queues(lio_dev))
912 lio_delete_instr_queue(lio_dev, iq_no);
915 rte_free(lio_dev->instr_queue[iq_no]);
916 lio_dev->instr_queue[iq_no] = NULL;
922 lio_wait_for_instr_fetch(struct lio_device *lio_dev)
924 int pending, instr_cnt;
930 for (i = 0; i < LIO_MAX_INSTR_QUEUES(lio_dev); i++) {
931 if (!(lio_dev->io_qmask.iq & (1ULL << i)))
934 if (lio_dev->instr_queue[i] == NULL)
937 pending = rte_atomic64_read(
938 &lio_dev->instr_queue[i]->instr_pending);
940 lio_flush_iq(lio_dev, lio_dev->instr_queue[i]);
942 instr_cnt += pending;
950 } while (retry-- && instr_cnt);
956 lio_ring_doorbell(struct lio_device *lio_dev,
957 struct lio_instr_queue *iq)
959 if (rte_atomic64_read(&lio_dev->status) == LIO_DEV_RUNNING) {
960 rte_write32(iq->fill_cnt, iq->doorbell_reg);
961 /* make sure doorbell write goes through */
968 copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd)
970 uint8_t *iqptr, cmdsize;
972 cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
973 iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
975 rte_memcpy(iqptr, cmd, cmdsize);
978 static inline struct lio_iq_post_status
979 post_command2(struct lio_instr_queue *iq, uint8_t *cmd)
981 struct lio_iq_post_status st;
983 st.status = LIO_IQ_SEND_OK;
985 /* This ensures that the read index does not wrap around to the same
986 * position if queue gets full before Octeon could fetch any instr.
988 if (rte_atomic64_read(&iq->instr_pending) >=
989 (int32_t)(iq->max_count - 1)) {
990 st.status = LIO_IQ_SEND_FAILED;
995 if (rte_atomic64_read(&iq->instr_pending) >=
996 (int32_t)(iq->max_count - 2))
997 st.status = LIO_IQ_SEND_STOP;
999 copy_cmd_into_iq(iq, cmd);
1001 /* "index" is returned, host_write_index is modified. */
1002 st.index = iq->host_write_index;
1003 iq->host_write_index = lio_incr_index(iq->host_write_index, 1,
1007 /* Flush the command into memory. We need to be sure the data is in
1008 * memory before indicating that the instruction is pending.
1012 rte_atomic64_inc(&iq->instr_pending);
1018 lio_add_to_request_list(struct lio_instr_queue *iq,
1019 int idx, void *buf, int reqtype)
1021 iq->request_list[idx].buf = buf;
1022 iq->request_list[idx].reqtype = reqtype;
1026 lio_free_netsgbuf(void *buf)
1028 struct lio_buf_free_info *finfo = buf;
1029 struct lio_device *lio_dev = finfo->lio_dev;
1030 struct rte_mbuf *m = finfo->mbuf;
1031 struct lio_gather *g = finfo->g;
1032 uint8_t iq = finfo->iq_no;
1034 /* This will take care of multiple segments also */
1035 rte_pktmbuf_free(m);
1037 rte_spinlock_lock(&lio_dev->glist_lock[iq]);
1038 STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq], &g->list, entries);
1039 rte_spinlock_unlock(&lio_dev->glist_lock[iq]);
1043 /* Can only run in process context */
1045 lio_process_iq_request_list(struct lio_device *lio_dev,
1046 struct lio_instr_queue *iq)
1048 struct octeon_instr_irh *irh = NULL;
1049 uint32_t old = iq->flush_index;
1050 struct lio_soft_command *sc;
1051 uint32_t inst_count = 0;
1055 while (old != iq->lio_read_index) {
1056 reqtype = iq->request_list[old].reqtype;
1057 buf = iq->request_list[old].buf;
1059 if (reqtype == LIO_REQTYPE_NONE)
1063 case LIO_REQTYPE_NORESP_NET:
1064 rte_pktmbuf_free((struct rte_mbuf *)buf);
1066 case LIO_REQTYPE_NORESP_NET_SG:
1067 lio_free_netsgbuf(buf);
1069 case LIO_REQTYPE_SOFT_COMMAND:
1071 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
1073 /* We're expecting a response from Octeon.
1074 * It's up to lio_process_ordered_list() to
1075 * process sc. Add sc to the ordered soft
1076 * command response list because we expect
1077 * a response from Octeon.
1079 rte_spinlock_lock(&lio_dev->response_list.lock);
1081 &lio_dev->response_list.pending_req_count);
1083 &lio_dev->response_list.head,
1084 &sc->node, entries);
1085 rte_spinlock_unlock(
1086 &lio_dev->response_list.lock);
1089 /* This callback must not sleep */
1090 sc->callback(LIO_REQUEST_DONE,
1096 lio_dev_err(lio_dev,
1097 "Unknown reqtype: %d buf: %p at idx %d\n",
1101 iq->request_list[old].buf = NULL;
1102 iq->request_list[old].reqtype = 0;
1106 old = lio_incr_index(old, 1, iq->max_count);
1109 iq->flush_index = old;
1115 lio_update_read_index(struct lio_instr_queue *iq)
1117 uint32_t pkt_in_done = rte_read32(iq->inst_cnt_reg);
1120 last_done = pkt_in_done - iq->pkt_in_done;
1121 iq->pkt_in_done = pkt_in_done;
1123 /* Add last_done and modulo with the IQ size to get new index */
1124 iq->lio_read_index = (iq->lio_read_index +
1125 (uint32_t)(last_done & LIO_PKT_IN_DONE_CNT_MASK)) %
1130 lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq)
1132 uint32_t tot_inst_processed = 0;
1133 uint32_t inst_processed = 0;
1136 if (rte_atomic64_test_and_set(&iq->iq_flush_running) == 0)
1139 rte_spinlock_lock(&iq->lock);
1141 lio_update_read_index(iq);
1144 /* Process any outstanding IQ packets. */
1145 if (iq->flush_index == iq->lio_read_index)
1148 inst_processed = lio_process_iq_request_list(lio_dev, iq);
1150 if (inst_processed) {
1151 rte_atomic64_sub(&iq->instr_pending, inst_processed);
1152 iq->stats.instr_processed += inst_processed;
1155 tot_inst_processed += inst_processed;
1160 rte_spinlock_unlock(&iq->lock);
1162 rte_atomic64_clear(&iq->iq_flush_running);
1168 lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd,
1169 void *buf, uint32_t datasize, uint32_t reqtype)
1171 struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
1172 struct lio_iq_post_status st;
1174 rte_spinlock_lock(&iq->post_lock);
1176 st = post_command2(iq, cmd);
1178 if (st.status != LIO_IQ_SEND_FAILED) {
1179 lio_add_to_request_list(iq, st.index, buf, reqtype);
1180 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, bytes_sent,
1182 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_posted, 1);
1184 lio_ring_doorbell(lio_dev, iq);
1186 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_dropped, 1);
1189 rte_spinlock_unlock(&iq->post_lock);
1195 lio_prepare_soft_command(struct lio_device *lio_dev,
1196 struct lio_soft_command *sc, uint8_t opcode,
1197 uint8_t subcode, uint32_t irh_ossp, uint64_t ossp0,
1200 struct octeon_instr_pki_ih3 *pki_ih3;
1201 struct octeon_instr_ih3 *ih3;
1202 struct octeon_instr_irh *irh;
1203 struct octeon_instr_rdp *rdp;
1205 RTE_ASSERT(opcode <= 15);
1206 RTE_ASSERT(subcode <= 127);
1208 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
1210 ih3->pkind = lio_dev->instr_queue[sc->iq_no]->txpciq.s.pkind;
1212 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
1217 pki_ih3->uqpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
1220 pki_ih3->tag = LIO_CONTROL;
1221 pki_ih3->tagtype = OCTEON_ATOMIC_TAG;
1222 pki_ih3->qpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.qpg;
1227 ih3->dlengsz = sc->datasize;
1229 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
1230 irh->opcode = opcode;
1231 irh->subcode = subcode;
1233 /* opcode/subcode specific parameters (ossp) */
1234 irh->ossp = irh_ossp;
1235 sc->cmd.cmd3.ossp[0] = ossp0;
1236 sc->cmd.cmd3.ossp[1] = ossp1;
1238 if (sc->rdatasize) {
1239 rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
1240 rdp->pcie_port = lio_dev->pcie_port;
1241 rdp->rlen = sc->rdatasize;
1244 ih3->fsz = OCTEON_SOFT_CMD_RESP_IH3;
1248 ih3->fsz = OCTEON_PCI_CMD_O3;
1253 lio_send_soft_command(struct lio_device *lio_dev,
1254 struct lio_soft_command *sc)
1256 struct octeon_instr_ih3 *ih3;
1257 struct octeon_instr_irh *irh;
1260 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
1262 RTE_ASSERT(sc->dmadptr);
1263 sc->cmd.cmd3.dptr = sc->dmadptr;
1266 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
1268 RTE_ASSERT(sc->dmarptr);
1269 RTE_ASSERT(sc->status_word != NULL);
1270 *sc->status_word = LIO_COMPLETION_WORD_INIT;
1271 sc->cmd.cmd3.rptr = sc->dmarptr;
1274 len = (uint32_t)ih3->dlengsz;
1277 sc->timeout = lio_uptime + sc->wait_time;
1279 return lio_send_command(lio_dev, sc->iq_no, &sc->cmd, sc, len,
1280 LIO_REQTYPE_SOFT_COMMAND);
1284 lio_setup_sc_buffer_pool(struct lio_device *lio_dev)
1286 char sc_pool_name[RTE_MEMPOOL_NAMESIZE];
1289 buf_size = LIO_SOFT_COMMAND_BUFFER_SIZE + RTE_PKTMBUF_HEADROOM;
1290 snprintf(sc_pool_name, sizeof(sc_pool_name),
1291 "lio_sc_pool_%u", lio_dev->port_id);
1292 lio_dev->sc_buf_pool = rte_pktmbuf_pool_create(sc_pool_name,
1293 LIO_MAX_SOFT_COMMAND_BUFFERS,
1294 0, 0, buf_size, SOCKET_ID_ANY);
1299 lio_free_sc_buffer_pool(struct lio_device *lio_dev)
1301 rte_mempool_free(lio_dev->sc_buf_pool);
1304 struct lio_soft_command *
1305 lio_alloc_soft_command(struct lio_device *lio_dev, uint32_t datasize,
1306 uint32_t rdatasize, uint32_t ctxsize)
1308 uint32_t offset = sizeof(struct lio_soft_command);
1309 struct lio_soft_command *sc;
1313 RTE_ASSERT((offset + datasize + rdatasize + ctxsize) <=
1314 LIO_SOFT_COMMAND_BUFFER_SIZE);
1316 m = rte_pktmbuf_alloc(lio_dev->sc_buf_pool);
1318 lio_dev_err(lio_dev, "Cannot allocate mbuf for sc\n");
1322 /* set rte_mbuf data size and there is only 1 segment */
1323 m->pkt_len = LIO_SOFT_COMMAND_BUFFER_SIZE;
1324 m->data_len = LIO_SOFT_COMMAND_BUFFER_SIZE;
1326 /* use rte_mbuf buffer for soft command */
1327 sc = rte_pktmbuf_mtod(m, struct lio_soft_command *);
1328 memset(sc, 0, LIO_SOFT_COMMAND_BUFFER_SIZE);
1329 sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE;
1330 sc->dma_addr = rte_mbuf_data_dma_addr(m);
1333 dma_addr = sc->dma_addr;
1336 sc->ctxptr = (uint8_t *)sc + offset;
1337 sc->ctxsize = ctxsize;
1340 /* Start data at 128 byte boundary */
1341 offset = (offset + ctxsize + 127) & 0xffffff80;
1344 sc->virtdptr = (uint8_t *)sc + offset;
1345 sc->dmadptr = dma_addr + offset;
1346 sc->datasize = datasize;
1349 /* Start rdata at 128 byte boundary */
1350 offset = (offset + datasize + 127) & 0xffffff80;
1353 RTE_ASSERT(rdatasize >= 16);
1354 sc->virtrptr = (uint8_t *)sc + offset;
1355 sc->dmarptr = dma_addr + offset;
1356 sc->rdatasize = rdatasize;
1357 sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) +
1365 lio_free_soft_command(struct lio_soft_command *sc)
1367 rte_pktmbuf_free(sc->mbuf);
1371 lio_setup_response_list(struct lio_device *lio_dev)
1373 STAILQ_INIT(&lio_dev->response_list.head);
1374 rte_spinlock_init(&lio_dev->response_list.lock);
1375 rte_atomic64_set(&lio_dev->response_list.pending_req_count, 0);
1379 lio_process_ordered_list(struct lio_device *lio_dev)
1381 int resp_to_process = LIO_MAX_ORD_REQS_TO_PROCESS;
1382 struct lio_response_list *ordered_sc_list;
1383 struct lio_soft_command *sc;
1384 int request_complete = 0;
1388 ordered_sc_list = &lio_dev->response_list;
1391 rte_spinlock_lock(&ordered_sc_list->lock);
1393 if (STAILQ_EMPTY(&ordered_sc_list->head)) {
1394 /* ordered_sc_list is empty; there is
1395 * nothing to process
1397 rte_spinlock_unlock(&ordered_sc_list->lock);
1401 sc = LIO_STQUEUE_FIRST_ENTRY(&ordered_sc_list->head,
1402 struct lio_soft_command, node);
1404 status = LIO_REQUEST_PENDING;
1406 /* check if octeon has finished DMA'ing a response
1407 * to where rptr is pointing to
1409 status64 = *sc->status_word;
1411 if (status64 != LIO_COMPLETION_WORD_INIT) {
1412 /* This logic ensures that all 64b have been written.
1413 * 1. check byte 0 for non-FF
1414 * 2. if non-FF, then swap result from BE to host order
1415 * 3. check byte 7 (swapped to 0) for non-FF
1416 * 4. if non-FF, use the low 32-bit status code
1417 * 5. if either byte 0 or byte 7 is FF, don't use status
1419 if ((status64 & 0xff) != 0xff) {
1420 lio_swap_8B_data(&status64, 1);
1421 if (((status64 & 0xff) != 0xff)) {
1422 /* retrieve 16-bit firmware status */
1423 status = (uint32_t)(status64 &
1427 LIO_FIRMWARE_STATUS_CODE(
1431 status = LIO_REQUEST_DONE;
1435 } else if ((sc->timeout && lio_check_timeout(lio_uptime,
1437 lio_dev_err(lio_dev,
1438 "cmd failed, timeout (%ld, %ld)\n",
1439 (long)lio_uptime, (long)sc->timeout);
1440 status = LIO_REQUEST_TIMEOUT;
1443 if (status != LIO_REQUEST_PENDING) {
1444 /* we have received a response or we have timed out.
1445 * remove node from linked list
1447 STAILQ_REMOVE(&ordered_sc_list->head,
1448 &sc->node, lio_stailq_node, entries);
1450 &lio_dev->response_list.pending_req_count);
1451 rte_spinlock_unlock(&ordered_sc_list->lock);
1454 sc->callback(status, sc->callback_arg);
1458 /* no response yet */
1459 request_complete = 0;
1460 rte_spinlock_unlock(&ordered_sc_list->lock);
1463 /* If we hit the Max Ordered requests to process every loop,
1464 * we quit and let this function be invoked the next time
1465 * the poll thread runs to process the remaining requests.
1466 * This function can take up the entire CPU if there is
1467 * no upper limit to the requests processed.
1469 if (request_complete >= resp_to_process)
1471 } while (request_complete);
1476 static inline struct lio_stailq_node *
1477 list_delete_first_node(struct lio_stailq_head *head)
1479 struct lio_stailq_node *node;
1481 if (STAILQ_EMPTY(head))
1484 node = STAILQ_FIRST(head);
1487 STAILQ_REMOVE(head, node, lio_stailq_node, entries);
1493 lio_delete_sglist(struct lio_instr_queue *txq)
1495 struct lio_device *lio_dev = txq->lio_dev;
1496 int iq_no = txq->q_index;
1497 struct lio_gather *g;
1499 if (lio_dev->glist_head == NULL)
1503 g = (struct lio_gather *)list_delete_first_node(
1504 &lio_dev->glist_head[iq_no]);
1508 (void *)((unsigned long)g->sg - g->adjust));
1515 * \brief Setup gather lists
1516 * @param lio per-network private data
1519 lio_setup_sglists(struct lio_device *lio_dev, int iq_no,
1520 int fw_mapped_iq, int num_descs, unsigned int socket_id)
1522 struct lio_gather *g;
1525 rte_spinlock_init(&lio_dev->glist_lock[iq_no]);
1527 STAILQ_INIT(&lio_dev->glist_head[iq_no]);
1529 for (i = 0; i < num_descs; i++) {
1530 g = rte_zmalloc_socket(NULL, sizeof(*g), RTE_CACHE_LINE_SIZE,
1533 lio_dev_err(lio_dev,
1534 "lio_gather memory allocation failed for qno %d\n",
1540 ((ROUNDUP4(LIO_MAX_SG) >> 2) * LIO_SG_ENTRY_SIZE);
1542 g->sg = rte_zmalloc_socket(NULL, g->sg_size + 8,
1543 RTE_CACHE_LINE_SIZE, socket_id);
1544 if (g->sg == NULL) {
1545 lio_dev_err(lio_dev,
1546 "sg list memory allocation failed for qno %d\n",
1552 /* The gather component should be aligned on 64-bit boundary */
1553 if (((unsigned long)g->sg) & 7) {
1554 g->adjust = 8 - (((unsigned long)g->sg) & 7);
1556 (struct lio_sg_entry *)((unsigned long)g->sg +
1560 STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq_no], &g->list,
1564 if (i != num_descs) {
1565 lio_delete_sglist(lio_dev->instr_queue[fw_mapped_iq]);
1573 lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no)
1575 lio_delete_instr_queue(lio_dev, iq_no);
1576 rte_free(lio_dev->instr_queue[iq_no]);
1577 lio_dev->instr_queue[iq_no] = NULL;
1581 static inline uint32_t
1582 lio_iq_get_available(struct lio_device *lio_dev, uint32_t q_no)
1584 return ((lio_dev->instr_queue[q_no]->max_count - 1) -
1585 (uint32_t)rte_atomic64_read(
1586 &lio_dev->instr_queue[q_no]->instr_pending));
1590 lio_iq_is_full(struct lio_device *lio_dev, uint32_t q_no)
1592 return ((uint32_t)rte_atomic64_read(
1593 &lio_dev->instr_queue[q_no]->instr_pending) >=
1594 (lio_dev->instr_queue[q_no]->max_count - 2));
1598 lio_dev_cleanup_iq(struct lio_device *lio_dev, int iq_no)
1600 struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
1601 uint32_t count = 10000;
1603 while ((lio_iq_get_available(lio_dev, iq_no) < LIO_FLUSH_WM(iq)) &&
1605 lio_flush_iq(lio_dev, iq);
1607 return count ? 0 : 1;
1611 lio_ctrl_cmd_callback(uint32_t status __rte_unused, void *sc_ptr)
1613 struct lio_soft_command *sc = sc_ptr;
1614 struct lio_dev_ctrl_cmd *ctrl_cmd;
1615 struct lio_ctrl_pkt *ctrl_pkt;
1617 ctrl_pkt = (struct lio_ctrl_pkt *)sc->ctxptr;
1618 ctrl_cmd = ctrl_pkt->ctrl_cmd;
1621 lio_free_soft_command(sc);
1624 static inline struct lio_soft_command *
1625 lio_alloc_ctrl_pkt_sc(struct lio_device *lio_dev,
1626 struct lio_ctrl_pkt *ctrl_pkt)
1628 struct lio_soft_command *sc = NULL;
1629 uint32_t uddsize, datasize;
1633 uddsize = (uint32_t)(ctrl_pkt->ncmd.s.more * 8);
1635 datasize = OCTEON_CMD_SIZE + uddsize;
1636 rdatasize = (ctrl_pkt->wait_time) ? 16 : 0;
1638 sc = lio_alloc_soft_command(lio_dev, datasize,
1639 rdatasize, sizeof(struct lio_ctrl_pkt));
1643 rte_memcpy(sc->ctxptr, ctrl_pkt, sizeof(struct lio_ctrl_pkt));
1645 data = (uint8_t *)sc->virtdptr;
1647 rte_memcpy(data, &ctrl_pkt->ncmd, OCTEON_CMD_SIZE);
1649 lio_swap_8B_data((uint64_t *)data, OCTEON_CMD_SIZE >> 3);
1652 /* Endian-Swap for UDD should have been done by caller. */
1653 rte_memcpy(data + OCTEON_CMD_SIZE, ctrl_pkt->udd, uddsize);
1656 sc->iq_no = (uint32_t)ctrl_pkt->iq_no;
1658 lio_prepare_soft_command(lio_dev, sc,
1659 LIO_OPCODE, LIO_OPCODE_CMD,
1662 sc->callback = lio_ctrl_cmd_callback;
1663 sc->callback_arg = sc;
1664 sc->wait_time = ctrl_pkt->wait_time;
1670 lio_send_ctrl_pkt(struct lio_device *lio_dev, struct lio_ctrl_pkt *ctrl_pkt)
1672 struct lio_soft_command *sc = NULL;
1675 sc = lio_alloc_ctrl_pkt_sc(lio_dev, ctrl_pkt);
1677 lio_dev_err(lio_dev, "soft command allocation failed\n");
1681 retval = lio_send_soft_command(lio_dev, sc);
1682 if (retval == LIO_IQ_SEND_FAILED) {
1683 lio_free_soft_command(sc);
1684 lio_dev_err(lio_dev, "Port: %d soft command: %d send failed status: %x\n",
1685 lio_dev->port_id, ctrl_pkt->ncmd.s.cmd, retval);
1692 /** Send data packet to the device
1693 * @param lio_dev - lio device pointer
1694 * @param ndata - control structure with queueing, and buffer information
1696 * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the
1697 * queue should be stopped, and LIO_IQ_SEND_OK if it sent okay.
1700 lio_send_data_pkt(struct lio_device *lio_dev, struct lio_data_pkt *ndata)
1702 return lio_send_command(lio_dev, ndata->q_no, &ndata->cmd,
1703 ndata->buf, ndata->datasize, ndata->reqtype);
1707 lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
1709 struct lio_instr_queue *txq = tx_queue;
1710 union lio_cmd_setup cmdsetup;
1711 struct lio_device *lio_dev;
1712 struct lio_iq_stats *stats;
1713 struct lio_data_pkt ndata;
1714 int i, processed = 0;
1720 lio_dev = txq->lio_dev;
1721 iq_no = txq->txpciq.s.q_no;
1722 stats = &lio_dev->instr_queue[iq_no]->stats;
1724 if (!lio_dev->intf_open || !lio_dev->linfo.link.s.link_up) {
1725 PMD_TX_LOG(lio_dev, ERR, "Transmit failed link_status : %d\n",
1726 lio_dev->linfo.link.s.link_up);
1730 lio_dev_cleanup_iq(lio_dev, iq_no);
1732 for (i = 0; i < nb_pkts; i++) {
1733 uint32_t pkt_len = 0;
1737 /* Prepare the attributes for the data to be passed to BASE. */
1738 memset(&ndata, 0, sizeof(struct lio_data_pkt));
1743 if (lio_iq_is_full(lio_dev, ndata.q_no)) {
1744 stats->tx_iq_busy++;
1745 if (lio_dev_cleanup_iq(lio_dev, iq_no)) {
1746 PMD_TX_LOG(lio_dev, ERR,
1747 "Transmit failed iq:%d full\n",
1753 cmdsetup.cmd_setup64 = 0;
1754 cmdsetup.s.iq_no = iq_no;
1756 /* check checksum offload flags to form cmd */
1757 if (m->ol_flags & PKT_TX_IP_CKSUM)
1758 cmdsetup.s.ip_csum = 1;
1760 if (m->ol_flags & PKT_TX_OUTER_IP_CKSUM)
1761 cmdsetup.s.tnl_csum = 1;
1762 else if ((m->ol_flags & PKT_TX_TCP_CKSUM) ||
1763 (m->ol_flags & PKT_TX_UDP_CKSUM))
1764 cmdsetup.s.transport_csum = 1;
1766 if (m->nb_segs == 1) {
1767 pkt_len = rte_pktmbuf_data_len(m);
1768 cmdsetup.s.u.datasize = pkt_len;
1769 lio_prepare_pci_cmd(lio_dev, &ndata.cmd,
1771 ndata.cmd.cmd3.dptr = rte_mbuf_data_dma_addr(m);
1772 ndata.reqtype = LIO_REQTYPE_NORESP_NET;
1774 struct lio_buf_free_info *finfo;
1775 struct lio_gather *g;
1776 phys_addr_t phyaddr;
1779 finfo = (struct lio_buf_free_info *)rte_malloc(NULL,
1781 if (finfo == NULL) {
1782 PMD_TX_LOG(lio_dev, ERR,
1783 "free buffer alloc failed\n");
1787 rte_spinlock_lock(&lio_dev->glist_lock[iq_no]);
1788 g = (struct lio_gather *)list_delete_first_node(
1789 &lio_dev->glist_head[iq_no]);
1790 rte_spinlock_unlock(&lio_dev->glist_lock[iq_no]);
1792 PMD_TX_LOG(lio_dev, ERR,
1793 "Transmit scatter gather: glist null!\n");
1797 cmdsetup.s.gather = 1;
1798 cmdsetup.s.u.gatherptrs = m->nb_segs;
1799 lio_prepare_pci_cmd(lio_dev, &ndata.cmd,
1802 memset(g->sg, 0, g->sg_size);
1803 g->sg[0].ptr[0] = rte_mbuf_data_dma_addr(m);
1804 lio_add_sg_size(&g->sg[0], m->data_len, 0);
1805 pkt_len = m->data_len;
1808 /* First seg taken care above */
1809 frags = m->nb_segs - 1;
1813 g->sg[(i >> 2)].ptr[(i & 3)] =
1814 rte_mbuf_data_dma_addr(m);
1815 lio_add_sg_size(&g->sg[(i >> 2)],
1816 m->data_len, (i & 3));
1817 pkt_len += m->data_len;
1822 phyaddr = rte_mem_virt2phy(g->sg);
1823 if (phyaddr == RTE_BAD_PHYS_ADDR) {
1824 PMD_TX_LOG(lio_dev, ERR, "bad phys addr\n");
1828 ndata.cmd.cmd3.dptr = phyaddr;
1829 ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG;
1832 finfo->lio_dev = lio_dev;
1833 finfo->iq_no = (uint64_t)iq_no;
1837 ndata.datasize = pkt_len;
1839 status = lio_send_data_pkt(lio_dev, &ndata);
1841 if (unlikely(status == LIO_IQ_SEND_FAILED)) {
1842 PMD_TX_LOG(lio_dev, ERR, "send failed\n");
1846 if (unlikely(status == LIO_IQ_SEND_STOP)) {
1847 PMD_TX_LOG(lio_dev, DEBUG, "iq full\n");
1848 /* create space as iq is full */
1849 lio_dev_cleanup_iq(lio_dev, iq_no);
1853 stats->tx_tot_bytes += pkt_len;
1858 stats->tx_dropped += (nb_pkts - processed);
1864 lio_dev_clear_queues(struct rte_eth_dev *eth_dev)
1866 struct lio_instr_queue *txq;
1867 struct lio_droq *rxq;
1870 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1871 txq = eth_dev->data->tx_queues[i];
1873 lio_dev_tx_queue_release(txq);
1874 eth_dev->data->tx_queues[i] = NULL;
1878 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1879 rxq = eth_dev->data->rx_queues[i];
1881 lio_dev_rx_queue_release(rxq);
1882 eth_dev->data->rx_queues[i] = NULL;