-/*
- * BSD LICENSE
- *
- * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Cavium, Inc. nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Cavium, Inc
*/
-#include <rte_ethdev.h>
+#include <rte_ethdev_driver.h>
#include <rte_cycles.h>
#include <rte_malloc.h>
#include "lio_rxtx.h"
#define LIO_MAX_SG 12
+/* Flush iq if available tx_desc fall below LIO_FLUSH_WM */
+#define LIO_FLUSH_WM(_iq) ((_iq)->nb_desc / 2)
+#define LIO_PKT_IN_DONE_CNT_MASK 0x00000000FFFFFFFFULL
static void
lio_droq_compute_max_packet_bufs(struct lio_droq *droq)
{
uint32_t i;
- for (i = 0; i < droq->max_count; i++) {
+ for (i = 0; i < droq->nb_desc; i++) {
if (droq->recv_buf_list[i].buffer) {
rte_pktmbuf_free((struct rte_mbuf *)
droq->recv_buf_list[i].buffer);
lio_droq_reset_indices(droq);
}
-static void *
-lio_recv_buffer_alloc(struct lio_device *lio_dev, int q_no)
-{
- struct lio_droq *droq = lio_dev->droq[q_no];
- struct rte_mempool *mpool = droq->mpool;
- struct rte_mbuf *m;
-
- m = rte_pktmbuf_alloc(mpool);
- if (m == NULL) {
- lio_dev_err(lio_dev, "Cannot allocate\n");
- return NULL;
- }
-
- rte_mbuf_refcnt_set(m, 1);
- m->next = NULL;
- m->data_off = RTE_PKTMBUF_HEADROOM;
- m->nb_segs = 1;
- m->pool = mpool;
-
- return m;
-}
-
static int
lio_droq_setup_ring_buffers(struct lio_device *lio_dev,
struct lio_droq *droq)
uint32_t i;
void *buf;
- for (i = 0; i < droq->max_count; i++) {
- buf = lio_recv_buffer_alloc(lio_dev, droq->q_no);
+ for (i = 0; i < droq->nb_desc; i++) {
+ buf = rte_pktmbuf_alloc(droq->mpool);
if (buf == NULL) {
lio_dev_err(lio_dev, "buffer alloc failed\n");
+ droq->stats.rx_alloc_failure++;
lio_droq_destroy_ring_buffers(droq);
return -ENOMEM;
}
{
droq->info_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev,
"info_list", droq->q_no,
- (droq->max_count *
+ (droq->nb_desc *
LIO_DROQ_INFO_SIZE),
RTE_CACHE_LINE_SIZE,
socket_id);
if (droq->info_mz == NULL)
return NULL;
- droq->info_list_dma = droq->info_mz->phys_addr;
+ droq->info_list_dma = droq->info_mz->iova;
droq->info_alloc_size = droq->info_mz->len;
droq->info_base_addr = (size_t)droq->info_mz->addr;
c_refill_threshold = LIO_OQ_REFILL_THRESHOLD_CFG(lio_dev);
- droq->max_count = num_descs;
+ droq->nb_desc = num_descs;
droq->buffer_size = desc_size;
- desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE;
+ desc_ring_size = droq->nb_desc * LIO_DROQ_DESC_SIZE;
droq->desc_ring_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev,
"droq", q_no,
desc_ring_size,
return -1;
}
- droq->desc_ring_dma = droq->desc_ring_mz->phys_addr;
+ droq->desc_ring_dma = droq->desc_ring_mz->iova;
droq->desc_ring = (struct lio_droq_desc *)droq->desc_ring_mz->addr;
lio_dev_dbg(lio_dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma);
lio_dev_dbg(lio_dev, "droq[%d]: num_desc: %d\n", q_no,
- droq->max_count);
+ droq->nb_desc);
droq->info_list = lio_alloc_info_buffer(lio_dev, droq, socket_id);
if (droq->info_list == NULL) {
}
droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list",
- (droq->max_count *
+ (droq->nb_desc *
LIO_DROQ_RECVBUF_SIZE),
RTE_CACHE_LINE_SIZE,
socket_id);
PMD_INIT_FUNC_TRACE();
- if (lio_dev->droq[oq_no]) {
- lio_dev_dbg(lio_dev, "Droq %d in use\n", oq_no);
- return 0;
- }
-
/* Allocate the DS for the new droq. */
droq = rte_zmalloc_socket("ethdev RX queue", sizeof(*droq),
RTE_CACHE_LINE_SIZE, socket_id);
/* Send credit for octeon output queues. credits are always
* sent after the output queue is enabled.
*/
- rte_write32(lio_dev->droq[oq_no]->max_count,
+ rte_write32(lio_dev->droq[oq_no]->nb_desc,
lio_dev->droq[oq_no]->pkts_credit_reg);
rte_wmb();
do {
droq->refill_idx = lio_incr_index(
droq->refill_idx, 1,
- droq->max_count);
+ droq->nb_desc);
desc_refilled++;
droq->refill_count--;
} while (droq->recv_buf_list[droq->refill_idx].buffer);
}
refill_index = lio_incr_index(refill_index, 1,
- droq->max_count);
+ droq->nb_desc);
} /* while */
return desc_refilled;
/* lio_droq_refill
*
- * @param lio_dev - pointer to the lio device structure
* @param droq - droq in which descriptors require new buffers.
*
* Description:
* This routine is called with droq->lock held.
*/
static uint32_t
-lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq)
+lio_droq_refill(struct lio_droq *droq)
{
struct lio_droq_desc *desc_ring;
uint32_t desc_refilled = 0;
desc_ring = droq->desc_ring;
- while (droq->refill_count && (desc_refilled < droq->max_count)) {
+ while (droq->refill_count && (desc_refilled < droq->nb_desc)) {
/* If a valid buffer exists (happens if there is no dispatch),
* reuse the buffer, else allocate.
*/
if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) {
- buf = lio_recv_buffer_alloc(lio_dev, droq->q_no);
+ buf = rte_pktmbuf_alloc(droq->mpool);
/* If a buffer could not be allocated, no point in
* continuing
*/
- if (buf == NULL)
+ if (buf == NULL) {
+ droq->stats.rx_alloc_failure++;
break;
+ }
droq->recv_buf_list[droq->refill_idx].buffer = buf;
}
droq->info_list[droq->refill_idx].length = 0;
droq->refill_idx = lio_incr_index(droq->refill_idx, 1,
- droq->max_count);
+ droq->nb_desc);
desc_refilled++;
droq->refill_count--;
}
buf_cnt = lio_droq_get_bufcount(droq->buffer_size,
(uint32_t)info->length);
droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt,
- droq->max_count);
+ droq->nb_desc);
droq->refill_count += buf_cnt;
} else {
if (info->length <= droq->buffer_size) {
droq->recv_buf_list[droq->read_idx].buffer = NULL;
droq->read_idx = lio_incr_index(
droq->read_idx, 1,
- droq->max_count);
+ droq->nb_desc);
droq->refill_count++;
if (likely(nicbuf != NULL)) {
- nicbuf->data_off = RTE_PKTMBUF_HEADROOM;
- nicbuf->nb_segs = 1;
- nicbuf->next = NULL;
/* We don't have a way to pass flags yet */
nicbuf->ol_flags = 0;
if (rh->r_dh.has_hash) {
if (!pkt_len)
first_buf = nicbuf;
- nicbuf->data_off = RTE_PKTMBUF_HEADROOM;
- nicbuf->nb_segs = 1;
- nicbuf->next = NULL;
nicbuf->port = lio_dev->port_id;
/* We don't have a way to pass
* flags yet
pkt_len += cpy_len;
droq->read_idx = lio_incr_index(
droq->read_idx,
- 1, droq->max_count);
+ 1, droq->nb_desc);
droq->refill_count++;
/* Prefetch buffer pointers when on a
}
if (droq->refill_count >= droq->refill_threshold) {
- int desc_refilled = lio_droq_refill(lio_dev, droq);
+ int desc_refilled = lio_droq_refill(droq);
/* Flush the droq descriptor data to memory to be sure
* that when we update the credits the data in memory is
info->length = 0;
info->rh.rh64 = 0;
+ droq->stats.pkts_received++;
+ droq->stats.rx_pkts_received += data_pkts;
+ droq->stats.rx_bytes_received += data_total_len;
+ droq->stats.bytes_received += total_len;
+
return data_pkts;
}
return -1;
}
- iq->base_addr_dma = iq->iq_mz->phys_addr;
+ iq->base_addr_dma = iq->iq_mz->iova;
iq->base_addr = (uint8_t *)iq->iq_mz->addr;
- iq->max_count = num_descs;
+ iq->nb_desc = num_descs;
/* Initialize a list to holds requests that have been posted to Octeon
* but has yet to be fetched by octeon
lio_dev_dbg(lio_dev, "IQ[%d]: base: %p basedma: %lx count: %d\n",
iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma,
- iq->max_count);
+ iq->nb_desc);
iq->lio_dev = lio_dev;
iq->txpciq.txpciq64 = txpciq.txpciq64;
{
uint32_t iq_no = (uint32_t)txpciq.s.q_no;
- if (lio_dev->instr_queue[iq_no]) {
- lio_dev_dbg(lio_dev, "IQ is in use. Cannot create the IQ: %d again\n",
- iq_no);
- lio_dev->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64;
- lio_dev->instr_queue[iq_no]->app_ctx = app_ctx;
- return 0;
- }
-
lio_dev->instr_queue[iq_no] = rte_zmalloc_socket("ethdev TX queue",
sizeof(struct lio_instr_queue),
RTE_CACHE_LINE_SIZE, socket_id);
lio_dev->instr_queue[iq_no]->q_index = q_index;
lio_dev->instr_queue[iq_no]->app_ctx = app_ctx;
- if (lio_init_instr_queue(lio_dev, txpciq, num_descs, socket_id))
- goto release_lio_iq;
+ if (lio_init_instr_queue(lio_dev, txpciq, num_descs, socket_id)) {
+ rte_free(lio_dev->instr_queue[iq_no]);
+ lio_dev->instr_queue[iq_no] = NULL;
+ return -1;
+ }
lio_dev->num_iqs++;
- if (lio_dev->fn_list.enable_io_queues(lio_dev))
- goto delete_lio_iq;
return 0;
+}
-delete_lio_iq:
- lio_delete_instr_queue(lio_dev, iq_no);
- lio_dev->num_iqs--;
-release_lio_iq:
- rte_free(lio_dev->instr_queue[iq_no]);
- lio_dev->instr_queue[iq_no] = NULL;
+int
+lio_wait_for_instr_fetch(struct lio_device *lio_dev)
+{
+ int pending, instr_cnt;
+ int i, retry = 1000;
- return -1;
+ do {
+ instr_cnt = 0;
+
+ for (i = 0; i < LIO_MAX_INSTR_QUEUES(lio_dev); i++) {
+ if (!(lio_dev->io_qmask.iq & (1ULL << i)))
+ continue;
+
+ if (lio_dev->instr_queue[i] == NULL)
+ break;
+
+ pending = rte_atomic64_read(
+ &lio_dev->instr_queue[i]->instr_pending);
+ if (pending)
+ lio_flush_iq(lio_dev, lio_dev->instr_queue[i]);
+
+ instr_cnt += pending;
+ }
+
+ if (instr_cnt == 0)
+ break;
+
+ rte_delay_ms(1);
+
+ } while (retry-- && instr_cnt);
+
+ return instr_cnt;
}
static inline void
* position if queue gets full before Octeon could fetch any instr.
*/
if (rte_atomic64_read(&iq->instr_pending) >=
- (int32_t)(iq->max_count - 1)) {
+ (int32_t)(iq->nb_desc - 1)) {
st.status = LIO_IQ_SEND_FAILED;
st.index = -1;
return st;
}
if (rte_atomic64_read(&iq->instr_pending) >=
- (int32_t)(iq->max_count - 2))
+ (int32_t)(iq->nb_desc - 2))
st.status = LIO_IQ_SEND_STOP;
copy_cmd_into_iq(iq, cmd);
/* "index" is returned, host_write_index is modified. */
st.index = iq->host_write_index;
iq->host_write_index = lio_incr_index(iq->host_write_index, 1,
- iq->max_count);
+ iq->nb_desc);
iq->fill_cnt++;
/* Flush the command into memory. We need to be sure the data is in
iq->request_list[idx].reqtype = reqtype;
}
+static inline void
+lio_free_netsgbuf(void *buf)
+{
+ struct lio_buf_free_info *finfo = buf;
+ struct lio_device *lio_dev = finfo->lio_dev;
+ struct rte_mbuf *m = finfo->mbuf;
+ struct lio_gather *g = finfo->g;
+ uint8_t iq = finfo->iq_no;
+
+ /* This will take care of multiple segments also */
+ rte_pktmbuf_free(m);
+
+ rte_spinlock_lock(&lio_dev->glist_lock[iq]);
+ STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq], &g->list, entries);
+ rte_spinlock_unlock(&lio_dev->glist_lock[iq]);
+ rte_free(finfo);
+}
+
+/* Can only run in process context */
+static int
+lio_process_iq_request_list(struct lio_device *lio_dev,
+ struct lio_instr_queue *iq)
+{
+ struct octeon_instr_irh *irh = NULL;
+ uint32_t old = iq->flush_index;
+ struct lio_soft_command *sc;
+ uint32_t inst_count = 0;
+ int reqtype;
+ void *buf;
+
+ while (old != iq->lio_read_index) {
+ reqtype = iq->request_list[old].reqtype;
+ buf = iq->request_list[old].buf;
+
+ if (reqtype == LIO_REQTYPE_NONE)
+ goto skip_this;
+
+ switch (reqtype) {
+ case LIO_REQTYPE_NORESP_NET:
+ rte_pktmbuf_free((struct rte_mbuf *)buf);
+ break;
+ case LIO_REQTYPE_NORESP_NET_SG:
+ lio_free_netsgbuf(buf);
+ break;
+ case LIO_REQTYPE_SOFT_COMMAND:
+ sc = buf;
+ irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
+ if (irh->rflag) {
+ /* We're expecting a response from Octeon.
+ * It's up to lio_process_ordered_list() to
+ * process sc. Add sc to the ordered soft
+ * command response list because we expect
+ * a response from Octeon.
+ */
+ rte_spinlock_lock(&lio_dev->response_list.lock);
+ rte_atomic64_inc(
+ &lio_dev->response_list.pending_req_count);
+ STAILQ_INSERT_TAIL(
+ &lio_dev->response_list.head,
+ &sc->node, entries);
+ rte_spinlock_unlock(
+ &lio_dev->response_list.lock);
+ } else {
+ if (sc->callback) {
+ /* This callback must not sleep */
+ sc->callback(LIO_REQUEST_DONE,
+ sc->callback_arg);
+ }
+ }
+ break;
+ default:
+ lio_dev_err(lio_dev,
+ "Unknown reqtype: %d buf: %p at idx %d\n",
+ reqtype, buf, old);
+ }
+
+ iq->request_list[old].buf = NULL;
+ iq->request_list[old].reqtype = 0;
+
+skip_this:
+ inst_count++;
+ old = lio_incr_index(old, 1, iq->nb_desc);
+ }
+
+ iq->flush_index = old;
+
+ return inst_count;
+}
+
+static void
+lio_update_read_index(struct lio_instr_queue *iq)
+{
+ uint32_t pkt_in_done = rte_read32(iq->inst_cnt_reg);
+ uint32_t last_done;
+
+ last_done = pkt_in_done - iq->pkt_in_done;
+ iq->pkt_in_done = pkt_in_done;
+
+ /* Add last_done and modulo with the IQ size to get new index */
+ iq->lio_read_index = (iq->lio_read_index +
+ (uint32_t)(last_done & LIO_PKT_IN_DONE_CNT_MASK)) %
+ iq->nb_desc;
+}
+
+int
+lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq)
+{
+ uint32_t tot_inst_processed = 0;
+ uint32_t inst_processed = 0;
+ int tx_done = 1;
+
+ if (rte_atomic64_test_and_set(&iq->iq_flush_running) == 0)
+ return tx_done;
+
+ rte_spinlock_lock(&iq->lock);
+
+ lio_update_read_index(iq);
+
+ do {
+ /* Process any outstanding IQ packets. */
+ if (iq->flush_index == iq->lio_read_index)
+ break;
+
+ inst_processed = lio_process_iq_request_list(lio_dev, iq);
+
+ if (inst_processed) {
+ rte_atomic64_sub(&iq->instr_pending, inst_processed);
+ iq->stats.instr_processed += inst_processed;
+ }
+
+ tot_inst_processed += inst_processed;
+ inst_processed = 0;
+
+ } while (1);
+
+ rte_spinlock_unlock(&iq->lock);
+
+ rte_atomic64_clear(&iq->iq_flush_running);
+
+ return tx_done;
+}
+
static int
lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd,
- void *buf, uint32_t datasize __rte_unused, uint32_t reqtype)
+ void *buf, uint32_t datasize, uint32_t reqtype)
{
struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
struct lio_iq_post_status st;
if (st.status != LIO_IQ_SEND_FAILED) {
lio_add_to_request_list(iq, st.index, buf, reqtype);
+ LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, bytes_sent,
+ datasize);
+ LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_posted, 1);
+
lio_ring_doorbell(lio_dev, iq);
+ } else {
+ LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_dropped, 1);
}
rte_spinlock_unlock(&iq->post_lock);
sc = rte_pktmbuf_mtod(m, struct lio_soft_command *);
memset(sc, 0, LIO_SOFT_COMMAND_BUFFER_SIZE);
sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE;
- sc->dma_addr = rte_mbuf_data_dma_addr(m);
+ sc->dma_addr = rte_mbuf_data_iova(m);
sc->mbuf = m;
dma_addr = sc->dma_addr;
return node;
}
-static void
+void
lio_delete_sglist(struct lio_instr_queue *txq)
{
struct lio_device *lio_dev = txq->lio_dev;
lio_dev->num_iqs--;
}
+static inline uint32_t
+lio_iq_get_available(struct lio_device *lio_dev, uint32_t q_no)
+{
+ return ((lio_dev->instr_queue[q_no]->nb_desc - 1) -
+ (uint32_t)rte_atomic64_read(
+ &lio_dev->instr_queue[q_no]->instr_pending));
+}
+
+static inline int
+lio_iq_is_full(struct lio_device *lio_dev, uint32_t q_no)
+{
+ return ((uint32_t)rte_atomic64_read(
+ &lio_dev->instr_queue[q_no]->instr_pending) >=
+ (lio_dev->instr_queue[q_no]->nb_desc - 2));
+}
+
+static int
+lio_dev_cleanup_iq(struct lio_device *lio_dev, int iq_no)
+{
+ struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
+ uint32_t count = 10000;
+
+ while ((lio_iq_get_available(lio_dev, iq_no) < LIO_FLUSH_WM(iq)) &&
+ --count)
+ lio_flush_iq(lio_dev, iq);
+
+ return count ? 0 : 1;
+}
+
+static void
+lio_ctrl_cmd_callback(uint32_t status __rte_unused, void *sc_ptr)
+{
+ struct lio_soft_command *sc = sc_ptr;
+ struct lio_dev_ctrl_cmd *ctrl_cmd;
+ struct lio_ctrl_pkt *ctrl_pkt;
+
+ ctrl_pkt = (struct lio_ctrl_pkt *)sc->ctxptr;
+ ctrl_cmd = ctrl_pkt->ctrl_cmd;
+ ctrl_cmd->cond = 1;
+
+ lio_free_soft_command(sc);
+}
+
+static inline struct lio_soft_command *
+lio_alloc_ctrl_pkt_sc(struct lio_device *lio_dev,
+ struct lio_ctrl_pkt *ctrl_pkt)
+{
+ struct lio_soft_command *sc = NULL;
+ uint32_t uddsize, datasize;
+ uint32_t rdatasize;
+ uint8_t *data;
+
+ uddsize = (uint32_t)(ctrl_pkt->ncmd.s.more * 8);
+
+ datasize = OCTEON_CMD_SIZE + uddsize;
+ rdatasize = (ctrl_pkt->wait_time) ? 16 : 0;
+
+ sc = lio_alloc_soft_command(lio_dev, datasize,
+ rdatasize, sizeof(struct lio_ctrl_pkt));
+ if (sc == NULL)
+ return NULL;
+
+ rte_memcpy(sc->ctxptr, ctrl_pkt, sizeof(struct lio_ctrl_pkt));
+
+ data = (uint8_t *)sc->virtdptr;
+
+ rte_memcpy(data, &ctrl_pkt->ncmd, OCTEON_CMD_SIZE);
+
+ lio_swap_8B_data((uint64_t *)data, OCTEON_CMD_SIZE >> 3);
+
+ if (uddsize) {
+ /* Endian-Swap for UDD should have been done by caller. */
+ rte_memcpy(data + OCTEON_CMD_SIZE, ctrl_pkt->udd, uddsize);
+ }
+
+ sc->iq_no = (uint32_t)ctrl_pkt->iq_no;
+
+ lio_prepare_soft_command(lio_dev, sc,
+ LIO_OPCODE, LIO_OPCODE_CMD,
+ 0, 0, 0);
+
+ sc->callback = lio_ctrl_cmd_callback;
+ sc->callback_arg = sc;
+ sc->wait_time = ctrl_pkt->wait_time;
+
+ return sc;
+}
+
+int
+lio_send_ctrl_pkt(struct lio_device *lio_dev, struct lio_ctrl_pkt *ctrl_pkt)
+{
+ struct lio_soft_command *sc = NULL;
+ int retval;
+
+ sc = lio_alloc_ctrl_pkt_sc(lio_dev, ctrl_pkt);
+ if (sc == NULL) {
+ lio_dev_err(lio_dev, "soft command allocation failed\n");
+ return -1;
+ }
+
+ retval = lio_send_soft_command(lio_dev, sc);
+ if (retval == LIO_IQ_SEND_FAILED) {
+ lio_free_soft_command(sc);
+ lio_dev_err(lio_dev, "Port: %d soft command: %d send failed status: %x\n",
+ lio_dev->port_id, ctrl_pkt->ncmd.s.cmd, retval);
+ return -1;
+ }
+
+ return retval;
+}
+
/** Send data packet to the device
* @param lio_dev - lio device pointer
* @param ndata - control structure with queueing, and buffer information
struct lio_instr_queue *txq = tx_queue;
union lio_cmd_setup cmdsetup;
struct lio_device *lio_dev;
+ struct lio_iq_stats *stats;
struct lio_data_pkt ndata;
int i, processed = 0;
struct rte_mbuf *m;
lio_dev = txq->lio_dev;
iq_no = txq->txpciq.s.q_no;
+ stats = &lio_dev->instr_queue[iq_no]->stats;
- if (!lio_dev->linfo.link.s.link_up) {
+ if (!lio_dev->intf_open || !lio_dev->linfo.link.s.link_up) {
PMD_TX_LOG(lio_dev, ERR, "Transmit failed link_status : %d\n",
lio_dev->linfo.link.s.link_up);
goto xmit_failed;
}
+ lio_dev_cleanup_iq(lio_dev, iq_no);
+
for (i = 0; i < nb_pkts; i++) {
uint32_t pkt_len = 0;
ndata.buf = m;
ndata.q_no = iq_no;
+ if (lio_iq_is_full(lio_dev, ndata.q_no)) {
+ stats->tx_iq_busy++;
+ if (lio_dev_cleanup_iq(lio_dev, iq_no)) {
+ PMD_TX_LOG(lio_dev, ERR,
+ "Transmit failed iq:%d full\n",
+ ndata.q_no);
+ break;
+ }
+ }
cmdsetup.cmd_setup64 = 0;
cmdsetup.s.iq_no = iq_no;
if (m->ol_flags & PKT_TX_IP_CKSUM)
cmdsetup.s.ip_csum = 1;
- if ((m->ol_flags & PKT_TX_TCP_CKSUM) ||
+ if (m->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+ cmdsetup.s.tnl_csum = 1;
+ else if ((m->ol_flags & PKT_TX_TCP_CKSUM) ||
(m->ol_flags & PKT_TX_UDP_CKSUM))
cmdsetup.s.transport_csum = 1;
cmdsetup.s.u.datasize = pkt_len;
lio_prepare_pci_cmd(lio_dev, &ndata.cmd,
&cmdsetup, tag);
- ndata.cmd.cmd3.dptr = rte_mbuf_data_dma_addr(m);
+ ndata.cmd.cmd3.dptr = rte_mbuf_data_iova(m);
ndata.reqtype = LIO_REQTYPE_NORESP_NET;
+ } else {
+ struct lio_buf_free_info *finfo;
+ struct lio_gather *g;
+ rte_iova_t phyaddr;
+ int i, frags;
+
+ finfo = (struct lio_buf_free_info *)rte_malloc(NULL,
+ sizeof(*finfo), 0);
+ if (finfo == NULL) {
+ PMD_TX_LOG(lio_dev, ERR,
+ "free buffer alloc failed\n");
+ goto xmit_failed;
+ }
+
+ rte_spinlock_lock(&lio_dev->glist_lock[iq_no]);
+ g = (struct lio_gather *)list_delete_first_node(
+ &lio_dev->glist_head[iq_no]);
+ rte_spinlock_unlock(&lio_dev->glist_lock[iq_no]);
+ if (g == NULL) {
+ PMD_TX_LOG(lio_dev, ERR,
+ "Transmit scatter gather: glist null!\n");
+ goto xmit_failed;
+ }
+
+ cmdsetup.s.gather = 1;
+ cmdsetup.s.u.gatherptrs = m->nb_segs;
+ lio_prepare_pci_cmd(lio_dev, &ndata.cmd,
+ &cmdsetup, tag);
+
+ memset(g->sg, 0, g->sg_size);
+ g->sg[0].ptr[0] = rte_mbuf_data_iova(m);
+ lio_add_sg_size(&g->sg[0], m->data_len, 0);
+ pkt_len = m->data_len;
+ finfo->mbuf = m;
+
+ /* First seg taken care above */
+ frags = m->nb_segs - 1;
+ i = 1;
+ m = m->next;
+ while (frags--) {
+ g->sg[(i >> 2)].ptr[(i & 3)] =
+ rte_mbuf_data_iova(m);
+ lio_add_sg_size(&g->sg[(i >> 2)],
+ m->data_len, (i & 3));
+ pkt_len += m->data_len;
+ i++;
+ m = m->next;
+ }
+
+ phyaddr = rte_mem_virt2iova(g->sg);
+ if (phyaddr == RTE_BAD_IOVA) {
+ PMD_TX_LOG(lio_dev, ERR, "bad phys addr\n");
+ goto xmit_failed;
+ }
+
+ ndata.cmd.cmd3.dptr = phyaddr;
+ ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG;
+
+ finfo->g = g;
+ finfo->lio_dev = lio_dev;
+ finfo->iq_no = (uint64_t)iq_no;
+ ndata.buf = finfo;
}
ndata.datasize = pkt_len;
break;
}
- if (unlikely(status == LIO_IQ_SEND_STOP))
+ if (unlikely(status == LIO_IQ_SEND_STOP)) {
PMD_TX_LOG(lio_dev, DEBUG, "iq full\n");
+ /* create space as iq is full */
+ lio_dev_cleanup_iq(lio_dev, iq_no);
+ }
+ stats->tx_done++;
+ stats->tx_tot_bytes += pkt_len;
processed++;
}
xmit_failed:
+ stats->tx_dropped += (nb_pkts - processed);
return processed;
}
+void
+lio_dev_clear_queues(struct rte_eth_dev *eth_dev)
+{
+ struct lio_instr_queue *txq;
+ struct lio_droq *rxq;
+ uint16_t i;
+
+ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
+ txq = eth_dev->data->tx_queues[i];
+ if (txq != NULL) {
+ lio_dev_tx_queue_release(txq);
+ eth_dev->data->tx_queues[i] = NULL;
+ }
+ }
+
+ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
+ rxq = eth_dev->data->rx_queues[i];
+ if (rxq != NULL) {
+ lio_dev_rx_queue_release(rxq);
+ eth_dev->data->rx_queues[i] = NULL;
+ }
+ }
+}
git.droids-corp.org / dpdk.git / blobdiff