#include "hns3_logs.h"
#define HNS3_CFG_DESC_NUM(num) ((num) / 8 - 1)
-#define DEFAULT_RX_FREE_THRESH 16
+#define DEFAULT_RX_FREE_THRESH 32
static void
hns3_rx_queue_release_mbufs(struct hns3_rx_queue *rxq)
{
uint16_t i;
+ /* Note: Fake rx queue will not enter here */
if (rxq->sw_ring) {
for (i = 0; i < rxq->nb_rx_desc; i++) {
if (rxq->sw_ring[i].mbuf) {
{
uint16_t i;
+ /* Note: Fake rx queue will not enter here */
if (txq->sw_ring) {
for (i = 0; i < txq->nb_tx_desc; i++) {
if (txq->sw_ring[i].mbuf) {
rte_spinlock_unlock(&hns->hw.lock);
}
-void
-hns3_free_all_queues(struct rte_eth_dev *dev)
+static void
+hns3_fake_rx_queue_release(struct hns3_rx_queue *queue)
+{
+ struct hns3_rx_queue *rxq = queue;
+ struct hns3_adapter *hns;
+ struct hns3_hw *hw;
+ uint16_t idx;
+
+ if (rxq == NULL)
+ return;
+
+ hns = rxq->hns;
+ hw = &hns->hw;
+ idx = rxq->queue_id;
+ if (hw->fkq_data.rx_queues[idx]) {
+ hns3_rx_queue_release(hw->fkq_data.rx_queues[idx]);
+ hw->fkq_data.rx_queues[idx] = NULL;
+ }
+
+ /* free fake rx queue arrays */
+ if (idx == (hw->fkq_data.nb_fake_rx_queues - 1)) {
+ hw->fkq_data.nb_fake_rx_queues = 0;
+ rte_free(hw->fkq_data.rx_queues);
+ hw->fkq_data.rx_queues = NULL;
+ }
+}
+
+static void
+hns3_fake_tx_queue_release(struct hns3_tx_queue *queue)
{
+ struct hns3_tx_queue *txq = queue;
+ struct hns3_adapter *hns;
+ struct hns3_hw *hw;
+ uint16_t idx;
+
+ if (txq == NULL)
+ return;
+
+ hns = txq->hns;
+ hw = &hns->hw;
+ idx = txq->queue_id;
+ if (hw->fkq_data.tx_queues[idx]) {
+ hns3_tx_queue_release(hw->fkq_data.tx_queues[idx]);
+ hw->fkq_data.tx_queues[idx] = NULL;
+ }
+
+ /* free fake tx queue arrays */
+ if (idx == (hw->fkq_data.nb_fake_tx_queues - 1)) {
+ hw->fkq_data.nb_fake_tx_queues = 0;
+ rte_free(hw->fkq_data.tx_queues);
+ hw->fkq_data.tx_queues = NULL;
+ }
+}
+
+static void
+hns3_free_rx_queues(struct rte_eth_dev *dev)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ struct hns3_fake_queue_data *fkq_data;
+ struct hns3_hw *hw = &hns->hw;
+ uint16_t nb_rx_q;
uint16_t i;
- if (dev->data->rx_queues)
- for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ nb_rx_q = hw->data->nb_rx_queues;
+ for (i = 0; i < nb_rx_q; i++) {
+ if (dev->data->rx_queues[i]) {
hns3_rx_queue_release(dev->data->rx_queues[i]);
dev->data->rx_queues[i] = NULL;
}
+ }
+
+ /* Free fake Rx queues */
+ fkq_data = &hw->fkq_data;
+ for (i = 0; i < fkq_data->nb_fake_rx_queues; i++) {
+ if (fkq_data->rx_queues[i])
+ hns3_fake_rx_queue_release(fkq_data->rx_queues[i]);
+ }
+}
+
+static void
+hns3_free_tx_queues(struct rte_eth_dev *dev)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ struct hns3_fake_queue_data *fkq_data;
+ struct hns3_hw *hw = &hns->hw;
+ uint16_t nb_tx_q;
+ uint16_t i;
- if (dev->data->tx_queues)
- for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ nb_tx_q = hw->data->nb_tx_queues;
+ for (i = 0; i < nb_tx_q; i++) {
+ if (dev->data->tx_queues[i]) {
hns3_tx_queue_release(dev->data->tx_queues[i]);
dev->data->tx_queues[i] = NULL;
}
+ }
+
+ /* Free fake Tx queues */
+ fkq_data = &hw->fkq_data;
+ for (i = 0; i < fkq_data->nb_fake_tx_queues; i++) {
+ if (fkq_data->tx_queues[i])
+ hns3_fake_tx_queue_release(fkq_data->tx_queues[i]);
+ }
+}
+
+void
+hns3_free_all_queues(struct rte_eth_dev *dev)
+{
+ hns3_free_rx_queues(dev);
+ hns3_free_tx_queues(dev);
}
static int
static void
hns3_enable_all_queues(struct hns3_hw *hw, bool en)
{
+ uint16_t nb_rx_q = hw->data->nb_rx_queues;
+ uint16_t nb_tx_q = hw->data->nb_tx_queues;
struct hns3_rx_queue *rxq;
struct hns3_tx_queue *txq;
uint32_t rcb_reg;
int i;
- for (i = 0; i < hw->data->nb_rx_queues; i++) {
- rxq = hw->data->rx_queues[i];
- txq = hw->data->tx_queues[i];
+ for (i = 0; i < hw->cfg_max_queues; i++) {
+ if (i < nb_rx_q)
+ rxq = hw->data->rx_queues[i];
+ else
+ rxq = hw->fkq_data.rx_queues[i - nb_rx_q];
+ if (i < nb_tx_q)
+ txq = hw->data->tx_queues[i];
+ else
+ txq = hw->fkq_data.tx_queues[i - nb_tx_q];
if (rxq == NULL || txq == NULL ||
(en && (rxq->rx_deferred_start || txq->tx_deferred_start)))
continue;
+
rcb_reg = hns3_read_dev(rxq, HNS3_RING_EN_REG);
if (en)
rcb_reg |= BIT(HNS3_RING_EN_B);
hns3_reset_all_queues(struct hns3_adapter *hns)
{
struct hns3_hw *hw = &hns->hw;
- int ret;
- uint16_t i;
+ int ret, i;
- for (i = 0; i < hw->data->nb_rx_queues; i++) {
+ for (i = 0; i < hw->cfg_max_queues; i++) {
ret = hns3_reset_queue(hns, i);
if (ret) {
hns3_err(hw, "Failed to reset No.%d queue: %d", i, ret);
}
void
-hns3_tqp_intr_enable(struct hns3_hw *hw, uint16_t tpq_int_num, bool en)
+hns3_set_queue_intr_gl(struct hns3_hw *hw, uint16_t queue_id,
+ uint8_t gl_idx, uint16_t gl_value)
{
+ uint32_t offset[] = {HNS3_TQP_INTR_GL0_REG,
+ HNS3_TQP_INTR_GL1_REG,
+ HNS3_TQP_INTR_GL2_REG};
uint32_t addr, value;
- addr = HNS3_TQP_INTR_CTRL_REG + tpq_int_num * HNS3_VECTOR_REG_OFFSET;
+ if (gl_idx >= RTE_DIM(offset) || gl_value > HNS3_TQP_INTR_GL_MAX)
+ return;
+
+ addr = offset[gl_idx] + queue_id * HNS3_TQP_INTR_REG_SIZE;
+ value = HNS3_GL_USEC_TO_REG(gl_value);
+
+ hns3_write_dev(hw, addr, value);
+}
+
+void
+hns3_set_queue_intr_rl(struct hns3_hw *hw, uint16_t queue_id, uint16_t rl_value)
+{
+ uint32_t addr, value;
+
+ if (rl_value > HNS3_TQP_INTR_RL_MAX)
+ return;
+
+ addr = HNS3_TQP_INTR_RL_REG + queue_id * HNS3_TQP_INTR_REG_SIZE;
+ value = HNS3_RL_USEC_TO_REG(rl_value);
+ if (value > 0)
+ value |= HNS3_TQP_INTR_RL_ENABLE_MASK;
+
+ hns3_write_dev(hw, addr, value);
+}
+
+static void
+hns3_queue_intr_enable(struct hns3_hw *hw, uint16_t queue_id, bool en)
+{
+ uint32_t addr, value;
+
+ addr = HNS3_TQP_INTR_CTRL_REG + queue_id * HNS3_TQP_INTR_REG_SIZE;
value = en ? 1 : 0;
hns3_write_dev(hw, addr, value);
if (dev->data->dev_conf.intr_conf.rxq == 0)
return -ENOTSUP;
- /* enable the vectors */
- hns3_tqp_intr_enable(hw, queue_id, true);
+ hns3_queue_intr_enable(hw, queue_id, true);
return rte_intr_ack(intr_handle);
}
if (dev->data->dev_conf.intr_conf.rxq == 0)
return -ENOTSUP;
- /* disable the vectors */
- hns3_tqp_intr_enable(hw, queue_id, false);
+ hns3_queue_intr_enable(hw, queue_id, false);
return 0;
}
PMD_INIT_FUNC_TRACE();
- rxq = hw->data->rx_queues[idx];
-
+ rxq = (struct hns3_rx_queue *)hw->data->rx_queues[idx];
ret = hns3_alloc_rx_queue_mbufs(hw, rxq);
if (ret) {
hns3_err(hw, "Failed to alloc mbuf for No.%d rx queue: %d",
- idx, ret);
+ idx, ret);
return ret;
}
rxq->next_to_use = 0;
rxq->next_to_clean = 0;
+ rxq->nb_rx_hold = 0;
hns3_init_rx_queue_hw(rxq);
return 0;
}
static void
-hns3_dev_tx_queue_start(struct hns3_adapter *hns, uint16_t idx)
+hns3_fake_rx_queue_start(struct hns3_adapter *hns, uint16_t idx)
{
struct hns3_hw *hw = &hns->hw;
- struct hns3_tx_queue *txq;
+ struct hns3_rx_queue *rxq;
+
+ rxq = (struct hns3_rx_queue *)hw->fkq_data.rx_queues[idx];
+ rxq->next_to_use = 0;
+ rxq->next_to_clean = 0;
+ rxq->nb_rx_hold = 0;
+ hns3_init_rx_queue_hw(rxq);
+}
+
+static void
+hns3_init_tx_queue(struct hns3_tx_queue *queue)
+{
+ struct hns3_tx_queue *txq = queue;
struct hns3_desc *desc;
int i;
- txq = hw->data->tx_queues[idx];
-
/* Clear tx bd */
desc = txq->tx_ring;
for (i = 0; i < txq->nb_tx_desc; i++) {
txq->next_to_use = 0;
txq->next_to_clean = 0;
- txq->tx_bd_ready = txq->nb_tx_desc;
+ txq->tx_bd_ready = txq->nb_tx_desc - 1;
hns3_init_tx_queue_hw(txq);
}
+static void
+hns3_dev_tx_queue_start(struct hns3_adapter *hns, uint16_t idx)
+{
+ struct hns3_hw *hw = &hns->hw;
+ struct hns3_tx_queue *txq;
+
+ txq = (struct hns3_tx_queue *)hw->data->tx_queues[idx];
+ hns3_init_tx_queue(txq);
+}
+
+static void
+hns3_fake_tx_queue_start(struct hns3_adapter *hns, uint16_t idx)
+{
+ struct hns3_hw *hw = &hns->hw;
+ struct hns3_tx_queue *txq;
+
+ txq = (struct hns3_tx_queue *)hw->fkq_data.tx_queues[idx];
+ hns3_init_tx_queue(txq);
+}
+
static void
hns3_init_tx_ring_tc(struct hns3_adapter *hns)
{
for (j = 0; j < tc_queue->tqp_count; j++) {
num = tc_queue->tqp_offset + j;
- txq = hw->data->tx_queues[num];
+ txq = (struct hns3_tx_queue *)hw->data->tx_queues[num];
if (txq == NULL)
continue;
}
}
-int
-hns3_start_queues(struct hns3_adapter *hns, bool reset_queue)
+static int
+hns3_start_rx_queues(struct hns3_adapter *hns)
{
struct hns3_hw *hw = &hns->hw;
- struct rte_eth_dev_data *dev_data = hw->data;
struct hns3_rx_queue *rxq;
- struct hns3_tx_queue *txq;
+ int i, j;
int ret;
- int i;
- int j;
/* Initialize RSS for queues */
ret = hns3_config_rss(hns);
return ret;
}
- if (reset_queue) {
- ret = hns3_reset_all_queues(hns);
- if (ret) {
- hns3_err(hw, "Failed to reset all queues %d", ret);
- return ret;
- }
- }
-
- /*
- * Hardware does not support where the number of rx and tx queues is
- * not equal in hip08. In .dev_configure callback function we will
- * check the two values, here we think that the number of rx and tx
- * queues is equal.
- */
for (i = 0; i < hw->data->nb_rx_queues; i++) {
- rxq = dev_data->rx_queues[i];
- txq = dev_data->tx_queues[i];
- if (rxq == NULL || txq == NULL || rxq->rx_deferred_start ||
- txq->tx_deferred_start)
+ rxq = (struct hns3_rx_queue *)hw->data->rx_queues[i];
+ if (rxq == NULL || rxq->rx_deferred_start)
continue;
-
ret = hns3_dev_rx_queue_start(hns, i);
if (ret) {
hns3_err(hw, "Failed to start No.%d rx queue: %d", i,
ret);
goto out;
}
- hns3_dev_tx_queue_start(hns, i);
}
- hns3_init_tx_ring_tc(hns);
- hns3_enable_all_queues(hw, true);
+ for (i = 0; i < hw->fkq_data.nb_fake_rx_queues; i++) {
+ rxq = (struct hns3_rx_queue *)hw->fkq_data.rx_queues[i];
+ if (rxq == NULL || rxq->rx_deferred_start)
+ continue;
+ hns3_fake_rx_queue_start(hns, i);
+ }
return 0;
out:
for (j = 0; j < i; j++) {
- rxq = dev_data->rx_queues[j];
+ rxq = (struct hns3_rx_queue *)hw->data->rx_queues[j];
hns3_rx_queue_release_mbufs(rxq);
}
return ret;
}
+static void
+hns3_start_tx_queues(struct hns3_adapter *hns)
+{
+ struct hns3_hw *hw = &hns->hw;
+ struct hns3_tx_queue *txq;
+ int i;
+
+ for (i = 0; i < hw->data->nb_tx_queues; i++) {
+ txq = (struct hns3_tx_queue *)hw->data->tx_queues[i];
+ if (txq == NULL || txq->tx_deferred_start)
+ continue;
+ hns3_dev_tx_queue_start(hns, i);
+ }
+
+ for (i = 0; i < hw->fkq_data.nb_fake_tx_queues; i++) {
+ txq = (struct hns3_tx_queue *)hw->fkq_data.tx_queues[i];
+ if (txq == NULL || txq->tx_deferred_start)
+ continue;
+ hns3_fake_tx_queue_start(hns, i);
+ }
+
+ hns3_init_tx_ring_tc(hns);
+}
+
+int
+hns3_start_queues(struct hns3_adapter *hns, bool reset_queue)
+{
+ struct hns3_hw *hw = &hns->hw;
+ int ret;
+
+ if (reset_queue) {
+ ret = hns3_reset_all_queues(hns);
+ if (ret) {
+ hns3_err(hw, "Failed to reset all queues %d", ret);
+ return ret;
+ }
+ }
+
+ ret = hns3_start_rx_queues(hns);
+ if (ret) {
+ hns3_err(hw, "Failed to start rx queues: %d", ret);
+ return ret;
+ }
+
+ hns3_start_tx_queues(hns);
+ hns3_enable_all_queues(hw, true);
+
+ return 0;
+}
+
int
hns3_stop_queues(struct hns3_adapter *hns, bool reset_queue)
{
return 0;
}
+static void*
+hns3_alloc_rxq_and_dma_zone(struct rte_eth_dev *dev,
+ struct hns3_queue_info *q_info)
+{
+ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ const struct rte_memzone *rx_mz;
+ struct hns3_rx_queue *rxq;
+ unsigned int rx_desc;
+
+ rxq = rte_zmalloc_socket(q_info->type, sizeof(struct hns3_rx_queue),
+ RTE_CACHE_LINE_SIZE, q_info->socket_id);
+ if (rxq == NULL) {
+ hns3_err(hw, "Failed to allocate memory for No.%d rx ring!",
+ q_info->idx);
+ return NULL;
+ }
+
+ /* Allocate rx ring hardware descriptors. */
+ rxq->queue_id = q_info->idx;
+ rxq->nb_rx_desc = q_info->nb_desc;
+ rx_desc = rxq->nb_rx_desc * sizeof(struct hns3_desc);
+ rx_mz = rte_eth_dma_zone_reserve(dev, q_info->ring_name, q_info->idx,
+ rx_desc, HNS3_RING_BASE_ALIGN,
+ q_info->socket_id);
+ if (rx_mz == NULL) {
+ hns3_err(hw, "Failed to reserve DMA memory for No.%d rx ring!",
+ q_info->idx);
+ hns3_rx_queue_release(rxq);
+ return NULL;
+ }
+ rxq->mz = rx_mz;
+ rxq->rx_ring = (struct hns3_desc *)rx_mz->addr;
+ rxq->rx_ring_phys_addr = rx_mz->iova;
+
+ hns3_dbg(hw, "No.%d rx descriptors iova 0x%" PRIx64, q_info->idx,
+ rxq->rx_ring_phys_addr);
+
+ return rxq;
+}
+
+static int
+hns3_fake_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
+ uint16_t nb_desc, unsigned int socket_id)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ struct hns3_hw *hw = &hns->hw;
+ struct hns3_queue_info q_info;
+ struct hns3_rx_queue *rxq;
+ uint16_t nb_rx_q;
+
+ if (hw->fkq_data.rx_queues[idx]) {
+ hns3_rx_queue_release(hw->fkq_data.rx_queues[idx]);
+ hw->fkq_data.rx_queues[idx] = NULL;
+ }
+
+ q_info.idx = idx;
+ q_info.socket_id = socket_id;
+ q_info.nb_desc = nb_desc;
+ q_info.type = "hns3 fake RX queue";
+ q_info.ring_name = "rx_fake_ring";
+ rxq = hns3_alloc_rxq_and_dma_zone(dev, &q_info);
+ if (rxq == NULL) {
+ hns3_err(hw, "Failed to setup No.%d fake rx ring.", idx);
+ return -ENOMEM;
+ }
+
+ /* Don't need alloc sw_ring, because upper applications don't use it */
+ rxq->sw_ring = NULL;
+
+ rxq->hns = hns;
+ rxq->rx_deferred_start = false;
+ rxq->port_id = dev->data->port_id;
+ rxq->configured = true;
+ nb_rx_q = dev->data->nb_rx_queues;
+ rxq->io_base = (void *)((char *)hw->io_base + HNS3_TQP_REG_OFFSET +
+ (nb_rx_q + idx) * HNS3_TQP_REG_SIZE);
+ rxq->rx_buf_len = hw->rx_buf_len;
+
+ rte_spinlock_lock(&hw->lock);
+ hw->fkq_data.rx_queues[idx] = rxq;
+ rte_spinlock_unlock(&hw->lock);
+
+ return 0;
+}
+
+static void*
+hns3_alloc_txq_and_dma_zone(struct rte_eth_dev *dev,
+ struct hns3_queue_info *q_info)
+{
+ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ const struct rte_memzone *tx_mz;
+ struct hns3_tx_queue *txq;
+ struct hns3_desc *desc;
+ unsigned int tx_desc;
+ int i;
+
+ txq = rte_zmalloc_socket(q_info->type, sizeof(struct hns3_tx_queue),
+ RTE_CACHE_LINE_SIZE, q_info->socket_id);
+ if (txq == NULL) {
+ hns3_err(hw, "Failed to allocate memory for No.%d tx ring!",
+ q_info->idx);
+ return NULL;
+ }
+
+ /* Allocate tx ring hardware descriptors. */
+ txq->queue_id = q_info->idx;
+ txq->nb_tx_desc = q_info->nb_desc;
+ tx_desc = txq->nb_tx_desc * sizeof(struct hns3_desc);
+ tx_mz = rte_eth_dma_zone_reserve(dev, q_info->ring_name, q_info->idx,
+ tx_desc, HNS3_RING_BASE_ALIGN,
+ q_info->socket_id);
+ if (tx_mz == NULL) {
+ hns3_err(hw, "Failed to reserve DMA memory for No.%d tx ring!",
+ q_info->idx);
+ hns3_tx_queue_release(txq);
+ return NULL;
+ }
+ txq->mz = tx_mz;
+ txq->tx_ring = (struct hns3_desc *)tx_mz->addr;
+ txq->tx_ring_phys_addr = tx_mz->iova;
+
+ hns3_dbg(hw, "No.%d tx descriptors iova 0x%" PRIx64, q_info->idx,
+ txq->tx_ring_phys_addr);
+
+ /* Clear tx bd */
+ desc = txq->tx_ring;
+ for (i = 0; i < txq->nb_tx_desc; i++) {
+ desc->tx.tp_fe_sc_vld_ra_ri = 0;
+ desc++;
+ }
+
+ return txq;
+}
+
+static int
+hns3_fake_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
+ uint16_t nb_desc, unsigned int socket_id)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ struct hns3_hw *hw = &hns->hw;
+ struct hns3_queue_info q_info;
+ struct hns3_tx_queue *txq;
+ uint16_t nb_tx_q;
+
+ if (hw->fkq_data.tx_queues[idx] != NULL) {
+ hns3_tx_queue_release(hw->fkq_data.tx_queues[idx]);
+ hw->fkq_data.tx_queues[idx] = NULL;
+ }
+
+ q_info.idx = idx;
+ q_info.socket_id = socket_id;
+ q_info.nb_desc = nb_desc;
+ q_info.type = "hns3 fake TX queue";
+ q_info.ring_name = "tx_fake_ring";
+ txq = hns3_alloc_txq_and_dma_zone(dev, &q_info);
+ if (txq == NULL) {
+ hns3_err(hw, "Failed to setup No.%d fake tx ring.", idx);
+ return -ENOMEM;
+ }
+
+ /* Don't need alloc sw_ring, because upper applications don't use it */
+ txq->sw_ring = NULL;
+
+ txq->hns = hns;
+ txq->tx_deferred_start = false;
+ txq->port_id = dev->data->port_id;
+ txq->configured = true;
+ nb_tx_q = dev->data->nb_tx_queues;
+ txq->io_base = (void *)((char *)hw->io_base + HNS3_TQP_REG_OFFSET +
+ (nb_tx_q + idx) * HNS3_TQP_REG_SIZE);
+
+ rte_spinlock_lock(&hw->lock);
+ hw->fkq_data.tx_queues[idx] = txq;
+ rte_spinlock_unlock(&hw->lock);
+
+ return 0;
+}
+
+static int
+hns3_fake_rx_queue_config(struct hns3_hw *hw, uint16_t nb_queues)
+{
+ uint16_t old_nb_queues = hw->fkq_data.nb_fake_rx_queues;
+ void **rxq;
+ uint8_t i;
+
+ if (hw->fkq_data.rx_queues == NULL && nb_queues != 0) {
+ /* first time configuration */
+ uint32_t size;
+ size = sizeof(hw->fkq_data.rx_queues[0]) * nb_queues;
+ hw->fkq_data.rx_queues = rte_zmalloc("fake_rx_queues", size,
+ RTE_CACHE_LINE_SIZE);
+ if (hw->fkq_data.rx_queues == NULL) {
+ hw->fkq_data.nb_fake_rx_queues = 0;
+ return -ENOMEM;
+ }
+ } else if (hw->fkq_data.rx_queues != NULL && nb_queues != 0) {
+ /* re-configure */
+ rxq = hw->fkq_data.rx_queues;
+ for (i = nb_queues; i < old_nb_queues; i++)
+ hns3_dev_rx_queue_release(rxq[i]);
+
+ rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
+ RTE_CACHE_LINE_SIZE);
+ if (rxq == NULL)
+ return -ENOMEM;
+ if (nb_queues > old_nb_queues) {
+ uint16_t new_qs = nb_queues - old_nb_queues;
+ memset(rxq + old_nb_queues, 0, sizeof(rxq[0]) * new_qs);
+ }
+
+ hw->fkq_data.rx_queues = rxq;
+ } else if (hw->fkq_data.rx_queues != NULL && nb_queues == 0) {
+ rxq = hw->fkq_data.rx_queues;
+ for (i = nb_queues; i < old_nb_queues; i++)
+ hns3_dev_rx_queue_release(rxq[i]);
+
+ rte_free(hw->fkq_data.rx_queues);
+ hw->fkq_data.rx_queues = NULL;
+ }
+
+ hw->fkq_data.nb_fake_rx_queues = nb_queues;
+
+ return 0;
+}
+
+static int
+hns3_fake_tx_queue_config(struct hns3_hw *hw, uint16_t nb_queues)
+{
+ uint16_t old_nb_queues = hw->fkq_data.nb_fake_tx_queues;
+ void **txq;
+ uint8_t i;
+
+ if (hw->fkq_data.tx_queues == NULL && nb_queues != 0) {
+ /* first time configuration */
+ uint32_t size;
+ size = sizeof(hw->fkq_data.tx_queues[0]) * nb_queues;
+ hw->fkq_data.tx_queues = rte_zmalloc("fake_tx_queues", size,
+ RTE_CACHE_LINE_SIZE);
+ if (hw->fkq_data.tx_queues == NULL) {
+ hw->fkq_data.nb_fake_tx_queues = 0;
+ return -ENOMEM;
+ }
+ } else if (hw->fkq_data.tx_queues != NULL && nb_queues != 0) {
+ /* re-configure */
+ txq = hw->fkq_data.tx_queues;
+ for (i = nb_queues; i < old_nb_queues; i++)
+ hns3_dev_tx_queue_release(txq[i]);
+ txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
+ RTE_CACHE_LINE_SIZE);
+ if (txq == NULL)
+ return -ENOMEM;
+ if (nb_queues > old_nb_queues) {
+ uint16_t new_qs = nb_queues - old_nb_queues;
+ memset(txq + old_nb_queues, 0, sizeof(txq[0]) * new_qs);
+ }
+
+ hw->fkq_data.tx_queues = txq;
+ } else if (hw->fkq_data.tx_queues != NULL && nb_queues == 0) {
+ txq = hw->fkq_data.tx_queues;
+ for (i = nb_queues; i < old_nb_queues; i++)
+ hns3_dev_tx_queue_release(txq[i]);
+
+ rte_free(hw->fkq_data.tx_queues);
+ hw->fkq_data.tx_queues = NULL;
+ }
+ hw->fkq_data.nb_fake_tx_queues = nb_queues;
+
+ return 0;
+}
+
+int
+hns3_set_fake_rx_or_tx_queues(struct rte_eth_dev *dev, uint16_t nb_rx_q,
+ uint16_t nb_tx_q)
+{
+ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ uint16_t rx_need_add_nb_q;
+ uint16_t tx_need_add_nb_q;
+ uint16_t port_id;
+ uint16_t q;
+ int ret;
+
+ /* Setup new number of fake RX/TX queues and reconfigure device. */
+ hw->cfg_max_queues = RTE_MAX(nb_rx_q, nb_tx_q);
+ rx_need_add_nb_q = hw->cfg_max_queues - nb_rx_q;
+ tx_need_add_nb_q = hw->cfg_max_queues - nb_tx_q;
+ ret = hns3_fake_rx_queue_config(hw, rx_need_add_nb_q);
+ if (ret) {
+ hns3_err(hw, "Fail to configure fake rx queues: %d", ret);
+ goto cfg_fake_rx_q_fail;
+ }
+
+ ret = hns3_fake_tx_queue_config(hw, tx_need_add_nb_q);
+ if (ret) {
+ hns3_err(hw, "Fail to configure fake rx queues: %d", ret);
+ goto cfg_fake_tx_q_fail;
+ }
+
+ /* Allocate and set up fake RX queue per Ethernet port. */
+ port_id = hw->data->port_id;
+ for (q = 0; q < rx_need_add_nb_q; q++) {
+ ret = hns3_fake_rx_queue_setup(dev, q, HNS3_MIN_RING_DESC,
+ rte_eth_dev_socket_id(port_id));
+ if (ret)
+ goto setup_fake_rx_q_fail;
+ }
+
+ /* Allocate and set up fake TX queue per Ethernet port. */
+ for (q = 0; q < tx_need_add_nb_q; q++) {
+ ret = hns3_fake_tx_queue_setup(dev, q, HNS3_MIN_RING_DESC,
+ rte_eth_dev_socket_id(port_id));
+ if (ret)
+ goto setup_fake_tx_q_fail;
+ }
+
+ return 0;
+
+setup_fake_tx_q_fail:
+setup_fake_rx_q_fail:
+ (void)hns3_fake_tx_queue_config(hw, 0);
+cfg_fake_tx_q_fail:
+ (void)hns3_fake_rx_queue_config(hw, 0);
+cfg_fake_rx_q_fail:
+ hw->cfg_max_queues = 0;
+
+ return ret;
+}
+
void
hns3_dev_release_mbufs(struct hns3_adapter *hns)
{
struct rte_mempool *mp)
{
struct hns3_adapter *hns = dev->data->dev_private;
- const struct rte_memzone *rx_mz;
struct hns3_hw *hw = &hns->hw;
+ struct hns3_queue_info q_info;
struct hns3_rx_queue *rxq;
- unsigned int desc_size = sizeof(struct hns3_desc);
- unsigned int rx_desc;
int rx_entry_len;
if (dev->data->dev_started) {
dev->data->rx_queues[idx] = NULL;
}
- rxq = rte_zmalloc_socket("hns3 RX queue", sizeof(struct hns3_rx_queue),
- RTE_CACHE_LINE_SIZE, socket_id);
+ q_info.idx = idx;
+ q_info.socket_id = socket_id;
+ q_info.nb_desc = nb_desc;
+ q_info.type = "hns3 RX queue";
+ q_info.ring_name = "rx_ring";
+ rxq = hns3_alloc_rxq_and_dma_zone(dev, &q_info);
if (rxq == NULL) {
- hns3_err(hw, "Failed to allocate memory for rx queue!");
+ hns3_err(hw,
+ "Failed to alloc mem and reserve DMA mem for rx ring!");
return -ENOMEM;
}
rxq->hns = hns;
rxq->mb_pool = mp;
- rxq->nb_rx_desc = nb_desc;
- rxq->queue_id = idx;
if (conf->rx_free_thresh <= 0)
rxq->rx_free_thresh = DEFAULT_RX_FREE_THRESH;
else
return -ENOMEM;
}
- /* Allocate rx ring hardware descriptors. */
- rx_desc = rxq->nb_rx_desc * desc_size;
- rx_mz = rte_eth_dma_zone_reserve(dev, "rx_ring", idx, rx_desc,
- HNS3_RING_BASE_ALIGN, socket_id);
- if (rx_mz == NULL) {
- hns3_err(hw, "Failed to reserve DMA memory for No.%d rx ring!",
- idx);
- hns3_rx_queue_release(rxq);
- return -ENOMEM;
- }
- rxq->mz = rx_mz;
- rxq->rx_ring = (struct hns3_desc *)rx_mz->addr;
- rxq->rx_ring_phys_addr = rx_mz->iova;
-
- hns3_dbg(hw, "No.%d rx descriptors iova 0x%" PRIx64, idx,
- rxq->rx_ring_phys_addr);
-
rxq->next_to_use = 0;
rxq->next_to_clean = 0;
rxq->nb_rx_hold = 0;
rxq->io_base = (void *)((char *)hw->io_base + HNS3_TQP_REG_OFFSET +
idx * HNS3_TQP_REG_SIZE);
rxq->rx_buf_len = hw->rx_buf_len;
- rxq->non_vld_descs = 0;
rxq->l2_errors = 0;
rxq->pkt_len_errors = 0;
rxq->l3_csum_erros = 0;
uint16_t
hns3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
+ volatile struct hns3_desc *rx_ring; /* RX ring (desc) */
+ volatile struct hns3_desc *rxdp; /* pointer of the current desc */
struct hns3_rx_queue *rxq; /* RX queue */
- struct hns3_desc *rx_ring; /* RX ring (desc) */
struct hns3_entry *sw_ring;
struct hns3_entry *rxe;
- struct hns3_desc *rxdp; /* pointer of the current desc */
struct rte_mbuf *first_seg;
struct rte_mbuf *last_seg;
+ struct hns3_desc rxd;
struct rte_mbuf *nmb; /* pointer of the new mbuf */
struct rte_mbuf *rxm;
struct rte_eth_dev *dev;
uint16_t pkt_len;
uint16_t nb_rx;
uint16_t rx_id;
- int num; /* num of desc in ring */
int ret;
nb_rx = 0;
last_seg = rxq->pkt_last_seg;
sw_ring = rxq->sw_ring;
- /* Get num of packets in descriptor ring */
- num = hns3_read_dev(rxq, HNS3_RING_RX_FBDNUM_REG);
- while (nb_rx_bd < num && nb_rx < nb_pkts) {
+ while (nb_rx < nb_pkts) {
rxdp = &rx_ring[rx_id];
bd_base_info = rte_le_to_cpu_32(rxdp->rx.bd_base_info);
- if (unlikely(!hns3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
- rxq->non_vld_descs++;
+ if (unlikely(!hns3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
break;
- }
+ /*
+ * The interactive process between software and hardware of
+ * receiving a new packet in hns3 network engine:
+ * 1. Hardware network engine firstly writes the packet content
+ * to the memory pointed by the 'addr' field of the Rx Buffer
+ * Descriptor, secondly fills the result of parsing the
+ * packet include the valid field into the Rx Buffer
+ * Descriptor in one write operation.
+ * 2. Driver reads the Rx BD's valid field in the loop to check
+ * whether it's valid, if valid then assign a new address to
+ * the addr field, clear the valid field, get the other
+ * information of the packet by parsing Rx BD's other fields,
+ * finally write back the number of Rx BDs processed by the
+ * driver to the HNS3_RING_RX_HEAD_REG register to inform
+ * hardware.
+ * In the above process, the ordering is very important. We must
+ * make sure that CPU read Rx BD's other fields only after the
+ * Rx BD is valid.
+ *
+ * There are two type of re-ordering: compiler re-ordering and
+ * CPU re-ordering under the ARMv8 architecture.
+ * 1. we use volatile to deal with compiler re-ordering, so you
+ * can see that rx_ring/rxdp defined with volatile.
+ * 2. we commonly use memory barrier to deal with CPU
+ * re-ordering, but the cost is high.
+ *
+ * In order to solve the high cost of using memory barrier, we
+ * use the data dependency order under the ARMv8 architecture,
+ * for example:
+ * instr01: load A
+ * instr02: load B <- A
+ * the instr02 will always execute after instr01.
+ *
+ * To construct the data dependency ordering, we use the
+ * following assignment:
+ * rxd = rxdp[(bd_base_info & (1u << HNS3_RXD_VLD_B)) -
+ * (1u<<HNS3_RXD_VLD_B)]
+ * Using gcc compiler under the ARMv8 architecture, the related
+ * assembly code example as follows:
+ * note: (1u << HNS3_RXD_VLD_B) equal 0x10
+ * instr01: ldr w26, [x22, #28] --read bd_base_info
+ * instr02: and w0, w26, #0x10 --calc bd_base_info & 0x10
+ * instr03: sub w0, w0, #0x10 --calc (bd_base_info &
+ * 0x10) - 0x10
+ * instr04: add x0, x22, x0, lsl #5 --calc copy source addr
+ * instr05: ldp x2, x3, [x0]
+ * instr06: stp x2, x3, [x29, #256] --copy BD's [0 ~ 15]B
+ * instr07: ldp x4, x5, [x0, #16]
+ * instr08: stp x4, x5, [x29, #272] --copy BD's [16 ~ 31]B
+ * the instr05~08 depend on x0's value, x0 depent on w26's
+ * value, the w26 is the bd_base_info, this form the data
+ * dependency ordering.
+ * note: if BD is valid, (bd_base_info & (1u<<HNS3_RXD_VLD_B)) -
+ * (1u<<HNS3_RXD_VLD_B) will always zero, so the
+ * assignment is correct.
+ *
+ * So we use the data dependency ordering instead of memory
+ * barrier to improve receive performance.
+ */
+ rxd = rxdp[(bd_base_info & (1u << HNS3_RXD_VLD_B)) -
+ (1u << HNS3_RXD_VLD_B)];
nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
if (unlikely(nmb == NULL)) {
nb_rx_bd++;
rxe = &sw_ring[rx_id];
rx_id++;
- if (rx_id == rxq->nb_rx_desc)
+ if (unlikely(rx_id == rxq->nb_rx_desc))
rx_id = 0;
rte_prefetch0(sw_ring[rx_id].mbuf);
rxe->mbuf = nmb;
dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
- rxdp->addr = dma_addr;
rxdp->rx.bd_base_info = 0;
+ rxdp->addr = dma_addr;
- rte_cio_rmb();
/* Load remained descriptor data and extract necessary fields */
- data_len = (uint16_t)(rte_le_to_cpu_16(rxdp->rx.size));
- l234_info = rte_le_to_cpu_32(rxdp->rx.l234_info);
- ol_info = rte_le_to_cpu_32(rxdp->rx.ol_info);
+ data_len = (uint16_t)(rte_le_to_cpu_16(rxd.rx.size));
+ l234_info = rte_le_to_cpu_32(rxd.rx.l234_info);
+ ol_info = rte_le_to_cpu_32(rxd.rx.ol_info);
if (first_seg == NULL) {
first_seg = rxm;
}
/* The last buffer of the received packet */
- pkt_len = (uint16_t)(rte_le_to_cpu_16(rxdp->rx.pkt_len));
+ pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.rx.pkt_len));
first_seg->pkt_len = pkt_len;
first_seg->port = rxq->port_id;
- first_seg->hash.rss = rte_le_to_cpu_32(rxdp->rx.rss_hash);
+ first_seg->hash.rss = rte_le_to_cpu_32(rxd.rx.rss_hash);
first_seg->ol_flags |= PKT_RX_RSS_HASH;
if (unlikely(hns3_get_bit(bd_base_info, HNS3_RXD_LUM_B))) {
first_seg->hash.fdir.hi =
- rte_le_to_cpu_32(rxdp->rx.fd_id);
+ rte_le_to_cpu_32(rxd.rx.fd_id);
first_seg->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
}
rxm->next = NULL;
hns3_rx_set_cksum_flag(rxm, first_seg->packet_type,
cksum_err);
- first_seg->vlan_tci = rte_le_to_cpu_16(rxdp->rx.vlan_tag);
+ first_seg->vlan_tci = rte_le_to_cpu_16(rxd.rx.vlan_tag);
first_seg->vlan_tci_outer =
- rte_le_to_cpu_16(rxdp->rx.ot_vlan_tag);
+ rte_le_to_cpu_16(rxd.rx.ot_vlan_tag);
rx_pkts[nb_rx++] = first_seg;
first_seg = NULL;
continue;
rxq->next_to_clean = rx_id;
rxq->pkt_first_seg = first_seg;
rxq->pkt_last_seg = last_seg;
- hns3_clean_rx_buffers(rxq, nb_rx_bd);
+
+ nb_rx_bd = nb_rx_bd + rxq->nb_rx_hold;
+ if (nb_rx_bd > rxq->rx_free_thresh) {
+ hns3_clean_rx_buffers(rxq, nb_rx_bd);
+ nb_rx_bd = 0;
+ }
+ rxq->nb_rx_hold = nb_rx_bd;
return nb_rx;
}
unsigned int socket_id, const struct rte_eth_txconf *conf)
{
struct hns3_adapter *hns = dev->data->dev_private;
- const struct rte_memzone *tx_mz;
struct hns3_hw *hw = &hns->hw;
+ struct hns3_queue_info q_info;
struct hns3_tx_queue *txq;
- struct hns3_desc *desc;
- unsigned int desc_size = sizeof(struct hns3_desc);
- unsigned int tx_desc;
int tx_entry_len;
- int i;
if (dev->data->dev_started) {
hns3_err(hw, "tx_queue_setup after dev_start no supported");
dev->data->tx_queues[idx] = NULL;
}
- txq = rte_zmalloc_socket("hns3 TX queue", sizeof(struct hns3_tx_queue),
- RTE_CACHE_LINE_SIZE, socket_id);
+ q_info.idx = idx;
+ q_info.socket_id = socket_id;
+ q_info.nb_desc = nb_desc;
+ q_info.type = "hns3 TX queue";
+ q_info.ring_name = "tx_ring";
+ txq = hns3_alloc_txq_and_dma_zone(dev, &q_info);
if (txq == NULL) {
- hns3_err(hw, "Failed to allocate memory for tx queue!");
+ hns3_err(hw,
+ "Failed to alloc mem and reserve DMA mem for tx ring!");
return -ENOMEM;
}
- txq->nb_tx_desc = nb_desc;
- txq->queue_id = idx;
txq->tx_deferred_start = conf->tx_deferred_start;
-
tx_entry_len = sizeof(struct hns3_entry) * txq->nb_tx_desc;
txq->sw_ring = rte_zmalloc_socket("hns3 TX sw ring", tx_entry_len,
RTE_CACHE_LINE_SIZE, socket_id);
return -ENOMEM;
}
- /* Allocate tx ring hardware descriptors. */
- tx_desc = txq->nb_tx_desc * desc_size;
- tx_mz = rte_eth_dma_zone_reserve(dev, "tx_ring", idx, tx_desc,
- HNS3_RING_BASE_ALIGN, socket_id);
- if (tx_mz == NULL) {
- hns3_err(hw, "Failed to reserve DMA memory for No.%d tx ring!",
- idx);
- hns3_tx_queue_release(txq);
- return -ENOMEM;
- }
- txq->mz = tx_mz;
- txq->tx_ring = (struct hns3_desc *)tx_mz->addr;
- txq->tx_ring_phys_addr = tx_mz->iova;
-
- hns3_dbg(hw, "No.%d tx descriptors iova 0x%" PRIx64, idx,
- txq->tx_ring_phys_addr);
-
- /* Clear tx bd */
- desc = txq->tx_ring;
- for (i = 0; i < txq->nb_tx_desc; i++) {
- desc->tx.tp_fe_sc_vld_ra_ri = 0;
- desc++;
- }
-
txq->hns = hns;
txq->next_to_use = 0;
txq->next_to_clean = 0;
- txq->tx_bd_ready = txq->nb_tx_desc;
+ txq->tx_bd_ready = txq->nb_tx_desc - 1;
txq->port_id = dev->data->port_id;
txq->configured = true;
txq->io_base = (void *)((char *)hw->io_base + HNS3_TQP_REG_OFFSET +
return 0;
}
-static inline int
-tx_ring_dist(struct hns3_tx_queue *txq, int begin, int end)
-{
- return (end - begin + txq->nb_tx_desc) % txq->nb_tx_desc;
-}
-
-static inline int
-tx_ring_space(struct hns3_tx_queue *txq)
-{
- return txq->nb_tx_desc -
- tx_ring_dist(txq, txq->next_to_clean, txq->next_to_use) - 1;
-}
-
static inline void
hns3_queue_xmit(struct hns3_tx_queue *txq, uint32_t buf_num)
{
struct rte_mbuf *mbuf;
while ((!hns3_get_bit(desc->tx.tp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B)) &&
- (tx_next_use != tx_next_clean || tx_bd_ready < tx_bd_max)) {
+ tx_next_use != tx_next_clean) {
mbuf = tx_bak_pkt->mbuf;
if (mbuf) {
- mbuf->next = NULL;
- rte_pktmbuf_free(mbuf);
+ rte_pktmbuf_free_seg(mbuf);
tx_bak_pkt->mbuf = NULL;
}
txq->tx_bd_ready = tx_bd_ready;
}
+static int
+hns3_tso_proc_tunnel(struct hns3_desc *desc, uint64_t ol_flags,
+ struct rte_mbuf *rxm, uint8_t *l2_len)
+{
+ uint64_t tun_flags;
+ uint8_t ol4_len;
+ uint32_t otmp;
+
+ tun_flags = ol_flags & PKT_TX_TUNNEL_MASK;
+ if (tun_flags == 0)
+ return 0;
+
+ otmp = rte_le_to_cpu_32(desc->tx.ol_type_vlan_len_msec);
+ switch (tun_flags) {
+ case PKT_TX_TUNNEL_GENEVE:
+ case PKT_TX_TUNNEL_VXLAN:
+ *l2_len = rxm->l2_len - RTE_ETHER_VXLAN_HLEN;
+ break;
+ case PKT_TX_TUNNEL_GRE:
+ /*
+ * OL4 header size, defined in 4 Bytes, it contains outer
+ * L4(GRE) length and tunneling length.
+ */
+ ol4_len = hns3_get_field(otmp, HNS3_TXD_L4LEN_M,
+ HNS3_TXD_L4LEN_S);
+ *l2_len = rxm->l2_len - (ol4_len << HNS3_L4_LEN_UNIT);
+ break;
+ default:
+ /* For non UDP / GRE tunneling, drop the tunnel packet */
+ return -EINVAL;
+ }
+ hns3_set_field(otmp, HNS3_TXD_L2LEN_M, HNS3_TXD_L2LEN_S,
+ rxm->outer_l2_len >> HNS3_L2_LEN_UNIT);
+ desc->tx.ol_type_vlan_len_msec = rte_cpu_to_le_32(otmp);
+
+ return 0;
+}
+
+static void
+hns3_set_tso(struct hns3_desc *desc,
+ uint64_t ol_flags, struct rte_mbuf *rxm)
+{
+ uint32_t paylen, hdr_len;
+ uint32_t tmp;
+ uint8_t l2_len = rxm->l2_len;
+
+ if (!(ol_flags & PKT_TX_TCP_SEG))
+ return;
+
+ if (hns3_tso_proc_tunnel(desc, ol_flags, rxm, &l2_len))
+ return;
+
+ hdr_len = rxm->l2_len + rxm->l3_len + rxm->l4_len;
+ hdr_len += (ol_flags & PKT_TX_TUNNEL_MASK) ?
+ rxm->outer_l2_len + rxm->outer_l3_len : 0;
+ paylen = rxm->pkt_len - hdr_len;
+ if (paylen <= rxm->tso_segsz)
+ return;
+
+ tmp = rte_le_to_cpu_32(desc->tx.type_cs_vlan_tso_len);
+ hns3_set_bit(tmp, HNS3_TXD_TSO_B, 1);
+ hns3_set_bit(tmp, HNS3_TXD_L3CS_B, 1);
+ hns3_set_field(tmp, HNS3_TXD_L4T_M, HNS3_TXD_L4T_S, HNS3_L4T_TCP);
+ hns3_set_bit(tmp, HNS3_TXD_L4CS_B, 1);
+ hns3_set_field(tmp, HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
+ sizeof(struct rte_tcp_hdr) >> HNS3_L4_LEN_UNIT);
+ hns3_set_field(tmp, HNS3_TXD_L2LEN_M, HNS3_TXD_L2LEN_S,
+ l2_len >> HNS3_L2_LEN_UNIT);
+ desc->tx.type_cs_vlan_tso_len = rte_cpu_to_le_32(tmp);
+ desc->tx.mss = rte_cpu_to_le_16(rxm->tso_segsz);
+}
+
static void
fill_desc(struct hns3_tx_queue *txq, uint16_t tx_desc_id, struct rte_mbuf *rxm,
bool first, int offset)
struct hns3_desc *tx_ring = txq->tx_ring;
struct hns3_desc *desc = &tx_ring[tx_desc_id];
uint8_t frag_end = rxm->next == NULL ? 1 : 0;
+ uint64_t ol_flags = rxm->ol_flags;
uint16_t size = rxm->data_len;
uint16_t rrcfv = 0;
- uint64_t ol_flags = rxm->ol_flags;
uint32_t hdr_len;
uint32_t paylen;
uint32_t tmp;
rxm->outer_l2_len + rxm->outer_l3_len : 0;
paylen = rxm->pkt_len - hdr_len;
desc->tx.paylen = rte_cpu_to_le_32(paylen);
+ hns3_set_tso(desc, ol_flags, rxm);
}
hns3_set_bit(rrcfv, HNS3_TXD_FE_B, frag_end);
desc->tx.type_cs_vlan_tso_len |= rte_cpu_to_le_32(value);
}
+static bool
+hns3_pkt_need_linearized(struct rte_mbuf *tx_pkts, uint32_t bd_num)
+{
+ struct rte_mbuf *m_first = tx_pkts;
+ struct rte_mbuf *m_last = tx_pkts;
+ uint32_t tot_len = 0;
+ uint32_t hdr_len;
+ uint32_t i;
+
+ /*
+ * Hardware requires that the sum of the data length of every 8
+ * consecutive buffers is greater than MSS in hns3 network engine.
+ * We simplify it by ensuring pkt_headlen + the first 8 consecutive
+ * frags greater than gso header len + mss, and the remaining 7
+ * consecutive frags greater than MSS except the last 7 frags.
+ */
+ if (bd_num <= HNS3_MAX_NON_TSO_BD_PER_PKT)
+ return false;
+
+ for (i = 0; m_last && i < HNS3_MAX_NON_TSO_BD_PER_PKT - 1;
+ i++, m_last = m_last->next)
+ tot_len += m_last->data_len;
+
+ if (!m_last)
+ return true;
+
+ /* ensure the first 8 frags is greater than mss + header */
+ hdr_len = tx_pkts->l2_len + tx_pkts->l3_len + tx_pkts->l4_len;
+ hdr_len += (tx_pkts->ol_flags & PKT_TX_TUNNEL_MASK) ?
+ tx_pkts->outer_l2_len + tx_pkts->outer_l3_len : 0;
+ if (tot_len + m_last->data_len < tx_pkts->tso_segsz + hdr_len)
+ return true;
+
+ /*
+ * ensure the sum of the data length of every 7 consecutive buffer
+ * is greater than mss except the last one.
+ */
+ for (i = 0; m_last && i < bd_num - HNS3_MAX_NON_TSO_BD_PER_PKT; i++) {
+ tot_len -= m_first->data_len;
+ tot_len += m_last->data_len;
+
+ if (tot_len < tx_pkts->tso_segsz)
+ return true;
+
+ m_first = m_first->next;
+ m_last = m_last->next;
+ }
+
+ return false;
+}
+
+static void
+hns3_outer_header_cksum_prepare(struct rte_mbuf *m)
+{
+ uint64_t ol_flags = m->ol_flags;
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_udp_hdr *udp_hdr;
+ uint32_t paylen, hdr_len;
+
+ if (!(ol_flags & (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IPV6)))
+ return;
+
+ if (ol_flags & PKT_TX_IPV4) {
+ ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *,
+ m->outer_l2_len);
+
+ if (ol_flags & PKT_TX_IP_CKSUM)
+ ipv4_hdr->hdr_checksum = 0;
+ }
+
+ if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM &&
+ ol_flags & PKT_TX_TCP_SEG) {
+ hdr_len = m->l2_len + m->l3_len + m->l4_len;
+ hdr_len += (ol_flags & PKT_TX_TUNNEL_MASK) ?
+ m->outer_l2_len + m->outer_l3_len : 0;
+ paylen = m->pkt_len - hdr_len;
+ if (paylen <= m->tso_segsz)
+ return;
+ udp_hdr = rte_pktmbuf_mtod_offset(m, struct rte_udp_hdr *,
+ m->outer_l2_len +
+ m->outer_l3_len);
+ udp_hdr->dgram_cksum = 0;
+ }
+}
+
+static inline bool
+hns3_pkt_is_tso(struct rte_mbuf *m)
+{
+ return (m->tso_segsz != 0 && m->ol_flags & PKT_TX_TCP_SEG);
+}
+
+static int
+hns3_check_tso_pkt_valid(struct rte_mbuf *m)
+{
+ uint32_t tmp_data_len_sum = 0;
+ uint16_t nb_buf = m->nb_segs;
+ uint32_t paylen, hdr_len;
+ struct rte_mbuf *m_seg;
+ int i;
+
+ if (nb_buf > HNS3_MAX_TSO_BD_PER_PKT)
+ return -EINVAL;
+
+ hdr_len = m->l2_len + m->l3_len + m->l4_len;
+ hdr_len += (m->ol_flags & PKT_TX_TUNNEL_MASK) ?
+ m->outer_l2_len + m->outer_l3_len : 0;
+ if (hdr_len > HNS3_MAX_TSO_HDR_SIZE)
+ return -EINVAL;
+
+ paylen = m->pkt_len - hdr_len;
+ if (paylen > HNS3_MAX_BD_PAYLEN)
+ return -EINVAL;
+
+ /*
+ * The TSO header (include outer and inner L2, L3 and L4 header)
+ * should be provided by three descriptors in maximum in hns3 network
+ * engine.
+ */
+ m_seg = m;
+ for (i = 0; m_seg != NULL && i < HNS3_MAX_TSO_HDR_BD_NUM && i < nb_buf;
+ i++, m_seg = m_seg->next) {
+ tmp_data_len_sum += m_seg->data_len;
+ }
+
+ if (hdr_len > tmp_data_len_sum)
+ return -EINVAL;
+
+ return 0;
+}
+
uint16_t
hns3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
return i;
}
+ if (hns3_pkt_is_tso(m) &&
+ (hns3_pkt_need_linearized(m, m->nb_segs) ||
+ hns3_check_tso_pkt_valid(m))) {
+ rte_errno = EINVAL;
+ return i;
+ }
+
#ifdef RTE_LIBRTE_ETHDEV_DEBUG
ret = rte_validate_tx_offload(m);
if (ret != 0) {
rte_errno = -ret;
return i;
}
+
+ hns3_outer_header_cksum_prepare(m);
}
return i;
return 0;
}
+static int
+hns3_check_non_tso_pkt(uint16_t nb_buf, struct rte_mbuf **m_seg,
+ struct rte_mbuf *tx_pkt, struct hns3_tx_queue *txq)
+{
+ struct rte_mbuf *new_pkt;
+ int ret;
+
+ if (hns3_pkt_is_tso(*m_seg))
+ return 0;
+
+ /*
+ * If packet length is greater than HNS3_MAX_FRAME_LEN
+ * driver support, the packet will be ignored.
+ */
+ if (unlikely(rte_pktmbuf_pkt_len(tx_pkt) > HNS3_MAX_FRAME_LEN))
+ return -EINVAL;
+
+ if (unlikely(nb_buf > HNS3_MAX_NON_TSO_BD_PER_PKT)) {
+ ret = hns3_reassemble_tx_pkts(txq, tx_pkt, &new_pkt);
+ if (ret)
+ return ret;
+ *m_seg = new_pkt;
+ }
+
+ return 0;
+}
+
uint16_t
hns3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
struct rte_net_hdr_lens hdr_lens = {0};
struct hns3_tx_queue *txq = tx_queue;
struct hns3_entry *tx_bak_pkt;
- struct rte_mbuf *new_pkt;
struct rte_mbuf *tx_pkt;
struct rte_mbuf *m_seg;
- struct rte_mbuf *temp;
uint32_t nb_hold = 0;
- uint16_t tx_next_clean;
uint16_t tx_next_use;
- uint16_t tx_bd_ready;
uint16_t tx_pkt_num;
uint16_t tx_bd_max;
uint16_t nb_buf;
/* free useless buffer */
hns3_tx_free_useless_buffer(txq);
- tx_bd_ready = txq->tx_bd_ready;
- if (tx_bd_ready == 0)
- return 0;
- tx_next_clean = txq->next_to_clean;
tx_next_use = txq->next_to_use;
tx_bd_max = txq->nb_tx_desc;
- tx_bak_pkt = &txq->sw_ring[tx_next_clean];
-
- tx_pkt_num = (tx_bd_ready < nb_pkts) ? tx_bd_ready : nb_pkts;
+ tx_pkt_num = nb_pkts;
/* send packets */
tx_bak_pkt = &txq->sw_ring[tx_next_use];
nb_buf = tx_pkt->nb_segs;
- if (nb_buf > tx_ring_space(txq)) {
+ if (nb_buf > txq->tx_bd_ready) {
if (nb_tx == 0)
return 0;
goto end_of_tx;
}
- /*
- * If packet length is greater than HNS3_MAX_FRAME_LEN
- * driver support, the packet will be ignored.
- */
- if (unlikely(rte_pktmbuf_pkt_len(tx_pkt) > HNS3_MAX_FRAME_LEN))
- break;
-
/*
* If packet length is less than minimum packet size, driver
* need to pad it.
}
m_seg = tx_pkt;
- if (unlikely(nb_buf > HNS3_MAX_TX_BD_PER_PKT)) {
- if (hns3_reassemble_tx_pkts(txq, tx_pkt, &new_pkt))
- goto end_of_tx;
- m_seg = new_pkt;
- nb_buf = m_seg->nb_segs;
- }
+
+ if (hns3_check_non_tso_pkt(nb_buf, &m_seg, tx_pkt, txq))
+ goto end_of_tx;
if (hns3_parse_cksum(txq, tx_next_use, m_seg, &hdr_lens))
goto end_of_tx;
i = 0;
do {
fill_desc(txq, tx_next_use, m_seg, (i == 0), 0);
- temp = m_seg->next;
tx_bak_pkt->mbuf = m_seg;
- m_seg = temp;
+ m_seg = m_seg->next;
tx_next_use++;
tx_bak_pkt++;
if (tx_next_use >= tx_bd_max) {
nb_hold += i;
txq->next_to_use = tx_next_use;
+ txq->tx_bd_ready -= i;
}
end_of_tx:
- if (likely(nb_tx)) {
+ if (likely(nb_tx))
hns3_queue_xmit(txq, nb_hold);
- txq->next_to_clean = tx_next_clean;
- txq->tx_bd_ready = tx_bd_ready - nb_hold;
- }
return nb_tx;
}
git.droids-corp.org / dpdk.git / blobdiff