#define I40E_TX_OFFLOAD_SIMPLE_NOTSUP_MASK \
(PKT_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_SIMPLE_SUP_MASK)
+static int
+i40e_monitor_callback(const uint64_t value,
+ const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused)
+{
+ const uint64_t m = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT);
+ /*
+ * we expect the DD bit to be set to 1 if this descriptor was already
+ * written to.
+ */
+ return (value & m) == m ? -1 : 0;
+}
+
int
i40e_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
{
/* watch for changes in status bit */
pmc->addr = &rxdp->wb.qword1.status_error_len;
- /*
- * we expect the DD bit to be set to 1 if this descriptor was already
- * written to.
- */
- pmc->val = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT);
- pmc->mask = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT);
+ /* comparison callback */
+ pmc->fn = i40e_monitor_callback;
/* registers are 64-bit */
pmc->size = sizeof(uint64_t);
uint16_t pkt_len;
uint64_t qword1;
uint32_t rx_status;
- int32_t s[I40E_LOOK_AHEAD], nb_dd;
+ int32_t s[I40E_LOOK_AHEAD], var, nb_dd;
int32_t i, j, nb_rx = 0;
uint64_t pkt_flags;
uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
I40E_RXD_QW1_STATUS_SHIFT;
}
- rte_smp_rmb();
+ /* This barrier is to order loads of different words in the descriptor */
+ rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
/* Compute how many status bits were set */
- for (j = 0, nb_dd = 0; j < I40E_LOOK_AHEAD; j++)
- nb_dd += s[j] & (1 << I40E_RX_DESC_STATUS_DD_SHIFT);
+ for (j = 0, nb_dd = 0; j < I40E_LOOK_AHEAD; j++) {
+ var = s[j] & (1 << I40E_RX_DESC_STATUS_DD_SHIFT);
+#ifdef RTE_ARCH_ARM
+ /* For Arm platforms, only compute continuous status bits */
+ if (var)
+ nb_dd += 1;
+ else
+ break;
+#else
+ nb_dd += var;
+#endif
+ }
nb_rx += nb_dd;
i40e_tx_free_bufs(struct i40e_tx_queue *txq)
{
struct i40e_tx_entry *txep;
- uint16_t i;
+ uint16_t tx_rs_thresh = txq->tx_rs_thresh;
+ uint16_t i = 0, j = 0;
+ struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
+ const uint16_t k = RTE_ALIGN_FLOOR(tx_rs_thresh, RTE_I40E_TX_MAX_FREE_BUF_SZ);
+ const uint16_t m = tx_rs_thresh % RTE_I40E_TX_MAX_FREE_BUF_SZ;
if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
return 0;
- txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
+ txep = &txq->sw_ring[txq->tx_next_dd - (tx_rs_thresh - 1)];
- for (i = 0; i < txq->tx_rs_thresh; i++)
+ for (i = 0; i < tx_rs_thresh; i++)
rte_prefetch0((txep + i)->mbuf);
if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) {
- for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
- rte_mempool_put(txep->mbuf->pool, txep->mbuf);
- txep->mbuf = NULL;
+ if (k) {
+ for (j = 0; j != k; j += RTE_I40E_TX_MAX_FREE_BUF_SZ) {
+ for (i = 0; i < RTE_I40E_TX_MAX_FREE_BUF_SZ; ++i, ++txep) {
+ free[i] = txep->mbuf;
+ txep->mbuf = NULL;
+ }
+ rte_mempool_put_bulk(free[0]->pool, (void **)free,
+ RTE_I40E_TX_MAX_FREE_BUF_SZ);
+ }
+ }
+
+ if (m) {
+ for (i = 0; i < m; ++i, ++txep) {
+ free[i] = txep->mbuf;
+ txep->mbuf = NULL;
+ }
+ rte_mempool_put_bulk(free[0]->pool, (void **)free, m);
}
} else {
for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
const struct rte_eth_rxconf *rx_conf,
struct rte_mempool *mp)
{
- struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct i40e_adapter *ad =
I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
struct i40e_vsi *vsi;
struct i40e_pf *pf = NULL;
- struct i40e_vf *vf = NULL;
struct i40e_rx_queue *rxq;
const struct rte_memzone *rz;
uint32_t ring_size;
offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
- if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) {
- vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
- vsi = &vf->vsi;
- if (!vsi)
- return -EINVAL;
- reg_idx = queue_idx;
- } else {
- pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
- vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
- if (!vsi)
- return -EINVAL;
- q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx);
- if (q_offset < 0)
- return -EINVAL;
- reg_idx = vsi->base_queue + q_offset;
- }
+ pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
+ vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
+ if (!vsi)
+ return -EINVAL;
+ q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx);
+ if (q_offset < 0)
+ return -EINVAL;
+ reg_idx = vsi->base_queue + q_offset;
if (nb_desc % I40E_ALIGN_RING_DESC != 0 ||
(nb_desc > I40E_MAX_RING_DESC) ||
/* Free memory if needed */
if (dev->data->rx_queues[queue_idx]) {
- i40e_dev_rx_queue_release(dev->data->rx_queues[queue_idx]);
+ i40e_rx_queue_release(dev->data->rx_queues[queue_idx]);
dev->data->rx_queues[queue_idx] = NULL;
}
rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
ring_size, I40E_RING_BASE_ALIGN, socket_id);
if (!rz) {
- i40e_dev_rx_queue_release(rxq);
+ i40e_rx_queue_release(rxq);
PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX");
return -ENOMEM;
}
+ rxq->mz = rz;
/* Zero all the descriptors in the ring. */
memset(rz->addr, 0, ring_size);
RTE_CACHE_LINE_SIZE,
socket_id);
if (!rxq->sw_ring) {
- i40e_dev_rx_queue_release(rxq);
+ i40e_rx_queue_release(rxq);
PMD_DRV_LOG(ERR, "Failed to allocate memory for SW ring");
return -ENOMEM;
}
if (dev->data->dev_started) {
if (i40e_dev_rx_queue_setup_runtime(dev, rxq)) {
- i40e_dev_rx_queue_release(rxq);
+ i40e_rx_queue_release(rxq);
return -EINVAL;
}
} else {
}
void
-i40e_dev_rx_queue_release(void *rxq)
+i40e_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+ i40e_rx_queue_release(dev->data->rx_queues[qid]);
+}
+
+void
+i40e_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+ i40e_tx_queue_release(dev->data->tx_queues[qid]);
+}
+
+void
+i40e_rx_queue_release(void *rxq)
{
struct i40e_rx_queue *q = (struct i40e_rx_queue *)rxq;
i40e_rx_queue_release_mbufs(q);
rte_free(q->sw_ring);
+ rte_memzone_free(q->mz);
rte_free(q);
}
uint32_t
-i40e_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+i40e_dev_rx_queue_count(void *rx_queue)
{
#define I40E_RXQ_SCAN_INTERVAL 4
volatile union i40e_rx_desc *rxdp;
struct i40e_rx_queue *rxq;
uint16_t desc = 0;
- rxq = dev->data->rx_queues[rx_queue_id];
+ rxq = rx_queue;
rxdp = &(rxq->rx_ring[rxq->rx_tail]);
while ((desc < rxq->nb_rx_desc) &&
((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
return desc;
}
-int
-i40e_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
-{
- volatile union i40e_rx_desc *rxdp;
- struct i40e_rx_queue *rxq = rx_queue;
- uint16_t desc;
- int ret;
-
- if (unlikely(offset >= rxq->nb_rx_desc)) {
- PMD_DRV_LOG(ERR, "Invalid RX descriptor id %u", offset);
- return 0;
- }
-
- desc = rxq->rx_tail + offset;
- if (desc >= rxq->nb_rx_desc)
- desc -= rxq->nb_rx_desc;
-
- rxdp = &(rxq->rx_ring[desc]);
-
- ret = !!(((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
- I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) &
- (1 << I40E_RX_DESC_STATUS_DD_SHIFT));
-
- return ret;
-}
-
int
i40e_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
{
unsigned int socket_id,
const struct rte_eth_txconf *tx_conf)
{
- struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct i40e_vsi *vsi;
struct i40e_pf *pf = NULL;
- struct i40e_vf *vf = NULL;
struct i40e_tx_queue *txq;
const struct rte_memzone *tz;
uint32_t ring_size;
offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
- if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) {
- vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
- vsi = &vf->vsi;
- reg_idx = queue_idx;
- } else {
- pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
- vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
- if (!vsi)
- return -EINVAL;
- q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx);
- if (q_offset < 0)
- return -EINVAL;
- reg_idx = vsi->base_queue + q_offset;
- }
+ pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
+ vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
+ if (!vsi)
+ return -EINVAL;
+ q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx);
+ if (q_offset < 0)
+ return -EINVAL;
+ reg_idx = vsi->base_queue + q_offset;
if (nb_desc % I40E_ALIGN_RING_DESC != 0 ||
(nb_desc > I40E_MAX_RING_DESC) ||
/* Free memory if needed. */
if (dev->data->tx_queues[queue_idx]) {
- i40e_dev_tx_queue_release(dev->data->tx_queues[queue_idx]);
+ i40e_tx_queue_release(dev->data->tx_queues[queue_idx]);
dev->data->tx_queues[queue_idx] = NULL;
}
tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
ring_size, I40E_RING_BASE_ALIGN, socket_id);
if (!tz) {
- i40e_dev_tx_queue_release(txq);
+ i40e_tx_queue_release(txq);
PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX");
return -ENOMEM;
}
+ txq->mz = tz;
txq->nb_tx_desc = nb_desc;
txq->tx_rs_thresh = tx_rs_thresh;
txq->tx_free_thresh = tx_free_thresh;
RTE_CACHE_LINE_SIZE,
socket_id);
if (!txq->sw_ring) {
- i40e_dev_tx_queue_release(txq);
+ i40e_tx_queue_release(txq);
PMD_DRV_LOG(ERR, "Failed to allocate memory for SW TX ring");
return -ENOMEM;
}
if (dev->data->dev_started) {
if (i40e_dev_tx_queue_setup_runtime(dev, txq)) {
- i40e_dev_tx_queue_release(txq);
+ i40e_tx_queue_release(txq);
return -EINVAL;
}
} else {
}
void
-i40e_dev_tx_queue_release(void *txq)
+i40e_tx_queue_release(void *txq)
{
struct i40e_tx_queue *q = (struct i40e_tx_queue *)txq;
i40e_tx_queue_release_mbufs(q);
rte_free(q->sw_ring);
+ rte_memzone_free(q->mz);
rte_free(q);
}
#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
rxq->rx_tail = 0;
rxq->nb_rx_hold = 0;
+
+ if (rxq->pkt_first_seg != NULL)
+ rte_pktmbuf_free(rxq->pkt_first_seg);
+
rxq->pkt_first_seg = NULL;
rxq->pkt_last_seg = NULL;
}
rxq->max_pkt_len =
- RTE_MIN((uint32_t)(hw->func_caps.rx_buf_chain_len *
- rxq->rx_buf_len), data->dev_conf.rxmode.max_rx_pkt_len);
- if (data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
+ RTE_MIN(hw->func_caps.rx_buf_chain_len * rxq->rx_buf_len,
+ data->mtu + I40E_ETH_OVERHEAD);
+ if (data->mtu > RTE_ETHER_MTU) {
if (rxq->max_pkt_len <= I40E_ETH_MAX_LEN ||
rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
PMD_DRV_LOG(ERR, "maximum packet length must "
for (i = 0; i < dev->data->nb_rx_queues; i++) {
if (!dev->data->rx_queues[i])
continue;
- i40e_dev_rx_queue_release(dev->data->rx_queues[i]);
+ i40e_rx_queue_release(dev->data->rx_queues[i]);
dev->data->rx_queues[i] = NULL;
- rte_eth_dma_zone_free(dev, "rx_ring", i);
}
for (i = 0; i < dev->data->nb_tx_queues; i++) {
if (!dev->data->tx_queues[i])
continue;
- i40e_dev_tx_queue_release(dev->data->tx_queues[i]);
+ i40e_tx_queue_release(dev->data->tx_queues[i]);
dev->data->tx_queues[i] = NULL;
- rte_eth_dma_zone_free(dev, "tx_ring", i);
}
}
return I40E_ERR_BAD_PTR;
}
- dev = pf->adapter->eth_dev;
+ dev = &rte_eth_devices[pf->dev_data->port_id];
/* Allocate the TX queue data structure. */
txq = rte_zmalloc_socket("i40e fdir tx queue",
I40E_FDIR_QUEUE_ID, ring_size,
I40E_RING_BASE_ALIGN, SOCKET_ID_ANY);
if (!tz) {
- i40e_dev_tx_queue_release(txq);
+ i40e_tx_queue_release(txq);
PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX.");
return I40E_ERR_NO_MEMORY;
}
+ txq->mz = tz;
txq->nb_tx_desc = I40E_FDIR_NUM_TX_DESC;
txq->queue_id = I40E_FDIR_QUEUE_ID;
txq->reg_idx = pf->fdir.fdir_vsi->base_queue;
return I40E_ERR_BAD_PTR;
}
- dev = pf->adapter->eth_dev;
+ dev = &rte_eth_devices[pf->dev_data->port_id];
/* Allocate the RX queue data structure. */
rxq = rte_zmalloc_socket("i40e fdir rx queue",
I40E_FDIR_QUEUE_ID, ring_size,
I40E_RING_BASE_ALIGN, SOCKET_ID_ANY);
if (!rz) {
- i40e_dev_rx_queue_release(rxq);
+ i40e_rx_queue_release(rxq);
PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX.");
return I40E_ERR_NO_MEMORY;
}
+ rxq->mz = rz;
rxq->nb_rx_desc = I40E_FDIR_NUM_RX_DESC;
rxq->queue_id = I40E_FDIR_QUEUE_ID;
rxq->reg_idx = pf->fdir.fdir_vsi->base_queue;
qinfo->conf.offloads = txq->offloads;
}
+#ifdef RTE_ARCH_X86
static inline bool
get_avx_supported(bool request_avx512)
{
-#ifdef RTE_ARCH_X86
if (request_avx512) {
if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 &&
rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 &&
return false;
#endif
}
-#else
- RTE_SET_USED(request_avx512);
-#endif /* RTE_ARCH_X86 */
return false;
}
+#endif /* RTE_ARCH_X86 */
void __rte_cold
git.droids-corp.org / dpdk.git / blobdiff