PKT_TX_VLAN_PKT | \
PKT_TX_IP_CKSUM | \
PKT_TX_L4_MASK | \
- PKT_TX_TCP_SEG)
+ PKT_TX_TCP_SEG | \
+ PKT_TX_OUTER_IP_CKSUM)
static inline struct rte_mbuf *
rte_rxmbuf_alloc(struct rte_mempool *mp)
m = __rte_mbuf_raw_alloc(mp);
__rte_mbuf_sanity_check_raw(m, 0);
- return (m);
+ return m;
}
*/
#define rte_ixgbe_prefetch(p) rte_prefetch0(p)
#else
-#define rte_ixgbe_prefetch(p) do {} while(0)
+#define rte_ixgbe_prefetch(p) do {} while (0)
#endif
/*********************************************************************
{
struct ixgbe_tx_entry *txep;
uint32_t status;
- int i;
+ int i, nb_free = 0;
+ struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ];
/* check DD bit on threshold descriptor */
status = txq->tx_ring[txq->tx_next_dd].wb.status;
*/
txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
- /* free buffers one at a time */
- if ((txq->txq_flags & (uint32_t)ETH_TXQ_FLAGS_NOREFCOUNT) != 0) {
- for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
- txep->mbuf->next = NULL;
- rte_mempool_put(txep->mbuf->pool, txep->mbuf);
- txep->mbuf = NULL;
- }
- } else {
- for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
- rte_pktmbuf_free_seg(txep->mbuf);
- txep->mbuf = NULL;
+ for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
+ /* free buffers one at a time */
+ m = __rte_pktmbuf_prefree_seg(txep->mbuf);
+ txep->mbuf = NULL;
+
+ if (unlikely(m == NULL))
+ continue;
+
+ if (nb_free >= RTE_IXGBE_TX_MAX_FREE_BUF_SZ ||
+ (nb_free > 0 && m->pool != free[0]->pool)) {
+ rte_mempool_put_bulk(free[0]->pool,
+ (void **)free, nb_free);
+ nb_free = 0;
}
+
+ free[nb_free++] = m;
}
+ if (nb_free > 0)
+ rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
+
/* buffers were freed, update counters */
txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
int i;
for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
- buf_dma_addr = RTE_MBUF_DATA_DMA_ADDR(*pkts);
+ buf_dma_addr = rte_mbuf_data_dma_addr(*pkts);
pkt_len = (*pkts)->data_len;
/* write data to descriptor */
uint64_t buf_dma_addr;
uint32_t pkt_len;
- buf_dma_addr = RTE_MBUF_DATA_DMA_ADDR(*pkts);
+ buf_dma_addr = rte_mbuf_data_dma_addr(*pkts);
pkt_len = (*pkts)->data_len;
/* write data to descriptor */
uint32_t ctx_idx;
uint32_t vlan_macip_lens;
union ixgbe_tx_offload tx_offload_mask;
+ uint32_t seqnum_seed = 0;
ctx_idx = txq->ctx_curr;
- tx_offload_mask.data = 0;
+ tx_offload_mask.data[0] = 0;
+ tx_offload_mask.data[1] = 0;
type_tucmd_mlhl = 0;
/* Specify which HW CTX to upload. */
}
}
+ if (ol_flags & PKT_TX_OUTER_IP_CKSUM) {
+ tx_offload_mask.outer_l2_len |= ~0;
+ tx_offload_mask.outer_l3_len |= ~0;
+ tx_offload_mask.l2_len |= ~0;
+ seqnum_seed |= tx_offload.outer_l3_len
+ << IXGBE_ADVTXD_OUTER_IPLEN;
+ seqnum_seed |= tx_offload.l2_len
+ << IXGBE_ADVTXD_TUNNEL_LEN;
+ }
+
txq->ctx_cache[ctx_idx].flags = ol_flags;
- txq->ctx_cache[ctx_idx].tx_offload.data =
- tx_offload_mask.data & tx_offload.data;
+ txq->ctx_cache[ctx_idx].tx_offload.data[0] =
+ tx_offload_mask.data[0] & tx_offload.data[0];
+ txq->ctx_cache[ctx_idx].tx_offload.data[1] =
+ tx_offload_mask.data[1] & tx_offload.data[1];
txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
vlan_macip_lens = tx_offload.l3_len;
- vlan_macip_lens |= (tx_offload.l2_len << IXGBE_ADVTXD_MACLEN_SHIFT);
+ if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
+ vlan_macip_lens |= (tx_offload.outer_l2_len <<
+ IXGBE_ADVTXD_MACLEN_SHIFT);
+ else
+ vlan_macip_lens |= (tx_offload.l2_len <<
+ IXGBE_ADVTXD_MACLEN_SHIFT);
vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx);
- ctx_txd->seqnum_seed = 0;
+ ctx_txd->seqnum_seed = seqnum_seed;
}
/*
{
/* If match with the current used context */
if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
- (txq->ctx_cache[txq->ctx_curr].tx_offload.data ==
- (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data & tx_offload.data)))) {
+ (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
+ (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
+ & tx_offload.data[0])) &&
+ (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
+ (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
+ & tx_offload.data[1])))) {
return txq->ctx_curr;
}
/* What if match with the next context */
txq->ctx_curr ^= 1;
if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
- (txq->ctx_cache[txq->ctx_curr].tx_offload.data ==
- (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data & tx_offload.data)))) {
+ (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
+ (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
+ & tx_offload.data[0])) &&
+ (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
+ (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
+ & tx_offload.data[1])))) {
return txq->ctx_curr;
}
/* Mismatch, use the previous context */
- return (IXGBE_CTX_NUM);
+ return IXGBE_CTX_NUM;
}
static inline uint32_t
cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
if (ol_flags & PKT_TX_TCP_SEG)
cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
+ if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
+ cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT);
return cmdtype;
}
txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
/* No Error */
- return (0);
+ return 0;
}
uint16_t
uint64_t tx_ol_req;
uint32_t ctx = 0;
uint32_t new_ctx;
- union ixgbe_tx_offload tx_offload = {0};
+ union ixgbe_tx_offload tx_offload;
+ tx_offload.data[0] = 0;
+ tx_offload.data[1] = 0;
txq = tx_queue;
sw_ring = txq->sw_ring;
txr = txq->tx_ring;
tx_offload.l4_len = tx_pkt->l4_len;
tx_offload.vlan_tci = tx_pkt->vlan_tci;
tx_offload.tso_segsz = tx_pkt->tso_segsz;
+ tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
+ tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
/* If new context need be built or reuse the exist ctx. */
ctx = what_advctx_update(txq, tx_ol_req,
if (ixgbe_xmit_cleanup(txq) != 0) {
/* Could not clean any descriptors */
if (nb_tx == 0)
- return (0);
+ return 0;
goto end_of_tx;
}
* descriptors
*/
if (nb_tx == 0)
- return (0);
+ return 0;
goto end_of_tx;
}
}
* Set up Transmit Data Descriptor.
*/
slen = m_seg->data_len;
- buf_dma_addr = RTE_MBUF_DATA_DMA_ADDR(m_seg);
+ buf_dma_addr = rte_mbuf_data_dma_addr(m_seg);
txd->read.buffer_addr =
rte_cpu_to_le_64(buf_dma_addr);
txd->read.cmd_type_len =
IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, tx_id);
txq->tx_tail = tx_id;
- return (nb_tx);
+ return nb_tx;
}
/*********************************************************************
static inline uint64_t
rx_desc_error_to_pkt_flags(uint32_t rx_status)
{
+ uint64_t pkt_flags;
+
/*
* Bit 31: IPE, IPv4 checksum error
* Bit 30: L4I, L4I integrity error
0, PKT_RX_L4_CKSUM_BAD, PKT_RX_IP_CKSUM_BAD,
PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD
};
- return error_to_pkt_flags_map[(rx_status >>
+ pkt_flags = error_to_pkt_flags_map[(rx_status >>
IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK];
+
+ if ((rx_status & IXGBE_RXD_STAT_OUTERIPCS) &&
+ (rx_status & IXGBE_RXDADV_ERR_OUTERIPER)) {
+ pkt_flags |= PKT_RX_EIP_CKSUM_BAD;
+ }
+
+ return pkt_flags;
}
/*
diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
rxq->rx_free_thresh);
if (unlikely(diag != 0))
- return (-ENOMEM);
+ return -ENOMEM;
rxdp = &rxq->rx_ring[alloc_idx];
for (i = 0; i < rxq->rx_free_thresh; ++i) {
mb->data_off = RTE_PKTMBUF_HEADROOM;
/* populate the descriptors */
- dma_addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb));
+ dma_addr = rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mb));
rxdp[i].read.hdr_addr = 0;
rxdp[i].read.pkt_addr = dma_addr;
}
rxm = rxe->mbuf;
rxe->mbuf = nmb;
dma_addr =
- rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR_DEFAULT(nmb));
+ rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb));
rxdp->read.hdr_addr = 0;
rxdp->read.pkt_addr = dma_addr;
nb_hold = 0;
}
rxq->nb_rx_hold = nb_hold;
- return (nb_rx);
+ return nb_rx;
}
/**
if (!bulk_alloc) {
__le64 dma =
- rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR_DEFAULT(nmb));
+ rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb));
/*
* Update RX descriptor with the physical address of the
* new data buffer of the new allocated mbuf.
*
**********************************************************************/
-/*
- * Create memzone for HW rings. malloc can't be used as the physical address is
- * needed. If the memzone is already created, then this function returns a ptr
- * to the old one.
- */
-static const struct rte_memzone * __attribute__((cold))
-ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
- uint16_t queue_id, uint32_t ring_size, int socket_id)
-{
- char z_name[RTE_MEMZONE_NAMESIZE];
- const struct rte_memzone *mz;
-
- snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
- dev->driver->pci_drv.name, ring_name,
- dev->data->port_id, queue_id);
-
- mz = rte_memzone_lookup(z_name);
- if (mz)
- return mz;
-
-#ifdef RTE_LIBRTE_XEN_DOM0
- return rte_memzone_reserve_bounded(z_name, ring_size,
- socket_id, 0, IXGBE_ALIGN, RTE_PGSIZE_2M);
-#else
- return rte_memzone_reserve_aligned(z_name, ring_size,
- socket_id, 0, IXGBE_ALIGN);
-#endif
-}
-
static void __attribute__((cold))
ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)
{
txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue),
RTE_CACHE_LINE_SIZE, socket_id);
if (txq == NULL)
- return (-ENOMEM);
+ return -ENOMEM;
/*
* Allocate TX ring hardware descriptors. A memzone large enough to
* handle the maximum ring size is allocated in order to allow for
* resizing in later calls to the queue setup function.
*/
- tz = ring_dma_zone_reserve(dev, "tx_ring", queue_idx,
+ tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC,
- socket_id);
+ IXGBE_ALIGN, socket_id);
if (tz == NULL) {
ixgbe_tx_queue_release(txq);
- return (-ENOMEM);
+ return -ENOMEM;
}
txq->nb_tx_desc = nb_desc;
txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx));
else
txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx));
-#ifndef RTE_LIBRTE_XEN_DOM0
- txq->tx_ring_phys_addr = (uint64_t) tz->phys_addr;
-#else
+
txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr);
-#endif
txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;
/* Allocate software ring */
RTE_CACHE_LINE_SIZE, socket_id);
if (txq->sw_ring == NULL) {
ixgbe_tx_queue_release(txq);
- return (-ENOMEM);
+ return -ENOMEM;
}
PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
dev->data->tx_queues[queue_idx] = txq;
- return (0);
+ return 0;
}
/**
if (nb_desc % IXGBE_RXD_ALIGN != 0 ||
(nb_desc > IXGBE_MAX_RING_DESC) ||
(nb_desc < IXGBE_MIN_RING_DESC)) {
- return (-EINVAL);
+ return -EINVAL;
}
/* Free memory prior to re-allocation if needed... */
rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue),
RTE_CACHE_LINE_SIZE, socket_id);
if (rxq == NULL)
- return (-ENOMEM);
+ return -ENOMEM;
rxq->mb_pool = mp;
rxq->nb_rx_desc = nb_desc;
rxq->rx_free_thresh = rx_conf->rx_free_thresh;
* handle the maximum ring size is allocated in order to allow for
* resizing in later calls to the queue setup function.
*/
- rz = ring_dma_zone_reserve(dev, "rx_ring", queue_idx,
- RX_RING_SZ, socket_id);
+ rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
+ RX_RING_SZ, IXGBE_ALIGN, socket_id);
if (rz == NULL) {
ixgbe_rx_queue_release(rxq);
- return (-ENOMEM);
+ return -ENOMEM;
}
/*
rxq->rdh_reg_addr =
IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx));
}
-#ifndef RTE_LIBRTE_XEN_DOM0
- rxq->rx_ring_phys_addr = (uint64_t) rz->phys_addr;
-#else
+
rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr);
-#endif
rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr;
/*
RTE_CACHE_LINE_SIZE, socket_id);
if (!rxq->sw_ring) {
ixgbe_rx_queue_release(rxq);
- return (-ENOMEM);
+ return -ENOMEM;
}
/*
RTE_CACHE_LINE_SIZE, socket_id);
if (!rxq->sw_sc_ring) {
ixgbe_rx_queue_release(rxq);
- return (-ENOMEM);
+ return -ENOMEM;
}
PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
break;
}
- for (i = 0 ; i < nb_tcs; i++) {
+ for (i = 0; i < nb_tcs; i++) {
uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
/* clear 10 bits. */
/* VLNCTRL: enable vlan filtering and allow all vlan tags through */
vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
- vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
+ vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
/* VFTA - enable all vlan filters */
/* VLNCTRL: enable vlan filtering and allow all vlan tags through */
vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
- vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
+ vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
/* VFTA - enable all vlan filters */
nb_tcs = dcb_config->num_tcs.pfc_tcs;
/* Unpack map */
ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
- if(nb_tcs == ETH_4_TCS) {
+ if (nb_tcs == ETH_4_TCS) {
/* Avoid un-configured priority mapping to TC0 */
uint8_t j = 4;
uint8_t mask = 0xFF;
break;
}
- if(config_dcb_rx) {
+ if (config_dcb_rx) {
/* Set RX buffer size */
pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT;
- for (i = 0 ; i < nb_tcs; i++) {
+ for (i = 0; i < nb_tcs; i++) {
IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
}
/* zero alloc all unused TCs */
IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
}
}
- if(config_dcb_tx) {
+ if (config_dcb_tx) {
/* Only support an equally distributed Tx packet buffer strategy. */
uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs;
uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX;
ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config,max_frame,
IXGBE_DCB_RX_CONFIG);
- if(config_dcb_rx) {
+ if (config_dcb_rx) {
/* Unpack CEE standard containers */
ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill);
ixgbe_dcb_unpack_max_cee(dcb_config, max);
ixgbe_dcb_hw_arbite_rx_config(hw,refill,max,bwgid,tsa,map);
}
- if(config_dcb_tx) {
+ if (config_dcb_tx) {
/* Unpack CEE standard containers */
ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill);
ixgbe_dcb_unpack_max_cee(dcb_config, max);
ixgbe_dcb_config_tc_stats_82599(hw, dcb_config);
/* Check if the PFC is supported */
- if(dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
+ if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
for (i = 0; i < nb_tcs; i++) {
/*
tc->pfc = ixgbe_dcb_pfc_enabled;
}
ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
- if(dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
+ if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
pfc_en &= 0x0F;
ret = ixgbe_dcb_config_pfc(hw, pfc_en, map);
}
/* VLNCTRL: enable vlan filtering and allow all vlan tags through */
vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
- vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
+ vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
/* VFTA - enable all vlan filters */
if (mbuf == NULL) {
PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
(unsigned) rxq->queue_id);
- return (-ENOMEM);
+ return -ENOMEM;
}
rte_mbuf_refcnt_set(mbuf, 1);
mbuf->port = rxq->port_id;
dma_addr =
- rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mbuf));
+ rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mbuf));
rxd = &rxq->rx_ring[i];
rxd->read.hdr_addr = 0;
rxd->read.pkt_addr = dma_addr;