#include "vnic_cq.h"
#include "vnic_intr.h"
#include "vnic_nic.h"
+#include "enic_vnic_wq.h"
static inline int enic_is_sriov_vf(struct enic *enic)
{
return !is_mcast_addr(addr) && !is_zero_addr(addr);
}
-static inline struct rte_mbuf *
-enic_rxmbuf_alloc(struct rte_mempool *mp)
+static void
+enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq)
{
- struct rte_mbuf *m;
+ uint16_t i;
+
+ if (!rq || !rq->mbuf_ring) {
+ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
+ return;
+ }
- m = __rte_mbuf_raw_alloc(mp);
- __rte_mbuf_sanity_check_raw(m, 0);
- return m;
+ for (i = 0; i < enic->config.rq_desc_count; i++) {
+ if (rq->mbuf_ring[i]) {
+ rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
+ rq->mbuf_ring[i] = NULL;
+ }
+ }
}
+
void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
{
vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
-1 /*wq_work_to_do*/, enic_wq_service, NULL);
}
+void enic_post_wq_index(struct vnic_wq *wq)
+{
+ enic_vnic_post_wq_index(wq);
+}
-int enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
- struct rte_mbuf *tx_pkt, unsigned short len,
- uint8_t sop, uint8_t eop,
- uint16_t ol_flags, uint16_t vlan_tag)
+void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
+ struct rte_mbuf *tx_pkt, unsigned short len,
+ uint8_t sop, uint8_t eop, uint8_t cq_entry,
+ uint16_t ol_flags, uint16_t vlan_tag)
{
struct wq_enet_desc *desc = vnic_wq_next_desc(wq);
uint16_t mss = 0;
- uint8_t cq_entry = eop;
uint8_t vlan_tag_insert = 0;
uint64_t bus_addr = (dma_addr_t)
- (tx_pkt->buf_physaddr + RTE_PKTMBUF_HEADROOM);
+ (tx_pkt->buf_physaddr + tx_pkt->data_off);
if (sop) {
if (ol_flags & PKT_TX_VLAN_PKT)
vlan_tag,
0 /* loopback */);
- vnic_wq_post(wq, (void *)tx_pkt, bus_addr, len,
- sop, eop,
- 1 /*desc_skip_cnt*/,
- cq_entry,
- 0 /*compressed send*/,
- 0 /*wrid*/);
+ enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len,
+ sop,
+ 1 /*desc_skip_cnt*/,
+ cq_entry,
+ 0 /*compressed send*/,
+ 0 /*wrid*/);
+}
- return 0;
+static void enic_clear_soft_stats(struct enic *enic)
+{
+ struct enic_soft_stats *soft_stats = &enic->soft_stats;
+ rte_atomic64_clear(&soft_stats->rx_nombuf);
+ rte_atomic64_clear(&soft_stats->rx_packet_errors);
+}
+
+static void enic_init_soft_stats(struct enic *enic)
+{
+ struct enic_soft_stats *soft_stats = &enic->soft_stats;
+ rte_atomic64_init(&soft_stats->rx_nombuf);
+ rte_atomic64_init(&soft_stats->rx_packet_errors);
+ enic_clear_soft_stats(enic);
}
void enic_dev_stats_clear(struct enic *enic)
{
if (vnic_dev_stats_clear(enic->vdev))
dev_err(enic, "Error in clearing stats\n");
+ enic_clear_soft_stats(enic);
}
void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
{
struct vnic_stats *stats;
+ struct enic_soft_stats *soft_stats = &enic->soft_stats;
+ int64_t rx_truncated;
+ uint64_t rx_packet_errors;
if (vnic_dev_stats_dump(enic->vdev, &stats)) {
dev_err(enic, "Error in getting stats\n");
return;
}
- r_stats->ipackets = stats->rx.rx_frames_ok;
+ /* The number of truncated packets can only be calculated by
+ * subtracting a hardware counter from error packets received by
+ * the driver. Note: this causes transient inaccuracies in the
+ * ipackets count. Also, the length of truncated packets are
+ * counted in ibytes even though truncated packets are dropped
+ * which can make ibytes be slightly higher than it should be.
+ */
+ rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
+ rx_truncated = rx_packet_errors - stats->rx.rx_errors;
+
+ r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
r_stats->opackets = stats->tx.tx_frames_ok;
r_stats->ibytes = stats->rx.rx_bytes_ok;
r_stats->obytes = stats->tx.tx_bytes_ok;
- r_stats->ierrors = stats->rx.rx_errors;
+ r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
r_stats->oerrors = stats->tx.tx_errors;
- r_stats->imcasts = stats->rx.rx_multicast_frames_ok;
- r_stats->rx_nombuf = stats->rx.rx_no_bufs;
+ r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
+
+ r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
}
void enic_del_mac_address(struct enic *enic)
}
static void
-enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf)
+enic_free_rq_buf(struct rte_mbuf **mbuf)
{
- if (!buf->os_buf)
+ if (*mbuf == NULL)
return;
- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf);
- buf->os_buf = NULL;
+ rte_pktmbuf_free(*mbuf);
+ mbuf = NULL;
}
void enic_init_vnic_resources(struct enic *enic)
}
-static int enic_rq_alloc_buf(struct vnic_rq *rq)
+static int
+enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
{
- struct enic *enic = vnic_dev_priv(rq->vdev);
+ struct rte_mbuf *mb;
+ struct rq_enet_desc *rqd = rq->ring.descs;
+ unsigned i;
dma_addr_t dma_addr;
- struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
- uint8_t type = RQ_ENET_TYPE_ONLY_SOP;
- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev);
- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp);
- struct rte_mbuf *hdr_mbuf = NULL;
-
- if (!mbuf) {
- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n");
- return -1;
- }
-
- if (unlikely(split_hdr_size)) {
- if (vnic_rq_desc_avail(rq) < 2) {
- rte_mempool_put(mbuf->pool, mbuf);
- return -1;
- }
- hdr_mbuf = enic_rxmbuf_alloc(rq->mp);
- if (!hdr_mbuf) {
- rte_mempool_put(mbuf->pool, mbuf);
- dev_err(enic,
- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n");
- return -1;
- }
-
- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
-
- hdr_mbuf->nb_segs = 2;
- hdr_mbuf->port = enic->port_id;
- hdr_mbuf->next = mbuf;
-
- dma_addr = (dma_addr_t)
- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off);
-
- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size);
-
- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr,
- (unsigned int)split_hdr_size, 0 /*wrid*/);
-
- desc = vnic_rq_next_desc(rq);
- type = RQ_ENET_TYPE_NOT_SOP;
- } else {
- mbuf->nb_segs = 1;
- mbuf->port = enic->port_id;
- }
- mbuf->data_off = RTE_PKTMBUF_HEADROOM;
- mbuf->next = NULL;
+ dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
+ rq->ring.desc_count);
- dma_addr = (dma_addr_t)
- (mbuf->buf_physaddr + mbuf->data_off);
-
- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len);
-
- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr,
- (unsigned int)mbuf->buf_len, 0 /*wrid*/);
-
- return 0;
-}
-
-static int enic_rq_indicate_buf(struct vnic_rq *rq,
- struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
- int skipped, void *opaque)
-{
- struct enic *enic = vnic_dev_priv(rq->vdev);
- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque;
- struct rte_mbuf *rx_pkt = NULL;
- struct rte_mbuf *hdr_rx_pkt = NULL;
-
- u8 type, color, eop, sop, ingress_port, vlan_stripped;
- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
- u8 packet_error;
- u16 q_number, completed_index, bytes_written, vlan_tci, checksum;
- u32 rss_hash;
-
- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc,
- &type, &color, &q_number, &completed_index,
- &ingress_port, &fcoe, &eop, &sop, &rss_type,
- &csum_not_calc, &rss_hash, &bytes_written,
- &packet_error, &vlan_stripped, &vlan_tci, &checksum,
- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error,
- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp,
- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment,
- &fcs_ok);
-
- rx_pkt = (struct rte_mbuf *)buf->os_buf;
- buf->os_buf = NULL;
-
- if (unlikely(packet_error)) {
- dev_err(enic, "packet error\n");
- rx_pkt->data_len = 0;
- return 0;
- }
-
- if (unlikely(skipped)) {
- rx_pkt->data_len = 0;
- return 0;
- }
-
- if (likely(!vnic_get_hdr_split_size(enic->vdev))) {
- /* No header split configured */
- *rx_pkt_bucket = rx_pkt;
- rx_pkt->pkt_len = bytes_written;
-
- if (ipv4) {
- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
- if (!csum_not_calc) {
- if (unlikely(!ipv4_csum_ok))
- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD;
-
- if ((tcp || udp) && (!tcp_udp_csum_ok))
- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD;
- }
- } else if (ipv6)
- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
- } else {
- /* Header split */
- if (sop && !eop) {
- /* This piece is header */
- *rx_pkt_bucket = rx_pkt;
- rx_pkt->pkt_len = bytes_written;
- } else {
- if (sop && eop) {
- /* The packet is smaller than split_hdr_size */
- *rx_pkt_bucket = rx_pkt;
- rx_pkt->pkt_len = bytes_written;
- if (ipv4) {
- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
- if (!csum_not_calc) {
- if (unlikely(!ipv4_csum_ok))
- rx_pkt->ol_flags |=
- PKT_RX_IP_CKSUM_BAD;
-
- if ((tcp || udp) &&
- (!tcp_udp_csum_ok))
- rx_pkt->ol_flags |=
- PKT_RX_L4_CKSUM_BAD;
- }
- } else if (ipv6)
- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
- } else {
- /* Payload */
- hdr_rx_pkt = *rx_pkt_bucket;
- hdr_rx_pkt->pkt_len += bytes_written;
- if (ipv4) {
- hdr_rx_pkt->packet_type =
- RTE_PTYPE_L3_IPV4;
- if (!csum_not_calc) {
- if (unlikely(!ipv4_csum_ok))
- hdr_rx_pkt->ol_flags |=
- PKT_RX_IP_CKSUM_BAD;
-
- if ((tcp || udp) &&
- (!tcp_udp_csum_ok))
- hdr_rx_pkt->ol_flags |=
- PKT_RX_L4_CKSUM_BAD;
- }
- } else if (ipv6)
- hdr_rx_pkt->packet_type =
- RTE_PTYPE_L3_IPV6;
- }
+ for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
+ mb = rte_mbuf_raw_alloc(rq->mp);
+ if (mb == NULL) {
+ dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
+ (unsigned)rq->index);
+ return -ENOMEM;
}
- }
- rx_pkt->data_len = bytes_written;
+ dma_addr = (dma_addr_t)(mb->buf_physaddr
+ + RTE_PKTMBUF_HEADROOM);
- if (rss_hash) {
- rx_pkt->ol_flags |= PKT_RX_RSS_HASH;
- rx_pkt->hash.rss = rss_hash;
+ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP,
+ mb->buf_len - RTE_PKTMBUF_HEADROOM);
+ rq->mbuf_ring[i] = mb;
}
- if (vlan_tci) {
- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT;
- rx_pkt->vlan_tci = vlan_tci;
- }
-
- return eop;
-}
+ /* make sure all prior writes are complete before doing the PIO write */
+ rte_rmb();
-static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
-{
- struct enic *enic = vnic_dev_priv(vdev);
+ /* Post all but the last 2 cache lines' worth of descriptors */
+ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE
+ / sizeof(struct rq_enet_desc));
+ rq->rx_nb_hold = 0;
- return vnic_rq_service(&enic->rq[q_number], cq_desc,
- completed_index, VNIC_RQ_RETURN_DESC,
- enic_rq_indicate_buf, opaque);
-
-}
-
-int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
- unsigned int budget, unsigned int *work_done)
-{
- struct enic *enic = vnic_dev_priv(rq->vdev);
- unsigned int cq = enic_cq_rq(enic, rq->index);
- int err = 0;
+ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
+ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
+ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
+ rte_rmb();
- *work_done = vnic_cq_service(&enic->cq[cq],
- budget, enic_rq_service, (void *)rx_pkts);
-
- if (*work_done) {
- vnic_rq_fill(rq, enic_rq_alloc_buf);
+ return 0;
- /* Need at least one buffer on ring to get going */
- if (vnic_rq_desc_used(rq) == 0) {
- dev_err(enic, "Unable to alloc receive buffers\n");
- err = -1;
- }
- }
- return err;
}
static void *
*dma_handle = 0;
rz = rte_memzone_reserve_aligned((const char *)name,
- size, 0, 0, ENIC_ALIGN);
+ size, SOCKET_ID_ANY, 0, ENIC_ALIGN);
if (!rz) {
- pr_err("%s : Failed to allocate memory requested for %s",
+ pr_err("%s : Failed to allocate memory requested for %s\n",
__func__, name);
return NULL;
}
int enic_enable(struct enic *enic)
{
unsigned int index;
+ int err;
struct rte_eth_dev *eth_dev = enic->rte_dev;
eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
dev_warning(enic, "Init of hash table for clsf failed."\
"Flow director feature will not work\n");
- /* Fill RQ bufs */
for (index = 0; index < enic->rq_count; index++) {
- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf);
-
- /* Need at least one buffer on ring to get going
- */
- if (vnic_rq_desc_used(&enic->rq[index]) == 0) {
- dev_err(enic, "Unable to alloc receive buffers\n");
- return -1;
+ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]);
+ if (err) {
+ dev_err(enic, "Failed to alloc RX queue mbufs\n");
+ return err;
}
}
struct vnic_rq *rq = (struct vnic_rq *)rxq;
struct enic *enic = vnic_dev_priv(rq->vdev);
+ enic_rxmbuf_queue_release(enic, rq);
+ rte_free(rq->mbuf_ring);
+ rq->mbuf_ring = NULL;
vnic_rq_free(rq);
vnic_cq_free(&enic->cq[rq->index]);
}
unsigned int socket_id, struct rte_mempool *mp,
uint16_t nb_desc)
{
- int err;
+ int rc;
struct vnic_rq *rq = &enic->rq[queue_idx];
rq->socket_id = socket_id;
"policy. Applying the value in the adapter "\
"policy (%d).\n",
queue_idx, nb_desc, enic->config.rq_desc_count);
- } else if (nb_desc != enic->config.rq_desc_count) {
- enic->config.rq_desc_count = nb_desc;
- dev_info(enic,
- "RX Queues - effective number of descs:%d\n",
- nb_desc);
+ nb_desc = enic->config.rq_desc_count;
}
+ dev_info(enic, "RX Queues - effective number of descs:%d\n",
+ nb_desc);
}
/* Allocate queue resources */
- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx,
- enic->config.rq_desc_count,
- sizeof(struct rq_enet_desc));
- if (err) {
+ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx,
+ nb_desc, sizeof(struct rq_enet_desc));
+ if (rc) {
dev_err(enic, "error in allocation of rq\n");
- return err;
+ goto err_exit;
}
- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
- socket_id, enic->config.rq_desc_count,
+ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
+ socket_id, nb_desc,
sizeof(struct cq_enet_rq_desc));
- if (err) {
- vnic_rq_free(rq);
+ if (rc) {
dev_err(enic, "error in allocation of cq for rq\n");
+ goto err_free_rq_exit;
}
- return err;
+ /* Allocate the mbuf ring */
+ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring",
+ sizeof(struct rte_mbuf *) * nb_desc,
+ RTE_CACHE_LINE_SIZE, rq->socket_id);
+
+ if (rq->mbuf_ring != NULL)
+ return 0;
+
+ /* cleanup on error */
+ vnic_cq_free(&enic->cq[queue_idx]);
+err_free_rq_exit:
+ vnic_rq_free(rq);
+err_exit:
+ return -ENOMEM;
}
void enic_free_wq(void *txq)
for (i = 0; i < enic->wq_count; i++)
vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
+
for (i = 0; i < enic->rq_count; i++)
vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
for (i = 0; i < enic->cq_count; i++)
{
int ret;
+ enic_init_soft_stats(enic);
+
ret = enic_set_rss_nic_cfg(enic);
if (ret) {
dev_err(enic, "Failed to config nic, aborting.\n");
/* Set ingress vlan rewrite mode before vnic initialization */
err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN);
+ IG_VLAN_REWRITE_MODE_PASS_THRU);
if (err) {
dev_err(enic,
"Failed to set ingress vlan rewrite mode, aborting.\n");