X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fraw%2Fntb%2Fntb.c;h=ad7f6abfd374686754e8742120910eb5319fb18f;hb=93520085efdaf5581710174555ab03dc31766391;hp=0e62ad433b0e80effe1539c23d90905eda688326;hpb=c39d1e082a4b426e915074ce30eb6f410ee2654a;p=dpdk.git diff --git a/drivers/raw/ntb/ntb.c b/drivers/raw/ntb/ntb.c index 0e62ad433b..ad7f6abfd3 100644 --- a/drivers/raw/ntb/ntb.c +++ b/drivers/raw/ntb/ntb.c @@ -30,6 +30,17 @@ static const struct rte_pci_id pci_id_ntb_map[] = { { .vendor_id = 0, /* sentinel */ }, }; +/* Align with enum ntb_xstats_idx */ +static struct rte_rawdev_xstats_name ntb_xstats_names[] = { + {"Tx-packets"}, + {"Tx-bytes"}, + {"Tx-errors"}, + {"Rx-packets"}, + {"Rx-bytes"}, + {"Rx-missed"}, +}; +#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) + static inline void ntb_link_cleanup(struct rte_rawdev *dev) { @@ -538,29 +549,149 @@ ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) txq->last_avail = 0; txq->nb_tx_free = txq->nb_tx_desc - 1; + /* Set per queue stats. */ + for (i = 0; i < NTB_XSTATS_NUM; i++) { + hw->ntb_xstats[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0; + hw->ntb_xstats_off[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0; + } + return 0; } +static inline void +ntb_enqueue_cleanup(struct ntb_tx_queue *txq) +{ + struct ntb_tx_entry *sw_ring = txq->sw_ring; + uint16_t tx_free = txq->last_avail; + uint16_t nb_to_clean, i; + + /* avail_cnt + 1 represents where to rx next in the peer. */ + nb_to_clean = (*txq->avail_cnt - txq->last_avail + 1 + + txq->nb_tx_desc) & (txq->nb_tx_desc - 1); + nb_to_clean = RTE_MIN(nb_to_clean, txq->tx_free_thresh); + for (i = 0; i < nb_to_clean; i++) { + if (sw_ring[tx_free].mbuf) + rte_pktmbuf_free_seg(sw_ring[tx_free].mbuf); + tx_free = (tx_free + 1) & (txq->nb_tx_desc - 1); + } + + txq->nb_tx_free += nb_to_clean; + txq->last_avail = tx_free; +} + static int ntb_enqueue_bufs(struct rte_rawdev *dev, struct rte_rawdev_buf **buffers, unsigned int count, rte_rawdev_obj_t context) { - /* Not FIFO right now. Just for testing memory write. */ struct ntb_hw *hw = dev->dev_private; - unsigned int i; - void *bar_addr; - size_t size; + struct ntb_tx_queue *txq = hw->tx_queues[(size_t)context]; + struct ntb_tx_entry *sw_ring = txq->sw_ring; + struct rte_mbuf *txm; + struct ntb_used tx_used[NTB_MAX_DESC_SIZE]; + volatile struct ntb_desc *tx_item; + uint16_t tx_last, nb_segs, off, last_used, avail_cnt; + uint16_t nb_mbufs = 0; + uint16_t nb_tx = 0; + uint64_t bytes = 0; + void *buf_addr; + int i; - if (hw->ntb_ops->get_peer_mw_addr == NULL) - return -ENOTSUP; - bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0); - size = (size_t)context; + if (unlikely(hw->ntb_ops->ioremap == NULL)) { + NTB_LOG(ERR, "Ioremap not supported."); + return nb_tx; + } - for (i = 0; i < count; i++) - rte_memcpy(bar_addr, buffers[i]->buf_addr, size); - return 0; + if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) { + NTB_LOG(DEBUG, "Link is not up."); + return nb_tx; + } + + if (txq->nb_tx_free < txq->tx_free_thresh) + ntb_enqueue_cleanup(txq); + + off = NTB_XSTATS_NUM * ((size_t)context + 1); + last_used = txq->last_used; + avail_cnt = *txq->avail_cnt;/* Where to alloc next. */ + for (nb_tx = 0; nb_tx < count; nb_tx++) { + txm = (struct rte_mbuf *)(buffers[nb_tx]->buf_addr); + if (txm == NULL || txq->nb_tx_free < txm->nb_segs) + break; + + tx_last = (txq->last_used + txm->nb_segs - 1) & + (txq->nb_tx_desc - 1); + nb_segs = txm->nb_segs; + for (i = 0; i < nb_segs; i++) { + /* Not enough ring space for tx. */ + if (txq->last_used == avail_cnt) + goto end_of_tx; + sw_ring[txq->last_used].mbuf = txm; + tx_item = txq->tx_desc_ring + txq->last_used; + + if (!tx_item->len) { + (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++; + goto end_of_tx; + } + if (txm->data_len > tx_item->len) { + NTB_LOG(ERR, "Data length exceeds buf length." + " Only %u data would be transmitted.", + tx_item->len); + txm->data_len = tx_item->len; + } + + /* translate remote virtual addr to bar virtual addr */ + buf_addr = (*hw->ntb_ops->ioremap)(dev, tx_item->addr); + if (buf_addr == NULL) { + (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++; + NTB_LOG(ERR, "Null remap addr."); + goto end_of_tx; + } + rte_memcpy(buf_addr, rte_pktmbuf_mtod(txm, void *), + txm->data_len); + + tx_used[nb_mbufs].len = txm->data_len; + tx_used[nb_mbufs++].flags = (txq->last_used == + tx_last) ? + NTB_FLAG_EOP : 0; + + /* update stats */ + bytes += txm->data_len; + + txm = txm->next; + + sw_ring[txq->last_used].next_id = (txq->last_used + 1) & + (txq->nb_tx_desc - 1); + sw_ring[txq->last_used].last_id = tx_last; + txq->last_used = (txq->last_used + 1) & + (txq->nb_tx_desc - 1); + } + txq->nb_tx_free -= nb_segs; + } + +end_of_tx: + if (nb_tx) { + uint16_t nb1, nb2; + if (nb_mbufs > txq->nb_tx_desc - last_used) { + nb1 = txq->nb_tx_desc - last_used; + nb2 = nb_mbufs - txq->nb_tx_desc + last_used; + } else { + nb1 = nb_mbufs; + nb2 = 0; + } + rte_memcpy(txq->tx_used_ring + last_used, tx_used, + sizeof(struct ntb_used) * nb1); + rte_memcpy(txq->tx_used_ring, tx_used + nb1, + sizeof(struct ntb_used) * nb2); + *txq->used_cnt = txq->last_used; + rte_wmb(); + + /* update queue stats */ + hw->ntb_xstats[NTB_TX_BYTES_ID + off] += bytes; + hw->ntb_xstats[NTB_TX_PKTS_ID + off] += nb_tx; + } + + return nb_tx; } static int @@ -569,16 +700,106 @@ ntb_dequeue_bufs(struct rte_rawdev *dev, unsigned int count, rte_rawdev_obj_t context) { - /* Not FIFO. Just for testing memory read. */ struct ntb_hw *hw = dev->dev_private; - unsigned int i; - size_t size; + struct ntb_rx_queue *rxq = hw->rx_queues[(size_t)context]; + struct ntb_rx_entry *sw_ring = rxq->sw_ring; + struct ntb_desc rx_desc[NTB_MAX_DESC_SIZE]; + struct rte_mbuf *first, *rxm_t; + struct rte_mbuf *prev = NULL; + volatile struct ntb_used *rx_item; + uint16_t nb_mbufs = 0; + uint16_t nb_rx = 0; + uint64_t bytes = 0; + uint16_t off, last_avail, used_cnt, used_nb; + int i; - size = (size_t)context; + if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) { + NTB_LOG(DEBUG, "Link is not up"); + return nb_rx; + } - for (i = 0; i < count; i++) - rte_memcpy(buffers[i]->buf_addr, hw->mz[i]->addr, size); - return 0; + used_cnt = *rxq->used_cnt; + + if (rxq->last_used == used_cnt) + return nb_rx; + + last_avail = rxq->last_avail; + used_nb = (used_cnt - rxq->last_used) & (rxq->nb_rx_desc - 1); + count = RTE_MIN(count, used_nb); + for (nb_rx = 0; nb_rx < count; nb_rx++) { + i = 0; + while (true) { + rx_item = rxq->rx_used_ring + rxq->last_used; + rxm_t = sw_ring[rxq->last_used].mbuf; + rxm_t->data_len = rx_item->len; + rxm_t->data_off = RTE_PKTMBUF_HEADROOM; + rxm_t->port = rxq->port_id; + + if (!i) { + rxm_t->nb_segs = 1; + first = rxm_t; + first->pkt_len = 0; + buffers[nb_rx]->buf_addr = rxm_t; + } else { + prev->next = rxm_t; + first->nb_segs++; + } + + prev = rxm_t; + first->pkt_len += prev->data_len; + rxq->last_used = (rxq->last_used + 1) & + (rxq->nb_rx_desc - 1); + + /* alloc new mbuf */ + rxm_t = rte_mbuf_raw_alloc(rxq->mpool); + if (unlikely(rxm_t == NULL)) { + NTB_LOG(ERR, "recv alloc mbuf failed."); + goto end_of_rx; + } + rxm_t->port = rxq->port_id; + sw_ring[rxq->last_avail].mbuf = rxm_t; + i++; + + /* fill new desc */ + rx_desc[nb_mbufs].addr = + rte_pktmbuf_mtod(rxm_t, size_t); + rx_desc[nb_mbufs++].len = rxm_t->buf_len - + RTE_PKTMBUF_HEADROOM; + rxq->last_avail = (rxq->last_avail + 1) & + (rxq->nb_rx_desc - 1); + + if (rx_item->flags & NTB_FLAG_EOP) + break; + } + /* update stats */ + bytes += first->pkt_len; + } + +end_of_rx: + if (nb_rx) { + uint16_t nb1, nb2; + if (nb_mbufs > rxq->nb_rx_desc - last_avail) { + nb1 = rxq->nb_rx_desc - last_avail; + nb2 = nb_mbufs - rxq->nb_rx_desc + last_avail; + } else { + nb1 = nb_mbufs; + nb2 = 0; + } + rte_memcpy(rxq->rx_desc_ring + last_avail, rx_desc, + sizeof(struct ntb_desc) * nb1); + rte_memcpy(rxq->rx_desc_ring, rx_desc + nb1, + sizeof(struct ntb_desc) * nb2); + *rxq->avail_cnt = rxq->last_avail; + rte_wmb(); + + /* update queue stats */ + off = NTB_XSTATS_NUM * ((size_t)context + 1); + hw->ntb_xstats[NTB_RX_BYTES_ID + off] += bytes; + hw->ntb_xstats[NTB_RX_PKTS_ID + off] += nb_rx; + hw->ntb_xstats[NTB_RX_MISS_ID + off] += (count - nb_rx); + } + + return nb_rx; } static void @@ -614,6 +835,7 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) { struct ntb_dev_config *conf = config; struct ntb_hw *hw = dev->dev_private; + uint32_t xstats_num; int ret; hw->queue_pairs = conf->num_queues; @@ -624,6 +846,12 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); hw->tx_queues = rte_zmalloc("ntb_tx_queues", sizeof(struct ntb_tx_queue *) * hw->queue_pairs, 0); + /* First total stats, then per queue stats. */ + xstats_num = (hw->queue_pairs + 1) * NTB_XSTATS_NUM; + hw->ntb_xstats = rte_zmalloc("ntb_xstats", xstats_num * + sizeof(uint64_t), 0); + hw->ntb_xstats_off = rte_zmalloc("ntb_xstats_off", xstats_num * + sizeof(uint64_t), 0); /* Start handshake with the peer. */ ret = ntb_handshake_work(dev); @@ -650,6 +878,12 @@ ntb_dev_start(struct rte_rawdev *dev) if (!hw->link_status || !hw->peer_dev_up) return -EINVAL; + /* Set total stats. */ + for (i = 0; i < NTB_XSTATS_NUM; i++) { + hw->ntb_xstats[i] = 0; + hw->ntb_xstats_off[i] = 0; + } + for (i = 0; i < hw->queue_pairs; i++) { ret = ntb_queue_init(dev, i); if (ret) { @@ -923,39 +1157,143 @@ ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, return -EINVAL; } +static inline uint64_t +ntb_stats_update(uint64_t offset, uint64_t stat) +{ + if (stat >= offset) + return (stat - offset); + else + return (uint64_t)(((uint64_t)-1) - offset + stat + 1); +} + static int -ntb_xstats_get(const struct rte_rawdev *dev __rte_unused, - const unsigned int ids[] __rte_unused, - uint64_t values[] __rte_unused, - unsigned int n __rte_unused) +ntb_xstats_get(const struct rte_rawdev *dev, + const unsigned int ids[], + uint64_t values[], + unsigned int n) { - return 0; + struct ntb_hw *hw = dev->dev_private; + uint32_t i, j, off, xstats_num; + + /* Calculate total stats of all queues. */ + for (i = 0; i < NTB_XSTATS_NUM; i++) { + hw->ntb_xstats[i] = 0; + for (j = 0; j < hw->queue_pairs; j++) { + off = NTB_XSTATS_NUM * (j + 1) + i; + hw->ntb_xstats[i] += + ntb_stats_update(hw->ntb_xstats_off[off], + hw->ntb_xstats[off]); + } + } + + xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); + for (i = 0; i < n && ids[i] < xstats_num; i++) { + if (ids[i] < NTB_XSTATS_NUM) + values[i] = hw->ntb_xstats[ids[i]]; + else + values[i] = + ntb_stats_update(hw->ntb_xstats_off[ids[i]], + hw->ntb_xstats[ids[i]]); + } + + return i; } static int -ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused, - struct rte_rawdev_xstats_name *xstats_names __rte_unused, - unsigned int size __rte_unused) +ntb_xstats_get_names(const struct rte_rawdev *dev, + struct rte_rawdev_xstats_name *xstats_names, + unsigned int size) { - return 0; + struct ntb_hw *hw = dev->dev_private; + uint32_t xstats_num, i, j, off; + + xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); + if (xstats_names == NULL || size < xstats_num) + return xstats_num; + + /* Total stats names */ + memcpy(xstats_names, ntb_xstats_names, sizeof(ntb_xstats_names)); + + /* Queue stats names */ + for (i = 0; i < hw->queue_pairs; i++) { + for (j = 0; j < NTB_XSTATS_NUM; j++) { + off = j + (i + 1) * NTB_XSTATS_NUM; + snprintf(xstats_names[off].name, + sizeof(xstats_names[0].name), + "%s_q%u", ntb_xstats_names[j].name, i); + } + } + + return xstats_num; } static uint64_t -ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused, - const char *name __rte_unused, - unsigned int *id __rte_unused) +ntb_xstats_get_by_name(const struct rte_rawdev *dev, + const char *name, unsigned int *id) { - return 0; + struct rte_rawdev_xstats_name *xstats_names; + struct ntb_hw *hw = dev->dev_private; + uint32_t xstats_num, i, j, off; + + if (name == NULL) + return -EINVAL; + + xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); + xstats_names = rte_zmalloc("ntb_stats_name", + sizeof(struct rte_rawdev_xstats_name) * + xstats_num, 0); + ntb_xstats_get_names(dev, xstats_names, xstats_num); + + /* Calculate total stats of all queues. */ + for (i = 0; i < NTB_XSTATS_NUM; i++) { + for (j = 0; j < hw->queue_pairs; j++) { + off = NTB_XSTATS_NUM * (j + 1) + i; + hw->ntb_xstats[i] += + ntb_stats_update(hw->ntb_xstats_off[off], + hw->ntb_xstats[off]); + } + } + + for (i = 0; i < xstats_num; i++) { + if (!strncmp(name, xstats_names[i].name, + RTE_RAW_DEV_XSTATS_NAME_SIZE)) { + *id = i; + rte_free(xstats_names); + if (i < NTB_XSTATS_NUM) + return hw->ntb_xstats[i]; + else + return ntb_stats_update(hw->ntb_xstats_off[i], + hw->ntb_xstats[i]); + } + } + + NTB_LOG(ERR, "Cannot find the xstats name."); + + return -EINVAL; } static int -ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, - const uint32_t ids[] __rte_unused, - uint32_t nb_ids __rte_unused) +ntb_xstats_reset(struct rte_rawdev *dev, + const uint32_t ids[], + uint32_t nb_ids) { - return 0; -} + struct ntb_hw *hw = dev->dev_private; + uint32_t i, j, off, xstats_num; + + xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); + for (i = 0; i < nb_ids && ids[i] < xstats_num; i++) { + if (ids[i] < NTB_XSTATS_NUM) { + for (j = 0; j < hw->queue_pairs; j++) { + off = NTB_XSTATS_NUM * (j + 1) + ids[i]; + hw->ntb_xstats_off[off] = hw->ntb_xstats[off]; + } + } else { + hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]]; + } + } + return i; +} static const struct rte_rawdev_ops ntb_ops = { .dev_info_get = ntb_dev_info_get, @@ -1158,7 +1496,7 @@ ntb_remove(struct rte_pci_device *pci_dev) static struct rte_pci_driver rte_ntb_pmd = { .id_table = pci_id_ntb_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_WC_ACTIVATE, .probe = ntb_probe, .remove = ntb_remove, };