{ .vendor_id = 0, /* sentinel */ },
};
+/* Align with enum ntb_xstats_idx */
+static struct rte_rawdev_xstats_name ntb_xstats_names[] = {
+ {"Tx-packets"},
+ {"Tx-bytes"},
+ {"Tx-errors"},
+ {"Rx-packets"},
+ {"Rx-bytes"},
+ {"Rx-missed"},
+};
+#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names)
+
static inline void
ntb_link_cleanup(struct rte_rawdev *dev)
{
txq->last_avail = 0;
txq->nb_tx_free = txq->nb_tx_desc - 1;
+ /* Set per queue stats. */
+ for (i = 0; i < NTB_XSTATS_NUM; i++) {
+ hw->ntb_xstats[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0;
+ hw->ntb_xstats_off[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0;
+ }
+
return 0;
}
+static inline void
+ntb_enqueue_cleanup(struct ntb_tx_queue *txq)
+{
+ struct ntb_tx_entry *sw_ring = txq->sw_ring;
+ uint16_t tx_free = txq->last_avail;
+ uint16_t nb_to_clean, i;
+
+ /* avail_cnt + 1 represents where to rx next in the peer. */
+ nb_to_clean = (*txq->avail_cnt - txq->last_avail + 1 +
+ txq->nb_tx_desc) & (txq->nb_tx_desc - 1);
+ nb_to_clean = RTE_MIN(nb_to_clean, txq->tx_free_thresh);
+ for (i = 0; i < nb_to_clean; i++) {
+ if (sw_ring[tx_free].mbuf)
+ rte_pktmbuf_free_seg(sw_ring[tx_free].mbuf);
+ tx_free = (tx_free + 1) & (txq->nb_tx_desc - 1);
+ }
+
+ txq->nb_tx_free += nb_to_clean;
+ txq->last_avail = tx_free;
+}
+
static int
ntb_enqueue_bufs(struct rte_rawdev *dev,
struct rte_rawdev_buf **buffers,
unsigned int count,
rte_rawdev_obj_t context)
{
- /* Not FIFO right now. Just for testing memory write. */
struct ntb_hw *hw = dev->dev_private;
- unsigned int i;
- void *bar_addr;
- size_t size;
+ struct ntb_tx_queue *txq = hw->tx_queues[(size_t)context];
+ struct ntb_tx_entry *sw_ring = txq->sw_ring;
+ struct rte_mbuf *txm;
+ struct ntb_used tx_used[NTB_MAX_DESC_SIZE];
+ volatile struct ntb_desc *tx_item;
+ uint16_t tx_last, nb_segs, off, last_used, avail_cnt;
+ uint16_t nb_mbufs = 0;
+ uint16_t nb_tx = 0;
+ uint64_t bytes = 0;
+ void *buf_addr;
+ int i;
- if (hw->ntb_ops->get_peer_mw_addr == NULL)
- return -ENOTSUP;
- bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0);
- size = (size_t)context;
+ if (unlikely(hw->ntb_ops->ioremap == NULL)) {
+ NTB_LOG(ERR, "Ioremap not supported.");
+ return nb_tx;
+ }
- for (i = 0; i < count; i++)
- rte_memcpy(bar_addr, buffers[i]->buf_addr, size);
- return 0;
+ if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) {
+ NTB_LOG(DEBUG, "Link is not up.");
+ return nb_tx;
+ }
+
+ if (txq->nb_tx_free < txq->tx_free_thresh)
+ ntb_enqueue_cleanup(txq);
+
+ off = NTB_XSTATS_NUM * ((size_t)context + 1);
+ last_used = txq->last_used;
+ avail_cnt = *txq->avail_cnt;/* Where to alloc next. */
+ for (nb_tx = 0; nb_tx < count; nb_tx++) {
+ txm = (struct rte_mbuf *)(buffers[nb_tx]->buf_addr);
+ if (txm == NULL || txq->nb_tx_free < txm->nb_segs)
+ break;
+
+ tx_last = (txq->last_used + txm->nb_segs - 1) &
+ (txq->nb_tx_desc - 1);
+ nb_segs = txm->nb_segs;
+ for (i = 0; i < nb_segs; i++) {
+ /* Not enough ring space for tx. */
+ if (txq->last_used == avail_cnt)
+ goto end_of_tx;
+ sw_ring[txq->last_used].mbuf = txm;
+ tx_item = txq->tx_desc_ring + txq->last_used;
+
+ if (!tx_item->len) {
+ (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++;
+ goto end_of_tx;
+ }
+ if (txm->data_len > tx_item->len) {
+ NTB_LOG(ERR, "Data length exceeds buf length."
+ " Only %u data would be transmitted.",
+ tx_item->len);
+ txm->data_len = tx_item->len;
+ }
+
+ /* translate remote virtual addr to bar virtual addr */
+ buf_addr = (*hw->ntb_ops->ioremap)(dev, tx_item->addr);
+ if (buf_addr == NULL) {
+ (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++;
+ NTB_LOG(ERR, "Null remap addr.");
+ goto end_of_tx;
+ }
+ rte_memcpy(buf_addr, rte_pktmbuf_mtod(txm, void *),
+ txm->data_len);
+
+ tx_used[nb_mbufs].len = txm->data_len;
+ tx_used[nb_mbufs++].flags = (txq->last_used ==
+ tx_last) ?
+ NTB_FLAG_EOP : 0;
+
+ /* update stats */
+ bytes += txm->data_len;
+
+ txm = txm->next;
+
+ sw_ring[txq->last_used].next_id = (txq->last_used + 1) &
+ (txq->nb_tx_desc - 1);
+ sw_ring[txq->last_used].last_id = tx_last;
+ txq->last_used = (txq->last_used + 1) &
+ (txq->nb_tx_desc - 1);
+ }
+ txq->nb_tx_free -= nb_segs;
+ }
+
+end_of_tx:
+ if (nb_tx) {
+ uint16_t nb1, nb2;
+ if (nb_mbufs > txq->nb_tx_desc - last_used) {
+ nb1 = txq->nb_tx_desc - last_used;
+ nb2 = nb_mbufs - txq->nb_tx_desc + last_used;
+ } else {
+ nb1 = nb_mbufs;
+ nb2 = 0;
+ }
+ rte_memcpy(txq->tx_used_ring + last_used, tx_used,
+ sizeof(struct ntb_used) * nb1);
+ rte_memcpy(txq->tx_used_ring, tx_used + nb1,
+ sizeof(struct ntb_used) * nb2);
+ *txq->used_cnt = txq->last_used;
+ rte_wmb();
+
+ /* update queue stats */
+ hw->ntb_xstats[NTB_TX_BYTES_ID + off] += bytes;
+ hw->ntb_xstats[NTB_TX_PKTS_ID + off] += nb_tx;
+ }
+
+ return nb_tx;
}
static int
unsigned int count,
rte_rawdev_obj_t context)
{
- /* Not FIFO. Just for testing memory read. */
struct ntb_hw *hw = dev->dev_private;
- unsigned int i;
- size_t size;
+ struct ntb_rx_queue *rxq = hw->rx_queues[(size_t)context];
+ struct ntb_rx_entry *sw_ring = rxq->sw_ring;
+ struct ntb_desc rx_desc[NTB_MAX_DESC_SIZE];
+ struct rte_mbuf *first, *rxm_t;
+ struct rte_mbuf *prev = NULL;
+ volatile struct ntb_used *rx_item;
+ uint16_t nb_mbufs = 0;
+ uint16_t nb_rx = 0;
+ uint64_t bytes = 0;
+ uint16_t off, last_avail, used_cnt, used_nb;
+ int i;
- size = (size_t)context;
+ if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) {
+ NTB_LOG(DEBUG, "Link is not up");
+ return nb_rx;
+ }
- for (i = 0; i < count; i++)
- rte_memcpy(buffers[i]->buf_addr, hw->mz[i]->addr, size);
- return 0;
+ used_cnt = *rxq->used_cnt;
+
+ if (rxq->last_used == used_cnt)
+ return nb_rx;
+
+ last_avail = rxq->last_avail;
+ used_nb = (used_cnt - rxq->last_used) & (rxq->nb_rx_desc - 1);
+ count = RTE_MIN(count, used_nb);
+ for (nb_rx = 0; nb_rx < count; nb_rx++) {
+ i = 0;
+ while (true) {
+ rx_item = rxq->rx_used_ring + rxq->last_used;
+ rxm_t = sw_ring[rxq->last_used].mbuf;
+ rxm_t->data_len = rx_item->len;
+ rxm_t->data_off = RTE_PKTMBUF_HEADROOM;
+ rxm_t->port = rxq->port_id;
+
+ if (!i) {
+ rxm_t->nb_segs = 1;
+ first = rxm_t;
+ first->pkt_len = 0;
+ buffers[nb_rx]->buf_addr = rxm_t;
+ } else {
+ prev->next = rxm_t;
+ first->nb_segs++;
+ }
+
+ prev = rxm_t;
+ first->pkt_len += prev->data_len;
+ rxq->last_used = (rxq->last_used + 1) &
+ (rxq->nb_rx_desc - 1);
+
+ /* alloc new mbuf */
+ rxm_t = rte_mbuf_raw_alloc(rxq->mpool);
+ if (unlikely(rxm_t == NULL)) {
+ NTB_LOG(ERR, "recv alloc mbuf failed.");
+ goto end_of_rx;
+ }
+ rxm_t->port = rxq->port_id;
+ sw_ring[rxq->last_avail].mbuf = rxm_t;
+ i++;
+
+ /* fill new desc */
+ rx_desc[nb_mbufs].addr =
+ rte_pktmbuf_mtod(rxm_t, size_t);
+ rx_desc[nb_mbufs++].len = rxm_t->buf_len -
+ RTE_PKTMBUF_HEADROOM;
+ rxq->last_avail = (rxq->last_avail + 1) &
+ (rxq->nb_rx_desc - 1);
+
+ if (rx_item->flags & NTB_FLAG_EOP)
+ break;
+ }
+ /* update stats */
+ bytes += first->pkt_len;
+ }
+
+end_of_rx:
+ if (nb_rx) {
+ uint16_t nb1, nb2;
+ if (nb_mbufs > rxq->nb_rx_desc - last_avail) {
+ nb1 = rxq->nb_rx_desc - last_avail;
+ nb2 = nb_mbufs - rxq->nb_rx_desc + last_avail;
+ } else {
+ nb1 = nb_mbufs;
+ nb2 = 0;
+ }
+ rte_memcpy(rxq->rx_desc_ring + last_avail, rx_desc,
+ sizeof(struct ntb_desc) * nb1);
+ rte_memcpy(rxq->rx_desc_ring, rx_desc + nb1,
+ sizeof(struct ntb_desc) * nb2);
+ *rxq->avail_cnt = rxq->last_avail;
+ rte_wmb();
+
+ /* update queue stats */
+ off = NTB_XSTATS_NUM * ((size_t)context + 1);
+ hw->ntb_xstats[NTB_RX_BYTES_ID + off] += bytes;
+ hw->ntb_xstats[NTB_RX_PKTS_ID + off] += nb_rx;
+ hw->ntb_xstats[NTB_RX_MISS_ID + off] += (count - nb_rx);
+ }
+
+ return nb_rx;
}
static void
{
struct ntb_dev_config *conf = config;
struct ntb_hw *hw = dev->dev_private;
+ uint32_t xstats_num;
int ret;
hw->queue_pairs = conf->num_queues;
sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0);
hw->tx_queues = rte_zmalloc("ntb_tx_queues",
sizeof(struct ntb_tx_queue *) * hw->queue_pairs, 0);
+ /* First total stats, then per queue stats. */
+ xstats_num = (hw->queue_pairs + 1) * NTB_XSTATS_NUM;
+ hw->ntb_xstats = rte_zmalloc("ntb_xstats", xstats_num *
+ sizeof(uint64_t), 0);
+ hw->ntb_xstats_off = rte_zmalloc("ntb_xstats_off", xstats_num *
+ sizeof(uint64_t), 0);
/* Start handshake with the peer. */
ret = ntb_handshake_work(dev);
if (!hw->link_status || !hw->peer_dev_up)
return -EINVAL;
+ /* Set total stats. */
+ for (i = 0; i < NTB_XSTATS_NUM; i++) {
+ hw->ntb_xstats[i] = 0;
+ hw->ntb_xstats_off[i] = 0;
+ }
+
for (i = 0; i < hw->queue_pairs; i++) {
ret = ntb_queue_init(dev, i);
if (ret) {
return -EINVAL;
}
+static inline uint64_t
+ntb_stats_update(uint64_t offset, uint64_t stat)
+{
+ if (stat >= offset)
+ return (stat - offset);
+ else
+ return (uint64_t)(((uint64_t)-1) - offset + stat + 1);
+}
+
static int
-ntb_xstats_get(const struct rte_rawdev *dev __rte_unused,
- const unsigned int ids[] __rte_unused,
- uint64_t values[] __rte_unused,
- unsigned int n __rte_unused)
+ntb_xstats_get(const struct rte_rawdev *dev,
+ const unsigned int ids[],
+ uint64_t values[],
+ unsigned int n)
{
- return 0;
+ struct ntb_hw *hw = dev->dev_private;
+ uint32_t i, j, off, xstats_num;
+
+ /* Calculate total stats of all queues. */
+ for (i = 0; i < NTB_XSTATS_NUM; i++) {
+ hw->ntb_xstats[i] = 0;
+ for (j = 0; j < hw->queue_pairs; j++) {
+ off = NTB_XSTATS_NUM * (j + 1) + i;
+ hw->ntb_xstats[i] +=
+ ntb_stats_update(hw->ntb_xstats_off[off],
+ hw->ntb_xstats[off]);
+ }
+ }
+
+ xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
+ for (i = 0; i < n && ids[i] < xstats_num; i++) {
+ if (ids[i] < NTB_XSTATS_NUM)
+ values[i] = hw->ntb_xstats[ids[i]];
+ else
+ values[i] =
+ ntb_stats_update(hw->ntb_xstats_off[ids[i]],
+ hw->ntb_xstats[ids[i]]);
+ }
+
+ return i;
}
static int
-ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused,
- struct rte_rawdev_xstats_name *xstats_names __rte_unused,
- unsigned int size __rte_unused)
+ntb_xstats_get_names(const struct rte_rawdev *dev,
+ struct rte_rawdev_xstats_name *xstats_names,
+ unsigned int size)
{
- return 0;
+ struct ntb_hw *hw = dev->dev_private;
+ uint32_t xstats_num, i, j, off;
+
+ xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
+ if (xstats_names == NULL || size < xstats_num)
+ return xstats_num;
+
+ /* Total stats names */
+ memcpy(xstats_names, ntb_xstats_names, sizeof(ntb_xstats_names));
+
+ /* Queue stats names */
+ for (i = 0; i < hw->queue_pairs; i++) {
+ for (j = 0; j < NTB_XSTATS_NUM; j++) {
+ off = j + (i + 1) * NTB_XSTATS_NUM;
+ snprintf(xstats_names[off].name,
+ sizeof(xstats_names[0].name),
+ "%s_q%u", ntb_xstats_names[j].name, i);
+ }
+ }
+
+ return xstats_num;
}
static uint64_t
-ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused,
- const char *name __rte_unused,
- unsigned int *id __rte_unused)
+ntb_xstats_get_by_name(const struct rte_rawdev *dev,
+ const char *name, unsigned int *id)
{
- return 0;
+ struct rte_rawdev_xstats_name *xstats_names;
+ struct ntb_hw *hw = dev->dev_private;
+ uint32_t xstats_num, i, j, off;
+
+ if (name == NULL)
+ return -EINVAL;
+
+ xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
+ xstats_names = rte_zmalloc("ntb_stats_name",
+ sizeof(struct rte_rawdev_xstats_name) *
+ xstats_num, 0);
+ ntb_xstats_get_names(dev, xstats_names, xstats_num);
+
+ /* Calculate total stats of all queues. */
+ for (i = 0; i < NTB_XSTATS_NUM; i++) {
+ for (j = 0; j < hw->queue_pairs; j++) {
+ off = NTB_XSTATS_NUM * (j + 1) + i;
+ hw->ntb_xstats[i] +=
+ ntb_stats_update(hw->ntb_xstats_off[off],
+ hw->ntb_xstats[off]);
+ }
+ }
+
+ for (i = 0; i < xstats_num; i++) {
+ if (!strncmp(name, xstats_names[i].name,
+ RTE_RAW_DEV_XSTATS_NAME_SIZE)) {
+ *id = i;
+ rte_free(xstats_names);
+ if (i < NTB_XSTATS_NUM)
+ return hw->ntb_xstats[i];
+ else
+ return ntb_stats_update(hw->ntb_xstats_off[i],
+ hw->ntb_xstats[i]);
+ }
+ }
+
+ NTB_LOG(ERR, "Cannot find the xstats name.");
+
+ return -EINVAL;
}
static int
-ntb_xstats_reset(struct rte_rawdev *dev __rte_unused,
- const uint32_t ids[] __rte_unused,
- uint32_t nb_ids __rte_unused)
+ntb_xstats_reset(struct rte_rawdev *dev,
+ const uint32_t ids[],
+ uint32_t nb_ids)
{
- return 0;
-}
+ struct ntb_hw *hw = dev->dev_private;
+ uint32_t i, j, off, xstats_num;
+
+ xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
+ for (i = 0; i < nb_ids && ids[i] < xstats_num; i++) {
+ if (ids[i] < NTB_XSTATS_NUM) {
+ for (j = 0; j < hw->queue_pairs; j++) {
+ off = NTB_XSTATS_NUM * (j + 1) + ids[i];
+ hw->ntb_xstats_off[off] = hw->ntb_xstats[off];
+ }
+ } else {
+ hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]];
+ }
+ }
+ return i;
+}
static const struct rte_rawdev_ops ntb_ops = {
.dev_info_get = ntb_dev_info_get,
static struct rte_pci_driver rte_ntb_pmd = {
.id_table = pci_id_ntb_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_WC_ACTIVATE,
.probe = ntb_probe,
.remove = ntb_remove,
};