#include "rte_pmd_ntb.h"
#include "ntb.h"
-int ntb_logtype;
-
static const struct rte_pci_id pci_id_ntb_map[] = {
{ RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_SKX) },
+ { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_ICX) },
{ .vendor_id = 0, /* sentinel */ },
};
hw->peer_dev_up = 0;
return;
}
+
+ /* Clear other received doorbells. */
+ (*hw->ntb_ops->db_clear)(dev, db_bits);
}
-static void
+static int
ntb_queue_conf_get(struct rte_rawdev *dev,
uint16_t queue_id,
- rte_rawdev_obj_t queue_conf)
+ rte_rawdev_obj_t queue_conf,
+ size_t conf_size)
{
struct ntb_queue_conf *q_conf = queue_conf;
struct ntb_hw *hw = dev->dev_private;
+ if (conf_size != sizeof(*q_conf))
+ return -EINVAL;
+
q_conf->tx_free_thresh = hw->tx_queues[queue_id]->tx_free_thresh;
q_conf->nb_desc = hw->rx_queues[queue_id]->nb_rx_desc;
q_conf->rx_mp = hw->rx_queues[queue_id]->mpool;
+
+ return 0;
}
static void
static int
ntb_rxq_setup(struct rte_rawdev *dev,
uint16_t qp_id,
- rte_rawdev_obj_t queue_conf)
+ rte_rawdev_obj_t queue_conf,
+ size_t conf_size)
{
struct ntb_queue_conf *rxq_conf = queue_conf;
struct ntb_hw *hw = dev->dev_private;
struct ntb_rx_queue *rxq;
+ if (conf_size != sizeof(*rxq_conf))
+ return -EINVAL;
+
/* Allocate the rx queue data structure */
rxq = rte_zmalloc_socket("ntb rx queue",
sizeof(struct ntb_rx_queue),
static int
ntb_txq_setup(struct rte_rawdev *dev,
uint16_t qp_id,
- rte_rawdev_obj_t queue_conf)
+ rte_rawdev_obj_t queue_conf,
+ size_t conf_size)
{
struct ntb_queue_conf *txq_conf = queue_conf;
struct ntb_hw *hw = dev->dev_private;
struct ntb_tx_queue *txq;
uint16_t i, prev;
+ if (conf_size != sizeof(*txq_conf))
+ return -EINVAL;
+
/* Allocate the TX queue data structure. */
txq = rte_zmalloc_socket("ntb tx queue",
sizeof(struct ntb_tx_queue),
static int
ntb_queue_setup(struct rte_rawdev *dev,
uint16_t queue_id,
- rte_rawdev_obj_t queue_conf)
+ rte_rawdev_obj_t queue_conf,
+ size_t conf_size)
{
struct ntb_hw *hw = dev->dev_private;
int ret;
if (queue_id >= hw->queue_pairs)
return -EINVAL;
- ret = ntb_txq_setup(dev, queue_id, queue_conf);
+ ret = ntb_txq_setup(dev, queue_id, queue_conf, conf_size);
if (ret < 0)
return ret;
- ret = ntb_rxq_setup(dev, queue_id, queue_conf);
+ ret = ntb_rxq_setup(dev, queue_id, queue_conf, conf_size);
return ret;
}
return 0;
}
+static inline void
+ntb_enqueue_cleanup(struct ntb_tx_queue *txq)
+{
+ struct ntb_tx_entry *sw_ring = txq->sw_ring;
+ uint16_t tx_free = txq->last_avail;
+ uint16_t nb_to_clean, i;
+
+ /* avail_cnt + 1 represents where to rx next in the peer. */
+ nb_to_clean = (*txq->avail_cnt - txq->last_avail + 1 +
+ txq->nb_tx_desc) & (txq->nb_tx_desc - 1);
+ nb_to_clean = RTE_MIN(nb_to_clean, txq->tx_free_thresh);
+ for (i = 0; i < nb_to_clean; i++) {
+ if (sw_ring[tx_free].mbuf)
+ rte_pktmbuf_free_seg(sw_ring[tx_free].mbuf);
+ tx_free = (tx_free + 1) & (txq->nb_tx_desc - 1);
+ }
+
+ txq->nb_tx_free += nb_to_clean;
+ txq->last_avail = tx_free;
+}
+
static int
ntb_enqueue_bufs(struct rte_rawdev *dev,
struct rte_rawdev_buf **buffers,
unsigned int count,
rte_rawdev_obj_t context)
{
- /* Not FIFO right now. Just for testing memory write. */
struct ntb_hw *hw = dev->dev_private;
- unsigned int i;
- void *bar_addr;
- size_t size;
+ struct ntb_tx_queue *txq = hw->tx_queues[(size_t)context];
+ struct ntb_tx_entry *sw_ring = txq->sw_ring;
+ struct rte_mbuf *txm;
+ struct ntb_used tx_used[NTB_MAX_DESC_SIZE];
+ volatile struct ntb_desc *tx_item;
+ uint16_t tx_last, nb_segs, off, last_used, avail_cnt;
+ uint16_t nb_mbufs = 0;
+ uint16_t nb_tx = 0;
+ uint64_t bytes = 0;
+ void *buf_addr;
+ int i;
- if (hw->ntb_ops->get_peer_mw_addr == NULL)
- return -ENOTSUP;
- bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0);
- size = (size_t)context;
+ if (unlikely(hw->ntb_ops->ioremap == NULL)) {
+ NTB_LOG(ERR, "Ioremap not supported.");
+ return nb_tx;
+ }
- for (i = 0; i < count; i++)
- rte_memcpy(bar_addr, buffers[i]->buf_addr, size);
- return 0;
+ if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) {
+ NTB_LOG(DEBUG, "Link is not up.");
+ return nb_tx;
+ }
+
+ if (txq->nb_tx_free < txq->tx_free_thresh)
+ ntb_enqueue_cleanup(txq);
+
+ off = NTB_XSTATS_NUM * ((size_t)context + 1);
+ last_used = txq->last_used;
+ avail_cnt = *txq->avail_cnt;/* Where to alloc next. */
+ for (nb_tx = 0; nb_tx < count; nb_tx++) {
+ txm = (struct rte_mbuf *)(buffers[nb_tx]->buf_addr);
+ if (txm == NULL || txq->nb_tx_free < txm->nb_segs)
+ break;
+
+ tx_last = (txq->last_used + txm->nb_segs - 1) &
+ (txq->nb_tx_desc - 1);
+ nb_segs = txm->nb_segs;
+ for (i = 0; i < nb_segs; i++) {
+ /* Not enough ring space for tx. */
+ if (txq->last_used == avail_cnt)
+ goto end_of_tx;
+ sw_ring[txq->last_used].mbuf = txm;
+ tx_item = txq->tx_desc_ring + txq->last_used;
+
+ if (!tx_item->len) {
+ (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++;
+ goto end_of_tx;
+ }
+ if (txm->data_len > tx_item->len) {
+ NTB_LOG(ERR, "Data length exceeds buf length."
+ " Only %u data would be transmitted.",
+ tx_item->len);
+ txm->data_len = tx_item->len;
+ }
+
+ /* translate remote virtual addr to bar virtual addr */
+ buf_addr = (*hw->ntb_ops->ioremap)(dev, tx_item->addr);
+ if (buf_addr == NULL) {
+ (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++;
+ NTB_LOG(ERR, "Null remap addr.");
+ goto end_of_tx;
+ }
+ rte_memcpy(buf_addr, rte_pktmbuf_mtod(txm, void *),
+ txm->data_len);
+
+ tx_used[nb_mbufs].len = txm->data_len;
+ tx_used[nb_mbufs++].flags = (txq->last_used ==
+ tx_last) ?
+ NTB_FLAG_EOP : 0;
+
+ /* update stats */
+ bytes += txm->data_len;
+
+ txm = txm->next;
+
+ sw_ring[txq->last_used].next_id = (txq->last_used + 1) &
+ (txq->nb_tx_desc - 1);
+ sw_ring[txq->last_used].last_id = tx_last;
+ txq->last_used = (txq->last_used + 1) &
+ (txq->nb_tx_desc - 1);
+ }
+ txq->nb_tx_free -= nb_segs;
+ }
+
+end_of_tx:
+ if (nb_tx) {
+ uint16_t nb1, nb2;
+ if (nb_mbufs > txq->nb_tx_desc - last_used) {
+ nb1 = txq->nb_tx_desc - last_used;
+ nb2 = nb_mbufs - txq->nb_tx_desc + last_used;
+ } else {
+ nb1 = nb_mbufs;
+ nb2 = 0;
+ }
+ rte_memcpy(txq->tx_used_ring + last_used, tx_used,
+ sizeof(struct ntb_used) * nb1);
+ rte_memcpy(txq->tx_used_ring, tx_used + nb1,
+ sizeof(struct ntb_used) * nb2);
+ rte_wmb();
+ *txq->used_cnt = txq->last_used;
+
+ /* update queue stats */
+ hw->ntb_xstats[NTB_TX_BYTES_ID + off] += bytes;
+ hw->ntb_xstats[NTB_TX_PKTS_ID + off] += nb_tx;
+ }
+
+ return nb_tx;
}
static int
unsigned int count,
rte_rawdev_obj_t context)
{
- /* Not FIFO. Just for testing memory read. */
struct ntb_hw *hw = dev->dev_private;
- unsigned int i;
- size_t size;
+ struct ntb_rx_queue *rxq = hw->rx_queues[(size_t)context];
+ struct ntb_rx_entry *sw_ring = rxq->sw_ring;
+ struct ntb_desc rx_desc[NTB_MAX_DESC_SIZE];
+ struct rte_mbuf *first, *rxm_t;
+ struct rte_mbuf *prev = NULL;
+ volatile struct ntb_used *rx_item;
+ uint16_t nb_mbufs = 0;
+ uint16_t nb_rx = 0;
+ uint64_t bytes = 0;
+ uint16_t off, last_avail, used_cnt, used_nb;
+ int i;
- size = (size_t)context;
+ if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) {
+ NTB_LOG(DEBUG, "Link is not up");
+ return nb_rx;
+ }
+
+ used_cnt = *rxq->used_cnt;
+
+ if (rxq->last_used == used_cnt)
+ return nb_rx;
+
+ last_avail = rxq->last_avail;
+ used_nb = (used_cnt - rxq->last_used) & (rxq->nb_rx_desc - 1);
+ count = RTE_MIN(count, used_nb);
+ for (nb_rx = 0; nb_rx < count; nb_rx++) {
+ i = 0;
+ while (true) {
+ rx_item = rxq->rx_used_ring + rxq->last_used;
+ rxm_t = sw_ring[rxq->last_used].mbuf;
+ rxm_t->data_len = rx_item->len;
+ rxm_t->data_off = RTE_PKTMBUF_HEADROOM;
+ rxm_t->port = rxq->port_id;
+
+ if (!i) {
+ rxm_t->nb_segs = 1;
+ first = rxm_t;
+ first->pkt_len = 0;
+ buffers[nb_rx]->buf_addr = rxm_t;
+ } else {
+ prev->next = rxm_t;
+ first->nb_segs++;
+ }
- for (i = 0; i < count; i++)
- rte_memcpy(buffers[i]->buf_addr, hw->mz[i]->addr, size);
- return 0;
+ prev = rxm_t;
+ first->pkt_len += prev->data_len;
+ rxq->last_used = (rxq->last_used + 1) &
+ (rxq->nb_rx_desc - 1);
+
+ /* alloc new mbuf */
+ rxm_t = rte_mbuf_raw_alloc(rxq->mpool);
+ if (unlikely(rxm_t == NULL)) {
+ NTB_LOG(ERR, "recv alloc mbuf failed.");
+ goto end_of_rx;
+ }
+ rxm_t->port = rxq->port_id;
+ sw_ring[rxq->last_avail].mbuf = rxm_t;
+ i++;
+
+ /* fill new desc */
+ rx_desc[nb_mbufs].addr =
+ rte_pktmbuf_mtod(rxm_t, size_t);
+ rx_desc[nb_mbufs++].len = rxm_t->buf_len -
+ RTE_PKTMBUF_HEADROOM;
+ rxq->last_avail = (rxq->last_avail + 1) &
+ (rxq->nb_rx_desc - 1);
+
+ if (rx_item->flags & NTB_FLAG_EOP)
+ break;
+ }
+ /* update stats */
+ bytes += first->pkt_len;
+ }
+
+end_of_rx:
+ if (nb_rx) {
+ uint16_t nb1, nb2;
+ if (nb_mbufs > rxq->nb_rx_desc - last_avail) {
+ nb1 = rxq->nb_rx_desc - last_avail;
+ nb2 = nb_mbufs - rxq->nb_rx_desc + last_avail;
+ } else {
+ nb1 = nb_mbufs;
+ nb2 = 0;
+ }
+ rte_memcpy(rxq->rx_desc_ring + last_avail, rx_desc,
+ sizeof(struct ntb_desc) * nb1);
+ rte_memcpy(rxq->rx_desc_ring, rx_desc + nb1,
+ sizeof(struct ntb_desc) * nb2);
+ rte_wmb();
+ *rxq->avail_cnt = rxq->last_avail;
+
+ /* update queue stats */
+ off = NTB_XSTATS_NUM * ((size_t)context + 1);
+ hw->ntb_xstats[NTB_RX_BYTES_ID + off] += bytes;
+ hw->ntb_xstats[NTB_RX_PKTS_ID + off] += nb_rx;
+ hw->ntb_xstats[NTB_RX_MISS_ID + off] += (count - nb_rx);
+ }
+
+ return nb_rx;
}
-static void
-ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info)
+static int
+ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info,
+ size_t dev_info_size)
{
struct ntb_hw *hw = dev->dev_private;
struct ntb_dev_info *info = dev_info;
+ if (dev_info_size != sizeof(*info)) {
+ NTB_LOG(ERR, "Invalid size parameter to %s", __func__);
+ return -EINVAL;
+ }
+
info->mw_cnt = hw->mw_cnt;
info->mw_size = hw->mw_size;
if (!hw->queue_size || !hw->queue_pairs) {
NTB_LOG(ERR, "No queue size and queue num assigned.");
- return;
+ return -EAGAIN;
}
hw->hdr_size_per_queue = RTE_ALIGN(sizeof(struct ntb_header) +
hw->queue_size * sizeof(struct ntb_used),
RTE_CACHE_LINE_SIZE);
info->ntb_hdr_size = hw->hdr_size_per_queue * hw->queue_pairs;
+
+ return 0;
}
static int
-ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config)
+ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config,
+ size_t config_size)
{
struct ntb_dev_config *conf = config;
struct ntb_hw *hw = dev->dev_private;
uint32_t xstats_num;
int ret;
+ if (conf == NULL || config_size != sizeof(*conf))
+ return -EINVAL;
+
hw->queue_pairs = conf->num_queues;
hw->queue_size = conf->queue_size;
hw->used_mw_num = conf->mz_num;
hw->peer_mw_base = rte_zmalloc("ntb_peer_mw_base", hw->mw_cnt *
sizeof(uint64_t), 0);
+ if (hw->peer_mw_base == NULL) {
+ NTB_LOG(ERR, "Cannot allocate memory for peer mw base.");
+ ret = -ENOMEM;
+ goto err_q_init;
+ }
if (hw->ntb_ops->spad_read == NULL) {
ret = -ENOTSUP;
ntb_queue_release(dev, i);
hw->queue_pairs = 0;
- intr_handle = &hw->pci_dev->intr_handle;
+ intr_handle = hw->pci_dev->intr_handle;
/* Clean datapath event and vec mapping */
rte_intr_efd_disable(intr_handle);
- if (intr_handle->intr_vec) {
- rte_free(intr_handle->intr_vec);
- intr_handle->intr_vec = NULL;
- }
+ rte_intr_vec_list_free(intr_handle);
/* Disable uio intr before callback unregister */
rte_intr_disable(intr_handle);
if (hw->ntb_ops->spad_write == NULL)
return -ENOTSUP;
index = atoi(&attr_name[NTB_SPAD_USER_LEN]);
+ if (index < 0 || index >= NTB_SPAD_USER_MAX_NUM) {
+ NTB_LOG(ERR, "Invalid attribute (%s)", attr_name);
+ return -EINVAL;
+ }
(*hw->ntb_ops->spad_write)(dev, hw->spad_user_list[index],
1, attr_value);
NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")",
if (hw->ntb_ops->spad_read == NULL)
return -ENOTSUP;
index = atoi(&attr_name[NTB_SPAD_USER_LEN]);
+ if (index < 0 || index >= NTB_SPAD_USER_MAX_NUM) {
+ NTB_LOG(ERR, "Attribute (%s) out of range", attr_name);
+ return -EINVAL;
+ }
*attr_value = (*hw->ntb_ops->spad_read)(dev,
hw->spad_user_list[index], 0);
NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
switch (pci_dev->id.device_id) {
case NTB_INTEL_DEV_ID_B2B_SKX:
+ case NTB_INTEL_DEV_ID_B2B_ICX:
hw->ntb_ops = &intel_ntb_ops;
break;
default:
/* Init doorbell. */
hw->db_valid_mask = RTE_LEN2MASK(hw->db_cnt, uint64_t);
- intr_handle = &pci_dev->intr_handle;
+ intr_handle = pci_dev->intr_handle;
/* Register callback func to eal lib */
rte_intr_callback_register(intr_handle,
ntb_dev_intr_handler, dev);
static struct rte_pci_driver rte_ntb_pmd = {
.id_table = pci_id_ntb_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_WC_ACTIVATE,
.probe = ntb_probe,
.remove = ntb_remove,
};
RTE_PMD_REGISTER_PCI(raw_ntb, rte_ntb_pmd);
RTE_PMD_REGISTER_PCI_TABLE(raw_ntb, pci_id_ntb_map);
RTE_PMD_REGISTER_KMOD_DEP(raw_ntb, "* igb_uio | uio_pci_generic | vfio-pci");
-
-RTE_INIT(ntb_init_log)
-{
- ntb_logtype = rte_log_register("pmd.raw.ntb");
- if (ntb_logtype >= 0)
- rte_log_set_level(ntb_logtype, RTE_LOG_INFO);
-}
+RTE_LOG_REGISTER_DEFAULT(ntb_logtype, INFO);
git.droids-corp.org / dpdk.git / blobdiff