#include <linux/if.h>
#include <linux/ethtool.h>
#include <linux/sockios.h>
+#include <fcntl.h>
/* Verbs header. */
/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-pedantic"
#endif
-#include <rte_config.h>
#include <rte_ether.h>
#include <rte_ethdev.h>
#include <rte_dev.h>
#include <rte_atomic.h>
#include <rte_version.h>
#include <rte_log.h>
+#include <rte_alarm.h>
#ifdef PEDANTIC
#pragma GCC diagnostic error "-pedantic"
#endif
#define WR_ID(o) (((wr_id_t *)&(o))->data)
-/* Compile-time check. */
-static inline void wr_id_t_check(void)
-{
- wr_id_t check[1 + (2 * -!(sizeof(wr_id_t) == sizeof(uint64_t)))];
-
- (void)check;
- (void)wr_id_t_check;
-}
+/* Transpose flags. Useful to convert IBV to DPDK flags. */
+#define TRANSPOSE(val, from, to) \
+ (((from) >= (to)) ? \
+ (((val) & (from)) / ((from) / (to))) : \
+ (((val) & (from)) * ((to) / (from))))
struct mlx4_rxq_stats {
unsigned int idx; /**< Mapping index. */
struct rxq_elt (*no_sp)[]; /* RX elements. */
} elts;
unsigned int sp:1; /* Use scattered RX elements. */
+ unsigned int csum:1; /* Enable checksum offloading. */
+ unsigned int csum_l2tun:1; /* Same for L2 tunnels. */
uint32_t mb_len; /* Length of a mp-issued mbuf. */
struct mlx4_rxq_stats stats; /* RX queue counters. */
unsigned int socket; /* CPU socket ID for allocations. */
+ struct ibv_exp_res_domain *rd; /* Resource Domain. */
};
/* TX element. */
struct txq_elt {
- struct ibv_send_wr wr; /* Work Request. */
- struct ibv_sge sges[MLX4_PMD_SGE_WR_N]; /* Scatter/Gather Elements. */
- /* mbuf pointer is derived from WR_ID(wr.wr_id).offset. */
+ struct rte_mbuf *buf;
};
/* Linear buffer type. It is used when transmitting buffers with too many
struct txq {
struct priv *priv; /* Back pointer to private data. */
struct {
- struct rte_mempool *mp; /* Cached Memory Pool. */
+ const struct rte_mempool *mp; /* Cached Memory Pool. */
struct ibv_mr *mr; /* Memory Region (for mp). */
uint32_t lkey; /* mr->lkey */
} mp2mr[MLX4_PMD_TX_MP_CACHE]; /* MP to MR translation table. */
linear_t (*elts_linear)[]; /* Linearized buffers. */
struct ibv_mr *mr_linear; /* Memory Region for linearized buffers. */
unsigned int socket; /* CPU socket ID for allocations. */
+ struct ibv_exp_res_domain *rd; /* Resource Domain. */
};
struct priv {
unsigned int hw_qpg:1; /* QP groups are supported. */
unsigned int hw_tss:1; /* TSS is supported. */
unsigned int hw_rss:1; /* RSS is supported. */
+ unsigned int hw_csum:1; /* Checksum offload is supported. */
+ unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */
unsigned int rss:1; /* RSS is enabled. */
unsigned int vf:1; /* This is a VF device. */
+ unsigned int pending_alarm:1; /* An alarm is pending. */
#ifdef INLINE_RECV
unsigned int inl_recv_size; /* Inline recv size */
#endif
unsigned int txqs_n; /* TX queues array size. */
struct rxq *(*rxqs)[]; /* RX queues. */
struct txq *(*txqs)[]; /* TX queues. */
+ struct rte_intr_handle intr_handle; /* Interrupt handler. */
rte_spinlock_t lock; /* Lock for control functions. */
};
+/* Local storage for secondary process data. */
+struct mlx4_secondary_data {
+ struct rte_eth_dev_data data; /* Local device data. */
+ struct priv *primary_priv; /* Private structure from primary. */
+ struct rte_eth_dev_data *shared_dev_data; /* Shared device data. */
+ rte_spinlock_t lock; /* Port configuration lock. */
+} mlx4_secondary_data[RTE_MAX_ETHPORTS];
+
+/**
+ * Check if running as a secondary process.
+ *
+ * @return
+ * Nonzero if running as a secondary process.
+ */
+static inline int
+mlx4_is_secondary(void)
+{
+ return rte_eal_process_type() != RTE_PROC_PRIMARY;
+}
+
+/**
+ * Return private structure associated with an Ethernet device.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * Pointer to private structure.
+ */
+static struct priv *
+mlx4_get_priv(struct rte_eth_dev *dev)
+{
+ struct mlx4_secondary_data *sd;
+
+ if (!mlx4_is_secondary())
+ return dev->data->dev_private;
+ sd = &mlx4_secondary_data[dev->data->port_id];
+ return sd->data.dev_private;
+}
+
/**
* Lock private structure to protect it from concurrent access in the
* control path.
/* Device configuration. */
+static int
+txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
+ unsigned int socket, const struct rte_eth_txconf *conf);
+
+static void
+txq_cleanup(struct txq *txq);
+
static int
rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
unsigned int socket, const struct rte_eth_rxconf *conf,
}
if (rxqs_n == priv->rxqs_n)
return 0;
+ if ((rxqs_n & (rxqs_n - 1)) != 0) {
+ ERROR("%p: invalid number of RX queues (%u),"
+ " must be a power of 2",
+ (void *)dev, rxqs_n);
+ return EINVAL;
+ }
INFO("%p: RX queues number update: %u -> %u",
(void *)dev, priv->rxqs_n, rxqs_n);
/* If RSS is enabled, disable it first. */
struct priv *priv = dev->data->dev_private;
int ret;
+ if (mlx4_is_secondary())
+ return -E_RTE_SECONDARY;
priv_lock(priv);
ret = dev_configure(dev);
assert(ret >= 0);
return -ret;
}
+static uint16_t mlx4_tx_burst(void *, struct rte_mbuf **, uint16_t);
+static uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t);
+
+/**
+ * Configure secondary process queues from a private data pointer (primary
+ * or secondary) and update burst callbacks. Can take place only once.
+ *
+ * All queues must have been previously created by the primary process to
+ * avoid undefined behavior.
+ *
+ * @param priv
+ * Private data pointer from either primary or secondary process.
+ *
+ * @return
+ * Private data pointer from secondary process, NULL in case of error.
+ */
+static struct priv *
+mlx4_secondary_data_setup(struct priv *priv)
+{
+ unsigned int port_id = 0;
+ struct mlx4_secondary_data *sd;
+ void **tx_queues;
+ void **rx_queues;
+ unsigned int nb_tx_queues;
+ unsigned int nb_rx_queues;
+ unsigned int i;
+
+ /* priv must be valid at this point. */
+ assert(priv != NULL);
+ /* priv->dev must also be valid but may point to local memory from
+ * another process, possibly with the same address and must not
+ * be dereferenced yet. */
+ assert(priv->dev != NULL);
+ /* Determine port ID by finding out where priv comes from. */
+ while (1) {
+ sd = &mlx4_secondary_data[port_id];
+ rte_spinlock_lock(&sd->lock);
+ /* Primary process? */
+ if (sd->primary_priv == priv)
+ break;
+ /* Secondary process? */
+ if (sd->data.dev_private == priv)
+ break;
+ rte_spinlock_unlock(&sd->lock);
+ if (++port_id == RTE_DIM(mlx4_secondary_data))
+ port_id = 0;
+ }
+ /* Switch to secondary private structure. If private data has already
+ * been updated by another thread, there is nothing else to do. */
+ priv = sd->data.dev_private;
+ if (priv->dev->data == &sd->data)
+ goto end;
+ /* Sanity checks. Secondary private structure is supposed to point
+ * to local eth_dev, itself still pointing to the shared device data
+ * structure allocated by the primary process. */
+ assert(sd->shared_dev_data != &sd->data);
+ assert(sd->data.nb_tx_queues == 0);
+ assert(sd->data.tx_queues == NULL);
+ assert(sd->data.nb_rx_queues == 0);
+ assert(sd->data.rx_queues == NULL);
+ assert(priv != sd->primary_priv);
+ assert(priv->dev->data == sd->shared_dev_data);
+ assert(priv->txqs_n == 0);
+ assert(priv->txqs == NULL);
+ assert(priv->rxqs_n == 0);
+ assert(priv->rxqs == NULL);
+ nb_tx_queues = sd->shared_dev_data->nb_tx_queues;
+ nb_rx_queues = sd->shared_dev_data->nb_rx_queues;
+ /* Allocate local storage for queues. */
+ tx_queues = rte_zmalloc("secondary ethdev->tx_queues",
+ sizeof(sd->data.tx_queues[0]) * nb_tx_queues,
+ RTE_CACHE_LINE_SIZE);
+ rx_queues = rte_zmalloc("secondary ethdev->rx_queues",
+ sizeof(sd->data.rx_queues[0]) * nb_rx_queues,
+ RTE_CACHE_LINE_SIZE);
+ if (tx_queues == NULL || rx_queues == NULL)
+ goto error;
+ /* Lock to prevent control operations during setup. */
+ priv_lock(priv);
+ /* TX queues. */
+ for (i = 0; i != nb_tx_queues; ++i) {
+ struct txq *primary_txq = (*sd->primary_priv->txqs)[i];
+ struct txq *txq;
+
+ if (primary_txq == NULL)
+ continue;
+ txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0,
+ primary_txq->socket);
+ if (txq != NULL) {
+ if (txq_setup(priv->dev,
+ txq,
+ primary_txq->elts_n * MLX4_PMD_SGE_WR_N,
+ primary_txq->socket,
+ NULL) == 0) {
+ txq->stats.idx = primary_txq->stats.idx;
+ tx_queues[i] = txq;
+ continue;
+ }
+ rte_free(txq);
+ }
+ while (i) {
+ txq = tx_queues[--i];
+ txq_cleanup(txq);
+ rte_free(txq);
+ }
+ goto error;
+ }
+ /* RX queues. */
+ for (i = 0; i != nb_rx_queues; ++i) {
+ struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i];
+
+ if (primary_rxq == NULL)
+ continue;
+ /* Not supported yet. */
+ rx_queues[i] = NULL;
+ }
+ /* Update everything. */
+ priv->txqs = (void *)tx_queues;
+ priv->txqs_n = nb_tx_queues;
+ priv->rxqs = (void *)rx_queues;
+ priv->rxqs_n = nb_rx_queues;
+ sd->data.rx_queues = rx_queues;
+ sd->data.tx_queues = tx_queues;
+ sd->data.nb_rx_queues = nb_rx_queues;
+ sd->data.nb_tx_queues = nb_tx_queues;
+ sd->data.dev_link = sd->shared_dev_data->dev_link;
+ sd->data.mtu = sd->shared_dev_data->mtu;
+ memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state,
+ sizeof(sd->data.rx_queue_state));
+ memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state,
+ sizeof(sd->data.tx_queue_state));
+ sd->data.dev_flags = sd->shared_dev_data->dev_flags;
+ /* Use local data from now on. */
+ rte_mb();
+ priv->dev->data = &sd->data;
+ rte_mb();
+ priv->dev->tx_pkt_burst = mlx4_tx_burst;
+ priv->dev->rx_pkt_burst = removed_rx_burst;
+ priv_unlock(priv);
+end:
+ /* More sanity checks. */
+ assert(priv->dev->tx_pkt_burst == mlx4_tx_burst);
+ assert(priv->dev->rx_pkt_burst == removed_rx_burst);
+ assert(priv->dev->data == &sd->data);
+ rte_spinlock_unlock(&sd->lock);
+ return priv;
+error:
+ priv_unlock(priv);
+ rte_free(tx_queues);
+ rte_free(rx_queues);
+ rte_spinlock_unlock(&sd->lock);
+ return NULL;
+}
+
/* TX queues handling. */
/**
}
for (i = 0; (i != elts_n); ++i) {
struct txq_elt *elt = &(*elts)[i];
- struct ibv_send_wr *wr = &elt->wr;
- /* Configure WR. */
- WR_ID(wr->wr_id).id = i;
- WR_ID(wr->wr_id).offset = 0;
- wr->sg_list = &elt->sges[0];
- wr->opcode = IBV_WR_SEND;
- /* Other fields are updated during TX. */
+ elt->buf = NULL;
}
DEBUG("%p: allocated and configured %u WRs", (void *)txq, elts_n);
txq->elts_n = elts_n;
for (i = 0; (i != elemof(*elts)); ++i) {
struct txq_elt *elt = &(*elts)[i];
- if (WR_ID(elt->wr.wr_id).offset == 0)
+ if (elt->buf == NULL)
continue;
- rte_pktmbuf_free((void *)((uintptr_t)elt->sges[0].addr -
- WR_ID(elt->wr.wr_id).offset));
+ rte_pktmbuf_free(elt->buf);
}
rte_free(elts);
}
claim_zero(ibv_destroy_qp(txq->qp));
if (txq->cq != NULL)
claim_zero(ibv_destroy_cq(txq->cq));
+ if (txq->rd != NULL) {
+ struct ibv_exp_destroy_res_domain_attr attr = {
+ .comp_mask = 0,
+ };
+
+ assert(txq->priv != NULL);
+ assert(txq->priv->ctx != NULL);
+ claim_zero(ibv_exp_destroy_res_domain(txq->priv->ctx,
+ txq->rd,
+ &attr));
+ }
for (i = 0; (i != elemof(txq->mp2mr)); ++i) {
if (txq->mp2mr[i].mp == NULL)
break;
return 0;
}
+/**
+ * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
+ * the cloned mbuf is allocated is returned instead.
+ *
+ * @param buf
+ * Pointer to mbuf.
+ *
+ * @return
+ * Memory pool where data is located for given mbuf.
+ */
+static struct rte_mempool *
+txq_mb2mp(struct rte_mbuf *buf)
+{
+ if (unlikely(RTE_MBUF_INDIRECT(buf)))
+ return rte_mbuf_from_indirect(buf)->pool;
+ return buf->pool;
+}
+
/**
* Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[].
* Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
* mr->lkey on success, (uint32_t)-1 on failure.
*/
static uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
+txq_mp2mr(struct txq *txq, const struct rte_mempool *mp)
{
unsigned int i;
struct ibv_mr *mr;
}
}
/* Add a new entry, register MR first. */
- DEBUG("%p: discovered new memory pool %p", (void *)txq, (void *)mp);
+ DEBUG("%p: discovered new memory pool \"%s\" (%p)",
+ (void *)txq, mp->name, (const void *)mp);
mr = ibv_reg_mr(txq->priv->pd,
(void *)mp->elt_va_start,
(mp->elt_va_end - mp->elt_va_start),
DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
(void *)txq);
--i;
- claim_zero(ibv_dereg_mr(txq->mp2mr[i].mr));
+ claim_zero(ibv_dereg_mr(txq->mp2mr[0].mr));
memmove(&txq->mp2mr[0], &txq->mp2mr[1],
(sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
}
txq->mp2mr[i].mp = mp;
txq->mp2mr[i].mr = mr;
txq->mp2mr[i].lkey = mr->lkey;
- DEBUG("%p: new MR lkey for MP %p: 0x%08" PRIu32,
- (void *)txq, (void *)mp, txq->mp2mr[i].lkey);
+ DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
+ (void *)txq, mp->name, (const void *)mp, txq->mp2mr[i].lkey);
return txq->mp2mr[i].lkey;
}
+struct txq_mp2mr_mbuf_check_data {
+ const struct rte_mempool *mp;
+ int ret;
+};
+
+/**
+ * Callback function for rte_mempool_obj_iter() to check whether a given
+ * mempool object looks like a mbuf.
+ *
+ * @param[in, out] arg
+ * Context data (struct txq_mp2mr_mbuf_check_data). Contains mempool pointer
+ * and return value.
+ * @param[in] start
+ * Object start address.
+ * @param[in] end
+ * Object end address.
+ * @param index
+ * Unused.
+ *
+ * @return
+ * Nonzero value when object is not a mbuf.
+ */
+static void
+txq_mp2mr_mbuf_check(void *arg, void *start, void *end,
+ uint32_t index __rte_unused)
+{
+ struct txq_mp2mr_mbuf_check_data *data = arg;
+ struct rte_mbuf *buf =
+ (void *)((uintptr_t)start + data->mp->header_size);
+
+ (void)index;
+ /* Check whether mbuf structure fits element size and whether mempool
+ * pointer is valid. */
+ if (((uintptr_t)end >= (uintptr_t)(buf + 1)) &&
+ (buf->pool == data->mp))
+ data->ret = 0;
+ else
+ data->ret = -1;
+}
+
+/**
+ * Iterator function for rte_mempool_walk() to register existing mempools and
+ * fill the MP to MR cache of a TX queue.
+ *
+ * @param[in] mp
+ * Memory Pool to register.
+ * @param *arg
+ * Pointer to TX queue structure.
+ */
+static void
+txq_mp2mr_iter(const struct rte_mempool *mp, void *arg)
+{
+ struct txq *txq = arg;
+ struct txq_mp2mr_mbuf_check_data data = {
+ .mp = mp,
+ .ret = -1,
+ };
+
+ /* Discard empty mempools. */
+ if (mp->size == 0)
+ return;
+ /* Register mempool only if the first element looks like a mbuf. */
+ rte_mempool_obj_iter((void *)mp->elt_va_start,
+ 1,
+ mp->header_size + mp->elt_size + mp->trailer_size,
+ 1,
+ mp->elt_pa,
+ mp->pg_num,
+ mp->pg_shift,
+ txq_mp2mr_mbuf_check,
+ &data);
+ if (data.ret)
+ return;
+ txq_mp2mr(txq, mp);
+}
+
+#if MLX4_PMD_SGE_WR_N > 1
+
/**
* Copy scattered mbuf contents to a single linear buffer.
*
return size;
}
+/**
+ * Handle scattered buffers for mlx4_tx_burst().
+ *
+ * @param txq
+ * TX queue structure.
+ * @param segs
+ * Number of segments in buf.
+ * @param elt
+ * TX queue element to fill.
+ * @param[in] buf
+ * Buffer to process.
+ * @param elts_head
+ * Index of the linear buffer to use if necessary (normally txq->elts_head).
+ * @param[out] sges
+ * Array filled with SGEs on success.
+ *
+ * @return
+ * A structure containing the processed packet size in bytes and the
+ * number of SGEs. Both fields are set to (unsigned int)-1 in case of
+ * failure.
+ */
+static struct tx_burst_sg_ret {
+ unsigned int length;
+ unsigned int num;
+}
+tx_burst_sg(struct txq *txq, unsigned int segs, struct txq_elt *elt,
+ struct rte_mbuf *buf, unsigned int elts_head,
+ struct ibv_sge (*sges)[MLX4_PMD_SGE_WR_N])
+{
+ unsigned int sent_size = 0;
+ unsigned int j;
+ int linearize = 0;
+
+ /* When there are too many segments, extra segments are
+ * linearized in the last SGE. */
+ if (unlikely(segs > elemof(*sges))) {
+ segs = (elemof(*sges) - 1);
+ linearize = 1;
+ }
+ /* Update element. */
+ elt->buf = buf;
+ /* Register segments as SGEs. */
+ for (j = 0; (j != segs); ++j) {
+ struct ibv_sge *sge = &(*sges)[j];
+ uint32_t lkey;
+
+ /* Retrieve Memory Region key for this memory pool. */
+ lkey = txq_mp2mr(txq, txq_mb2mp(buf));
+ if (unlikely(lkey == (uint32_t)-1)) {
+ /* MR does not exist. */
+ DEBUG("%p: unable to get MP <-> MR association",
+ (void *)txq);
+ /* Clean up TX element. */
+ elt->buf = NULL;
+ goto stop;
+ }
+ /* Update SGE. */
+ sge->addr = rte_pktmbuf_mtod(buf, uintptr_t);
+ if (txq->priv->vf)
+ rte_prefetch0((volatile void *)
+ (uintptr_t)sge->addr);
+ sge->length = DATA_LEN(buf);
+ sge->lkey = lkey;
+ sent_size += sge->length;
+ buf = NEXT(buf);
+ }
+ /* If buf is not NULL here and is not going to be linearized,
+ * nb_segs is not valid. */
+ assert(j == segs);
+ assert((buf == NULL) || (linearize));
+ /* Linearize extra segments. */
+ if (linearize) {
+ struct ibv_sge *sge = &(*sges)[segs];
+ linear_t *linear = &(*txq->elts_linear)[elts_head];
+ unsigned int size = linearize_mbuf(linear, buf);
+
+ assert(segs == (elemof(*sges) - 1));
+ if (size == 0) {
+ /* Invalid packet. */
+ DEBUG("%p: packet too large to be linearized.",
+ (void *)txq);
+ /* Clean up TX element. */
+ elt->buf = NULL;
+ goto stop;
+ }
+ /* If MLX4_PMD_SGE_WR_N is 1, free mbuf immediately. */
+ if (elemof(*sges) == 1) {
+ do {
+ struct rte_mbuf *next = NEXT(buf);
+
+ rte_pktmbuf_free_seg(buf);
+ buf = next;
+ } while (buf != NULL);
+ elt->buf = NULL;
+ }
+ /* Update SGE. */
+ sge->addr = (uintptr_t)&(*linear)[0];
+ sge->length = size;
+ sge->lkey = txq->mr_linear->lkey;
+ sent_size += size;
+ /* Include last segment. */
+ segs++;
+ }
+ return (struct tx_burst_sg_ret){
+ .length = sent_size,
+ .num = segs,
+ };
+stop:
+ return (struct tx_burst_sg_ret){
+ .length = -1,
+ .num = -1,
+ };
+}
+
+#endif /* MLX4_PMD_SGE_WR_N > 1 */
+
/**
* DPDK callback for TX.
*
mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
{
struct txq *txq = (struct txq *)dpdk_txq;
- struct ibv_send_wr head;
- struct ibv_send_wr **wr_next = &head.next;
unsigned int elts_head = txq->elts_head;
- const unsigned int elts_tail = txq->elts_tail;
const unsigned int elts_n = txq->elts_n;
unsigned int elts_comp_cd = txq->elts_comp_cd;
unsigned int elts_comp = 0;
assert(elts_comp_cd != 0);
txq_complete(txq);
- max = (elts_n - (elts_head - elts_tail));
+ max = (elts_n - (elts_head - txq->elts_tail));
if (max > elts_n)
max -= elts_n;
assert(max >= 1);
max = pkts_n;
for (i = 0; (i != max); ++i) {
struct rte_mbuf *buf = pkts[i];
+ unsigned int elts_head_next =
+ (((elts_head + 1) == elts_n) ? 0 : elts_head + 1);
+ struct txq_elt *elt_next = &(*txq->elts)[elts_head_next];
struct txq_elt *elt = &(*txq->elts)[elts_head];
- struct ibv_send_wr *wr = &elt->wr;
unsigned int segs = NB_SEGS(buf);
#ifdef MLX4_PMD_SOFT_COUNTERS
unsigned int sent_size = 0;
#endif
- unsigned int j;
- int linearize = 0;
uint32_t send_flags = 0;
/* Clean up old buffer. */
- if (likely(WR_ID(wr->wr_id).offset != 0)) {
- struct rte_mbuf *tmp = (void *)
- ((uintptr_t)elt->sges[0].addr -
- WR_ID(wr->wr_id).offset);
+ if (likely(elt->buf != NULL)) {
+ struct rte_mbuf *tmp = elt->buf;
/* Faster than rte_pktmbuf_free(). */
do {
tmp = next;
} while (tmp != NULL);
}
-#ifndef NDEBUG
- /* For assert(). */
- WR_ID(wr->wr_id).offset = 0;
- for (j = 0; ((int)j < wr->num_sge); ++j) {
- elt->sges[j].addr = 0;
- elt->sges[j].length = 0;
- elt->sges[j].lkey = 0;
+ /* Request TX completion. */
+ if (unlikely(--elts_comp_cd == 0)) {
+ elts_comp_cd = txq->elts_comp_cd_init;
+ ++elts_comp;
+ send_flags |= IBV_EXP_QP_BURST_SIGNALED;
}
- wr->next = NULL;
- wr->num_sge = 0;
-#endif
- /* Sanity checks, most of which are only relevant with
- * debugging enabled. */
- assert(WR_ID(wr->wr_id).id == elts_head);
- assert(WR_ID(wr->wr_id).offset == 0);
- assert(wr->next == NULL);
- assert(wr->sg_list == &elt->sges[0]);
- assert(wr->num_sge == 0);
- assert(wr->opcode == IBV_WR_SEND);
- /* When there are too many segments, extra segments are
- * linearized in the last SGE. */
- if (unlikely(segs > elemof(elt->sges))) {
- segs = (elemof(elt->sges) - 1);
- linearize = 1;
+ /* Should we enable HW CKSUM offload */
+ if (buf->ol_flags &
+ (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+ send_flags |= IBV_EXP_QP_BURST_IP_CSUM;
+ /* HW does not support checksum offloads at arbitrary
+ * offsets but automatically recognizes the packet
+ * type. For inner L3/L4 checksums, only VXLAN (UDP)
+ * tunnels are currently supported. */
+ if (RTE_ETH_IS_TUNNEL_PKT(buf->packet_type))
+ send_flags |= IBV_EXP_QP_BURST_TUNNEL;
}
- /* Set WR fields. */
- assert((rte_pktmbuf_mtod(buf, uintptr_t) -
- (uintptr_t)buf) <= 0xffff);
- WR_ID(wr->wr_id).offset =
- (rte_pktmbuf_mtod(buf, uintptr_t) -
- (uintptr_t)buf);
- wr->num_sge = segs;
- /* Register segments as SGEs. */
- for (j = 0; (j != segs); ++j) {
- struct ibv_sge *sge = &elt->sges[j];
+ if (likely(segs == 1)) {
+ uintptr_t addr;
+ uint32_t length;
uint32_t lkey;
+ /* Retrieve buffer information. */
+ addr = rte_pktmbuf_mtod(buf, uintptr_t);
+ length = DATA_LEN(buf);
/* Retrieve Memory Region key for this memory pool. */
- lkey = txq_mp2mr(txq, buf->pool);
+ lkey = txq_mp2mr(txq, txq_mb2mp(buf));
if (unlikely(lkey == (uint32_t)-1)) {
/* MR does not exist. */
DEBUG("%p: unable to get MP <-> MR"
" association", (void *)txq);
/* Clean up TX element. */
- WR_ID(elt->wr.wr_id).offset = 0;
-#ifndef NDEBUG
- /* For assert(). */
- while (j) {
- --j;
- --sge;
- sge->addr = 0;
- sge->length = 0;
- sge->lkey = 0;
- }
- wr->num_sge = 0;
-#endif
+ elt->buf = NULL;
goto stop;
}
- /* Sanity checks, only relevant with debugging
- * enabled. */
- assert(sge->addr == 0);
- assert(sge->length == 0);
- assert(sge->lkey == 0);
- /* Update SGE. */
- sge->addr = rte_pktmbuf_mtod(buf, uintptr_t);
+ /* Update element. */
+ elt->buf = buf;
if (txq->priv->vf)
rte_prefetch0((volatile void *)
- (uintptr_t)sge->addr);
- sge->length = DATA_LEN(buf);
- sge->lkey = lkey;
-#ifdef MLX4_PMD_SOFT_COUNTERS
- sent_size += sge->length;
-#endif
- buf = NEXT(buf);
- }
- /* If buf is not NULL here and is not going to be linearized,
- * nb_segs is not valid. */
- assert(j == segs);
- assert((buf == NULL) || (linearize));
- /* Linearize extra segments. */
- if (linearize) {
- struct ibv_sge *sge = &elt->sges[segs];
- linear_t *linear = &(*txq->elts_linear)[elts_head];
- unsigned int size = linearize_mbuf(linear, buf);
-
- assert(segs == (elemof(elt->sges) - 1));
- if (size == 0) {
- /* Invalid packet. */
- DEBUG("%p: packet too large to be linearized.",
- (void *)txq);
- /* Clean up TX element. */
- WR_ID(elt->wr.wr_id).offset = 0;
-#ifndef NDEBUG
- /* For assert(). */
- while (j) {
- --j;
- --sge;
- sge->addr = 0;
- sge->length = 0;
- sge->lkey = 0;
- }
- wr->num_sge = 0;
+ (uintptr_t)addr);
+ RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
+ /* Put packet into send queue. */
+#if MLX4_PMD_MAX_INLINE > 0
+ if (length <= txq->max_inline)
+ err = txq->if_qp->send_pending_inline
+ (txq->qp,
+ (void *)addr,
+ length,
+ send_flags);
+ else
#endif
+ err = txq->if_qp->send_pending
+ (txq->qp,
+ addr,
+ length,
+ lkey,
+ send_flags);
+ if (unlikely(err))
goto stop;
- }
- /* If MLX4_PMD_SGE_WR_N is 1, free mbuf immediately
- * and clear offset from WR ID. */
- if (elemof(elt->sges) == 1) {
- do {
- struct rte_mbuf *next = NEXT(buf);
+#ifdef MLX4_PMD_SOFT_COUNTERS
+ sent_size += length;
+#endif
+ } else {
+#if MLX4_PMD_SGE_WR_N > 1
+ struct ibv_sge sges[MLX4_PMD_SGE_WR_N];
+ struct tx_burst_sg_ret ret;
- rte_pktmbuf_free_seg(buf);
- buf = next;
- } while (buf != NULL);
- WR_ID(wr->wr_id).offset = 0;
- }
- /* Set WR fields and fill SGE with linear buffer. */
- ++wr->num_sge;
- /* Sanity checks, only relevant with debugging
- * enabled. */
- assert(sge->addr == 0);
- assert(sge->length == 0);
- assert(sge->lkey == 0);
- /* Update SGE. */
- sge->addr = (uintptr_t)&(*linear)[0];
- sge->length = size;
- sge->lkey = txq->mr_linear->lkey;
+ ret = tx_burst_sg(txq, segs, elt, buf, elts_head,
+ &sges);
+ if (ret.length == (unsigned int)-1)
+ goto stop;
+ RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
+ /* Put SG list into send queue. */
+ err = txq->if_qp->send_pending_sg_list
+ (txq->qp,
+ sges,
+ ret.num,
+ send_flags);
+ if (unlikely(err))
+ goto stop;
#ifdef MLX4_PMD_SOFT_COUNTERS
- sent_size += size;
+ sent_size += ret.length;
#endif
+#else /* MLX4_PMD_SGE_WR_N > 1 */
+ DEBUG("%p: TX scattered buffers support not"
+ " compiled in", (void *)txq);
+ goto stop;
+#endif /* MLX4_PMD_SGE_WR_N > 1 */
}
- /* Link WRs together for ibv_post_send(). */
- *wr_next = wr;
- wr_next = &wr->next;
- assert(wr->send_flags == 0);
- /* Request TX completion. */
- if (unlikely(--elts_comp_cd == 0)) {
- elts_comp_cd = txq->elts_comp_cd_init;
- ++elts_comp;
- send_flags |= IBV_EXP_QP_BURST_SIGNALED;
- }
- if (++elts_head >= elts_n)
- elts_head = 0;
+ elts_head = elts_head_next;
#ifdef MLX4_PMD_SOFT_COUNTERS
/* Increment sent bytes counter. */
txq->stats.obytes += sent_size;
#endif
- /* Put SG list into send queue and ask for completion event. */
-#if MLX4_PMD_MAX_INLINE > 0
- if ((segs == 1) &&
- (elt->sges[0].length <= txq->max_inline))
- err = txq->if_qp->send_pending_inline
- (txq->qp,
- (void *)(uintptr_t)elt->sges[0].addr,
- elt->sges[0].length,
- send_flags);
- else
-#endif
- err = txq->if_qp->send_pending_sg_list
- (txq->qp,
- elt->sges,
- segs,
- send_flags);
- if (unlikely(err))
- goto stop;
}
stop:
/* Take a shortcut if nothing must be sent. */
/* Increment sent packets counter. */
txq->stats.opackets += i;
#endif
- *wr_next = NULL;
/* Ring QP doorbell. */
err = txq->if_qp->send_flush(txq->qp);
if (unlikely(err)) {
return i;
}
+/**
+ * DPDK callback for TX in secondary processes.
+ *
+ * This function configures all queues from primary process information
+ * if necessary before reverting to the normal TX burst callback.
+ *
+ * @param dpdk_txq
+ * Generic pointer to TX queue structure.
+ * @param[in] pkts
+ * Packets to transmit.
+ * @param pkts_n
+ * Number of packets in array.
+ *
+ * @return
+ * Number of packets successfully transmitted (<= pkts_n).
+ */
+static uint16_t
+mlx4_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts,
+ uint16_t pkts_n)
+{
+ struct txq *txq = dpdk_txq;
+ struct priv *priv = mlx4_secondary_data_setup(txq->priv);
+ struct priv *primary_priv;
+ unsigned int index;
+
+ if (priv == NULL)
+ return 0;
+ primary_priv =
+ mlx4_secondary_data[priv->dev->data->port_id].primary_priv;
+ /* Look for queue index in both private structures. */
+ for (index = 0; index != priv->txqs_n; ++index)
+ if (((*primary_priv->txqs)[index] == txq) ||
+ ((*priv->txqs)[index] == txq))
+ break;
+ if (index == priv->txqs_n)
+ return 0;
+ txq = (*priv->txqs)[index];
+ return priv->dev->tx_pkt_burst(txq, pkts, pkts_n);
+}
+
/**
* Configure a TX queue.
*
txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
unsigned int socket, const struct rte_eth_txconf *conf)
{
- struct priv *priv = dev->data->dev_private;
+ struct priv *priv = mlx4_get_priv(dev);
struct txq tmpl = {
.priv = priv,
.socket = socket
};
union {
struct ibv_exp_query_intf_params params;
- struct ibv_qp_init_attr init;
+ struct ibv_exp_qp_init_attr init;
+ struct ibv_exp_res_domain_init_attr rd;
+ struct ibv_exp_cq_init_attr cq;
struct ibv_exp_qp_attr mod;
} attr;
enum ibv_exp_query_intf_status status;
int ret = 0;
(void)conf; /* Thresholds configuration (ignored). */
+ if (priv == NULL)
+ return EINVAL;
if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) {
ERROR("%p: invalid number of TX descriptors (must be a"
" multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N);
}
desc /= MLX4_PMD_SGE_WR_N;
/* MRs will be registered in mp2mr[] later. */
- tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0);
+ attr.rd = (struct ibv_exp_res_domain_init_attr){
+ .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL |
+ IBV_EXP_RES_DOMAIN_MSG_MODEL),
+ .thread_model = IBV_EXP_THREAD_SINGLE,
+ .msg_model = IBV_EXP_MSG_HIGH_BW,
+ };
+ tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd);
+ if (tmpl.rd == NULL) {
+ ret = ENOMEM;
+ ERROR("%p: RD creation failure: %s",
+ (void *)dev, strerror(ret));
+ goto error;
+ }
+ attr.cq = (struct ibv_exp_cq_init_attr){
+ .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
+ .res_domain = tmpl.rd,
+ };
+ tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq);
if (tmpl.cq == NULL) {
ret = ENOMEM;
ERROR("%p: CQ creation failure: %s",
priv->device_attr.max_qp_wr);
DEBUG("priv->device_attr.max_sge is %d",
priv->device_attr.max_sge);
- attr.init = (struct ibv_qp_init_attr){
+ attr.init = (struct ibv_exp_qp_init_attr){
/* CQ to be associated with the send queue. */
.send_cq = tmpl.cq,
/* CQ to be associated with the receive queue. */
.qp_type = IBV_QPT_RAW_PACKET,
/* Do *NOT* enable this, completions events are managed per
* TX burst. */
- .sq_sig_all = 0
+ .sq_sig_all = 0,
+ .pd = priv->pd,
+ .res_domain = tmpl.rd,
+ .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
+ IBV_EXP_QP_INIT_ATTR_RES_DOMAIN),
};
- tmpl.qp = ibv_create_qp(priv->pd, &attr.init);
+ tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init);
if (tmpl.qp == NULL) {
ret = (errno ? errno : EINVAL);
ERROR("%p: QP creation failure: %s",
.intf_scope = IBV_EXP_INTF_GLOBAL,
.intf = IBV_EXP_INTF_QP_BURST,
.obj = tmpl.qp,
+#ifdef HAVE_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK
+ /* MC loopback must be disabled when not using a VF. */
+ .family_flags =
+ (!priv->vf ?
+ IBV_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK :
+ 0),
+#endif
};
tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
if (tmpl.if_qp == NULL) {
txq_cleanup(txq);
*txq = tmpl;
DEBUG("%p: txq updated with %p", (void *)txq, (void *)&tmpl);
+ /* Pre-register known mempools. */
+ rte_mempool_walk(txq_mp2mr_iter, txq);
assert(ret == 0);
return 0;
error:
struct txq *txq = (*priv->txqs)[idx];
int ret;
+ if (mlx4_is_secondary())
+ return -E_RTE_SECONDARY;
priv_lock(priv);
DEBUG("%p: configuring queue %u for %u descriptors",
(void *)dev, idx, desc);
struct priv *priv;
unsigned int i;
+ if (mlx4_is_secondary())
+ return;
if (txq == NULL)
return;
priv = txq->priv;
}
if (rxq->cq != NULL)
claim_zero(ibv_destroy_cq(rxq->cq));
+ if (rxq->rd != NULL) {
+ struct ibv_exp_destroy_res_domain_attr attr = {
+ .comp_mask = 0,
+ };
+
+ assert(rxq->priv != NULL);
+ assert(rxq->priv->ctx != NULL);
+ claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx,
+ rxq->rd,
+ &attr));
+ }
if (rxq->mr != NULL)
claim_zero(ibv_dereg_mr(rxq->mr));
memset(rxq, 0, sizeof(*rxq));
}
+/**
+ * Translate RX completion flags to packet type.
+ *
+ * @param flags
+ * RX completion flags returned by poll_length_flags().
+ *
+ * @return
+ * Packet type for struct rte_mbuf.
+ */
+static inline uint32_t
+rxq_cq_to_pkt_type(uint32_t flags)
+{
+ uint32_t pkt_type;
+
+ if (flags & IBV_EXP_CQ_RX_TUNNEL_PACKET)
+ pkt_type =
+ TRANSPOSE(flags,
+ IBV_EXP_CQ_RX_OUTER_IPV4_PACKET, RTE_PTYPE_L3_IPV4) |
+ TRANSPOSE(flags,
+ IBV_EXP_CQ_RX_OUTER_IPV6_PACKET, RTE_PTYPE_L3_IPV6) |
+ TRANSPOSE(flags,
+ IBV_EXP_CQ_RX_IPV4_PACKET, RTE_PTYPE_INNER_L3_IPV4) |
+ TRANSPOSE(flags,
+ IBV_EXP_CQ_RX_IPV6_PACKET, RTE_PTYPE_INNER_L3_IPV6);
+ else
+ pkt_type =
+ TRANSPOSE(flags,
+ IBV_EXP_CQ_RX_IPV4_PACKET, RTE_PTYPE_L3_IPV4) |
+ TRANSPOSE(flags,
+ IBV_EXP_CQ_RX_IPV6_PACKET, RTE_PTYPE_L3_IPV6);
+ return pkt_type;
+}
+
+/**
+ * Translate RX completion flags to offload flags.
+ *
+ * @param[in] rxq
+ * Pointer to RX queue structure.
+ * @param flags
+ * RX completion flags returned by poll_length_flags().
+ *
+ * @return
+ * Offload flags (ol_flags) for struct rte_mbuf.
+ */
+static inline uint32_t
+rxq_cq_to_ol_flags(const struct rxq *rxq, uint32_t flags)
+{
+ uint32_t ol_flags = 0;
+
+ if (rxq->csum)
+ ol_flags |=
+ TRANSPOSE(~flags,
+ IBV_EXP_CQ_RX_IP_CSUM_OK,
+ PKT_RX_IP_CKSUM_BAD) |
+ TRANSPOSE(~flags,
+ IBV_EXP_CQ_RX_TCP_UDP_CSUM_OK,
+ PKT_RX_L4_CKSUM_BAD);
+ /*
+ * PKT_RX_IP_CKSUM_BAD and PKT_RX_L4_CKSUM_BAD are used in place
+ * of PKT_RX_EIP_CKSUM_BAD because the latter is not functional
+ * (its value is 0).
+ */
+ if ((flags & IBV_EXP_CQ_RX_TUNNEL_PACKET) && (rxq->csum_l2tun))
+ ol_flags |=
+ TRANSPOSE(~flags,
+ IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK,
+ PKT_RX_IP_CKSUM_BAD) |
+ TRANSPOSE(~flags,
+ IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK,
+ PKT_RX_L4_CKSUM_BAD);
+ return ol_flags;
+}
+
static uint16_t
mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n);
struct rte_mbuf **pkt_buf_next = &pkt_buf;
unsigned int seg_headroom = RTE_PKTMBUF_HEADROOM;
unsigned int j = 0;
+ uint32_t flags;
/* Sanity checks. */
#ifdef NDEBUG
assert(wr->num_sge == elemof(elt->sges));
assert(elts_head < rxq->elts_n);
assert(rxq->elts_head < rxq->elts_n);
- ret = rxq->if_cq->poll_length(rxq->cq, NULL, NULL);
+ ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL,
+ &flags);
if (unlikely(ret < 0)) {
struct ibv_wc wc;
int wcs_n;
NB_SEGS(pkt_buf) = j;
PORT(pkt_buf) = rxq->port_id;
PKT_LEN(pkt_buf) = pkt_buf_len;
- pkt_buf->ol_flags = 0;
+ pkt_buf->packet_type = rxq_cq_to_pkt_type(flags);
+ pkt_buf->ol_flags = rxq_cq_to_ol_flags(rxq, flags);
/* Return packet. */
*(pkts++) = pkt_buf;
struct rte_mbuf *seg = (void *)((uintptr_t)elt->sge.addr -
WR_ID(wr_id).offset);
struct rte_mbuf *rep;
+ uint32_t flags;
/* Sanity checks. */
assert(WR_ID(wr_id).id < rxq->elts_n);
assert(wr->num_sge == 1);
assert(elts_head < rxq->elts_n);
assert(rxq->elts_head < rxq->elts_n);
- ret = rxq->if_cq->poll_length(rxq->cq, NULL, NULL);
+ /*
+ * Fetch initial bytes of packet descriptor into a
+ * cacheline while allocating rep.
+ */
+ rte_prefetch0(seg);
+ rte_prefetch0(&seg->cacheline1);
+ ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL,
+ &flags);
if (unlikely(ret < 0)) {
struct ibv_wc wc;
int wcs_n;
if (ret == 0)
break;
len = ret;
- /*
- * Fetch initial bytes of packet descriptor into a
- * cacheline while allocating rep.
- */
- rte_prefetch0(seg);
rep = __rte_mbuf_raw_alloc(rxq->mp);
if (unlikely(rep == NULL)) {
/*
NEXT(seg) = NULL;
PKT_LEN(seg) = len;
DATA_LEN(seg) = len;
- seg->ol_flags = 0;
+ seg->packet_type = rxq_cq_to_pkt_type(flags);
+ seg->ol_flags = rxq_cq_to_ol_flags(rxq, flags);
/* Return packet. */
*(pkts++) = seg;
return pkts_ret;
}
+/**
+ * DPDK callback for RX in secondary processes.
+ *
+ * This function configures all queues from primary process information
+ * if necessary before reverting to the normal RX burst callback.
+ *
+ * @param dpdk_rxq
+ * Generic pointer to RX queue structure.
+ * @param[out] pkts
+ * Array to store received packets.
+ * @param pkts_n
+ * Maximum number of packets in array.
+ *
+ * @return
+ * Number of packets successfully received (<= pkts_n).
+ */
+static uint16_t
+mlx4_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts,
+ uint16_t pkts_n)
+{
+ struct rxq *rxq = dpdk_rxq;
+ struct priv *priv = mlx4_secondary_data_setup(rxq->priv);
+ struct priv *primary_priv;
+ unsigned int index;
+
+ if (priv == NULL)
+ return 0;
+ primary_priv =
+ mlx4_secondary_data[priv->dev->data->port_id].primary_priv;
+ /* Look for queue index in both private structures. */
+ for (index = 0; index != priv->rxqs_n; ++index)
+ if (((*primary_priv->rxqs)[index] == rxq) ||
+ ((*priv->rxqs)[index] == rxq))
+ break;
+ if (index == priv->rxqs_n)
+ return 0;
+ rxq = (*priv->rxqs)[index];
+ return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n);
+}
+
/**
* Allocate a Queue Pair.
* Optionally setup inline receive if supported.
* QP pointer or NULL in case of error.
*/
static struct ibv_qp *
-rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc)
+rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
+ struct ibv_exp_res_domain *rd)
{
struct ibv_exp_qp_init_attr attr = {
/* CQ to be associated with the send queue. */
MLX4_PMD_SGE_WR_N),
},
.qp_type = IBV_QPT_RAW_PACKET,
- .comp_mask = IBV_EXP_QP_INIT_ATTR_PD,
+ .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
+ IBV_EXP_QP_INIT_ATTR_RES_DOMAIN),
.pd = priv->pd,
+ .res_domain = rd,
};
#ifdef INLINE_RECV
*/
static struct ibv_qp *
rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
- int parent)
+ int parent, struct ibv_exp_res_domain *rd)
{
struct ibv_exp_qp_init_attr attr = {
/* CQ to be associated with the send queue. */
},
.qp_type = IBV_QPT_RAW_PACKET,
.comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
+ IBV_EXP_QP_INIT_ATTR_RES_DOMAIN |
IBV_EXP_QP_INIT_ATTR_QPG),
- .pd = priv->pd
+ .pd = priv->pd,
+ .res_domain = rd,
};
#ifdef INLINE_RECV
/* Number of descriptors and mbufs currently allocated. */
desc_n = (tmpl.elts_n * (tmpl.sp ? MLX4_PMD_SGE_WR_N : 1));
mbuf_n = desc_n;
+ /* Toggle RX checksum offload if hardware supports it. */
+ if (priv->hw_csum) {
+ tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+ rxq->csum = tmpl.csum;
+ }
+ if (priv->hw_csum_l2tun) {
+ tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+ rxq->csum_l2tun = tmpl.csum_l2tun;
+ }
/* Enable scattered packets support for this queue if necessary. */
if ((dev->data->dev_conf.rxmode.jumbo_frame) &&
(dev->data->dev_conf.rxmode.max_rx_pkt_len >
struct ibv_exp_qp_attr mod;
union {
struct ibv_exp_query_intf_params params;
+ struct ibv_exp_cq_init_attr cq;
+ struct ibv_exp_res_domain_init_attr rd;
} attr;
enum ibv_exp_query_intf_status status;
struct ibv_recv_wr *bad_wr;
rte_pktmbuf_tailroom(buf)) == tmpl.mb_len);
assert(rte_pktmbuf_headroom(buf) == RTE_PKTMBUF_HEADROOM);
rte_pktmbuf_free(buf);
+ /* Toggle RX checksum offload if hardware supports it. */
+ if (priv->hw_csum)
+ tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+ if (priv->hw_csum_l2tun)
+ tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
/* Enable scattered packets support for this queue if necessary. */
if ((dev->data->dev_conf.rxmode.jumbo_frame) &&
(dev->data->dev_conf.rxmode.max_rx_pkt_len >
goto error;
}
skip_mr:
- tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0);
+ attr.rd = (struct ibv_exp_res_domain_init_attr){
+ .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL |
+ IBV_EXP_RES_DOMAIN_MSG_MODEL),
+ .thread_model = IBV_EXP_THREAD_SINGLE,
+ .msg_model = IBV_EXP_MSG_HIGH_BW,
+ };
+ tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd);
+ if (tmpl.rd == NULL) {
+ ret = ENOMEM;
+ ERROR("%p: RD creation failure: %s",
+ (void *)dev, strerror(ret));
+ goto error;
+ }
+ attr.cq = (struct ibv_exp_cq_init_attr){
+ .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
+ .res_domain = tmpl.rd,
+ };
+ tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq);
if (tmpl.cq == NULL) {
ret = ENOMEM;
ERROR("%p: CQ creation failure: %s",
priv->device_attr.max_sge);
#ifdef RSS_SUPPORT
if (priv->rss)
- tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent);
+ tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent,
+ tmpl.rd);
else
#endif /* RSS_SUPPORT */
- tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc);
+ tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd);
if (tmpl.qp == NULL) {
ret = (errno ? errno : EINVAL);
ERROR("%p: QP creation failure: %s",
struct rxq *rxq = (*priv->rxqs)[idx];
int ret;
+ if (mlx4_is_secondary())
+ return -E_RTE_SECONDARY;
priv_lock(priv);
DEBUG("%p: configuring queue %u for %u descriptors",
(void *)dev, idx, desc);
struct priv *priv;
unsigned int i;
+ if (mlx4_is_secondary())
+ return;
if (rxq == NULL)
return;
priv = rxq->priv;
priv_unlock(priv);
}
+static void
+priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *);
+
/**
* DPDK callback to start the device.
*
unsigned int r;
struct rxq *rxq;
+ if (mlx4_is_secondary())
+ return -E_RTE_SECONDARY;
priv_lock(priv);
if (priv->started) {
priv_unlock(priv);
}
}
priv->started = 0;
+ priv_unlock(priv);
return -ret;
} while ((--r) && ((rxq = (*priv->rxqs)[++i]), i));
+ priv_dev_interrupt_handler_install(priv, dev);
priv_unlock(priv);
return 0;
}
unsigned int r;
struct rxq *rxq;
+ if (mlx4_is_secondary())
+ return;
priv_lock(priv);
if (!priv->started) {
priv_unlock(priv);
return 0;
}
+static void
+priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
+
/**
* DPDK callback to close the device.
*
static void
mlx4_dev_close(struct rte_eth_dev *dev)
{
- struct priv *priv = dev->data->dev_private;
+ struct priv *priv = mlx4_get_priv(dev);
void *tmp;
unsigned int i;
+ if (priv == NULL)
+ return;
priv_lock(priv);
DEBUG("%p: closing device \"%s\"",
(void *)dev,
claim_zero(ibv_close_device(priv->ctx));
} else
assert(priv->ctx == NULL);
+ priv_dev_interrupt_handler_uninstall(priv, dev);
priv_unlock(priv);
memset(priv, 0, sizeof(*priv));
}
static void
mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
{
- struct priv *priv = dev->data->dev_private;
+ struct priv *priv = mlx4_get_priv(dev);
unsigned int max;
+ char ifname[IF_NAMESIZE];
+ if (priv == NULL)
+ return;
priv_lock(priv);
/* FIXME: we should ask the device for these values. */
info->min_rx_bufsize = 32;
max = 65535;
info->max_rx_queues = max;
info->max_tx_queues = max;
- info->max_mac_addrs = elemof(priv->mac);
+ /* Last array entry is reserved for broadcast. */
+ info->max_mac_addrs = (elemof(priv->mac) - 1);
+ info->rx_offload_capa =
+ (priv->hw_csum ?
+ (DEV_RX_OFFLOAD_IPV4_CKSUM |
+ DEV_RX_OFFLOAD_UDP_CKSUM |
+ DEV_RX_OFFLOAD_TCP_CKSUM) :
+ 0);
+ info->tx_offload_capa =
+ (priv->hw_csum ?
+ (DEV_TX_OFFLOAD_IPV4_CKSUM |
+ DEV_TX_OFFLOAD_UDP_CKSUM |
+ DEV_TX_OFFLOAD_TCP_CKSUM) :
+ 0);
+ if (priv_get_ifname(priv, &ifname) == 0)
+ info->if_index = if_nametoindex(ifname);
priv_unlock(priv);
}
static void
mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
{
- struct priv *priv = dev->data->dev_private;
+ struct priv *priv = mlx4_get_priv(dev);
struct rte_eth_stats tmp = {0};
unsigned int i;
unsigned int idx;
+ if (priv == NULL)
+ return;
priv_lock(priv);
/* Add software counters. */
for (i = 0; (i != priv->rxqs_n); ++i) {
static void
mlx4_stats_reset(struct rte_eth_dev *dev)
{
- struct priv *priv = dev->data->dev_private;
+ struct priv *priv = mlx4_get_priv(dev);
unsigned int i;
unsigned int idx;
+ if (priv == NULL)
+ return;
priv_lock(priv);
for (i = 0; (i != priv->rxqs_n); ++i) {
if ((*priv->rxqs)[i] == NULL)
for (i = 0; (i != priv->txqs_n); ++i) {
if ((*priv->txqs)[i] == NULL)
continue;
- idx = (*priv->rxqs)[i]->stats.idx;
+ idx = (*priv->txqs)[i]->stats.idx;
(*priv->txqs)[i]->stats =
(struct mlx4_txq_stats){ .idx = idx };
}
{
struct priv *priv = dev->data->dev_private;
+ if (mlx4_is_secondary())
+ return;
priv_lock(priv);
DEBUG("%p: removing MAC address from index %" PRIu32,
(void *)dev, index);
- if (index >= MLX4_MAX_MAC_ADDRESSES)
- goto end;
- /* Refuse to remove the broadcast address, this one is special. */
- if (!memcmp(priv->mac[index].addr_bytes, "\xff\xff\xff\xff\xff\xff",
- ETHER_ADDR_LEN))
+ /* Last array entry is reserved for broadcast. */
+ if (index >= (elemof(priv->mac) - 1))
goto end;
priv_mac_addr_del(priv, index);
end:
{
struct priv *priv = dev->data->dev_private;
+ if (mlx4_is_secondary())
+ return;
(void)vmdq;
priv_lock(priv);
DEBUG("%p: adding MAC address at index %" PRIu32,
(void *)dev, index);
- if (index >= MLX4_MAX_MAC_ADDRESSES)
- goto end;
- /* Refuse to add the broadcast address, this one is special. */
- if (!memcmp(mac_addr->addr_bytes, "\xff\xff\xff\xff\xff\xff",
- ETHER_ADDR_LEN))
+ /* Last array entry is reserved for broadcast. */
+ if (index >= (elemof(priv->mac) - 1))
goto end;
priv_mac_addr_add(priv, index,
(const uint8_t (*)[ETHER_ADDR_LEN])
priv_unlock(priv);
}
+/**
+ * DPDK callback to set the primary MAC address.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param mac_addr
+ * MAC address to register.
+ */
+static void
+mlx4_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
+{
+ DEBUG("%p: setting primary MAC address", (void *)dev);
+ mlx4_mac_addr_remove(dev, 0);
+ mlx4_mac_addr_add(dev, mac_addr, 0, 0);
+}
+
/**
* DPDK callback to enable promiscuous mode.
*
unsigned int i;
int ret;
+ if (mlx4_is_secondary())
+ return;
priv_lock(priv);
if (priv->promisc) {
priv_unlock(priv);
struct priv *priv = dev->data->dev_private;
unsigned int i;
+ if (mlx4_is_secondary())
+ return;
priv_lock(priv);
if (!priv->promisc) {
priv_unlock(priv);
unsigned int i;
int ret;
+ if (mlx4_is_secondary())
+ return;
priv_lock(priv);
if (priv->allmulti) {
priv_unlock(priv);
struct priv *priv = dev->data->dev_private;
unsigned int i;
+ if (mlx4_is_secondary())
+ return;
priv_lock(priv);
if (!priv->allmulti) {
priv_unlock(priv);
static int
mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete)
{
- struct priv *priv = dev->data->dev_private;
+ struct priv *priv = mlx4_get_priv(dev);
struct ethtool_cmd edata = {
.cmd = ETHTOOL_GSET
};
struct rte_eth_link dev_link;
int link_speed = 0;
+ if (priv == NULL)
+ return -EINVAL;
(void)wait_to_complete;
if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) {
WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno));
static int
mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete)
{
- struct priv *priv = dev->data->dev_private;
+ struct priv *priv = mlx4_get_priv(dev);
int ret;
+ if (priv == NULL)
+ return -EINVAL;
priv_lock(priv);
ret = mlx4_link_update_unlocked(dev, wait_to_complete);
priv_unlock(priv);
uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) =
mlx4_rx_burst;
+ if (mlx4_is_secondary())
+ return -E_RTE_SECONDARY;
priv_lock(priv);
/* Set kernel interface MTU first. */
if (priv_set_mtu(priv, mtu)) {
};
int ret;
+ if (mlx4_is_secondary())
+ return -E_RTE_SECONDARY;
ifr.ifr_data = ðpause;
priv_lock(priv);
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
};
int ret;
+ if (mlx4_is_secondary())
+ return -E_RTE_SECONDARY;
ifr.ifr_data = ðpause;
ethpause.autoneg = fc_conf->autoneg;
if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
struct priv *priv = dev->data->dev_private;
int ret;
+ if (mlx4_is_secondary())
+ return -E_RTE_SECONDARY;
priv_lock(priv);
ret = vlan_filter_set(dev, vlan_id, on);
priv_unlock(priv);
.priority_flow_ctrl_set = NULL,
.mac_addr_remove = mlx4_mac_addr_remove,
.mac_addr_add = mlx4_mac_addr_add,
+ .mac_addr_set = mlx4_mac_addr_set,
.mtu_set = mlx4_dev_set_mtu,
- .fdir_add_signature_filter = NULL,
- .fdir_update_signature_filter = NULL,
- .fdir_remove_signature_filter = NULL,
- .fdir_add_perfect_filter = NULL,
- .fdir_update_perfect_filter = NULL,
- .fdir_remove_perfect_filter = NULL,
- .fdir_set_masks = NULL
};
/**
return atoi(val);
}
+static void
+mlx4_dev_link_status_handler(void *);
+static void
+mlx4_dev_interrupt_handler(struct rte_intr_handle *, void *);
+
+/**
+ * Link status handler.
+ *
+ * @param priv
+ * Pointer to private structure.
+ * @param dev
+ * Pointer to the rte_eth_dev structure.
+ *
+ * @return
+ * Nonzero if the callback process can be called immediately.
+ */
+static int
+priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev)
+{
+ struct ibv_async_event event;
+ int port_change = 0;
+ int ret = 0;
+
+ /* Read all message and acknowledge them. */
+ for (;;) {
+ if (ibv_get_async_event(priv->ctx, &event))
+ break;
+
+ if (event.event_type == IBV_EVENT_PORT_ACTIVE ||
+ event.event_type == IBV_EVENT_PORT_ERR)
+ port_change = 1;
+ else
+ DEBUG("event type %d on port %d not handled",
+ event.event_type, event.element.port_num);
+ ibv_ack_async_event(&event);
+ }
+
+ if (port_change ^ priv->pending_alarm) {
+ struct rte_eth_link *link = &dev->data->dev_link;
+
+ priv->pending_alarm = 0;
+ mlx4_link_update_unlocked(dev, 0);
+ if (((link->link_speed == 0) && link->link_status) ||
+ ((link->link_speed != 0) && !link->link_status)) {
+ /* Inconsistent status, check again later. */
+ priv->pending_alarm = 1;
+ rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US,
+ mlx4_dev_link_status_handler,
+ dev);
+ } else
+ ret = 1;
+ }
+ return ret;
+}
+
+/**
+ * Handle delayed link status event.
+ *
+ * @param arg
+ * Registered argument.
+ */
+static void
+mlx4_dev_link_status_handler(void *arg)
+{
+ struct rte_eth_dev *dev = arg;
+ struct priv *priv = dev->data->dev_private;
+ int ret;
+
+ priv_lock(priv);
+ assert(priv->pending_alarm == 1);
+ ret = priv_dev_link_status_handler(priv, dev);
+ priv_unlock(priv);
+ if (ret)
+ _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Handle interrupts from the NIC.
+ *
+ * @param[in] intr_handle
+ * Interrupt handler.
+ * @param cb_arg
+ * Callback argument.
+ */
+static void
+mlx4_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg)
+{
+ struct rte_eth_dev *dev = cb_arg;
+ struct priv *priv = dev->data->dev_private;
+ int ret;
+
+ (void)intr_handle;
+ priv_lock(priv);
+ ret = priv_dev_link_status_handler(priv, dev);
+ priv_unlock(priv);
+ if (ret)
+ _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Uninstall interrupt handler.
+ *
+ * @param priv
+ * Pointer to private structure.
+ * @param dev
+ * Pointer to the rte_eth_dev structure.
+ */
+static void
+priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
+{
+ if (!dev->data->dev_conf.intr_conf.lsc)
+ return;
+ rte_intr_callback_unregister(&priv->intr_handle,
+ mlx4_dev_interrupt_handler,
+ dev);
+ if (priv->pending_alarm)
+ rte_eal_alarm_cancel(mlx4_dev_link_status_handler, dev);
+ priv->pending_alarm = 0;
+ priv->intr_handle.fd = 0;
+ priv->intr_handle.type = 0;
+}
+
+/**
+ * Install interrupt handler.
+ *
+ * @param priv
+ * Pointer to private structure.
+ * @param dev
+ * Pointer to the rte_eth_dev structure.
+ */
+static void
+priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
+{
+ int rc, flags;
+
+ if (!dev->data->dev_conf.intr_conf.lsc)
+ return;
+ assert(priv->ctx->async_fd > 0);
+ flags = fcntl(priv->ctx->async_fd, F_GETFL);
+ rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+ if (rc < 0) {
+ INFO("failed to change file descriptor async event queue");
+ dev->data->dev_conf.intr_conf.lsc = 0;
+ } else {
+ priv->intr_handle.fd = priv->ctx->async_fd;
+ priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
+ rte_intr_callback_register(&priv->intr_handle,
+ mlx4_dev_interrupt_handler,
+ dev);
+ }
+}
+
static struct eth_driver mlx4_driver;
/**
struct ibv_port_attr port_attr;
struct ibv_pd *pd = NULL;
struct priv *priv = NULL;
- struct rte_eth_dev *eth_dev;
+ struct rte_eth_dev *eth_dev = NULL;
#ifdef HAVE_EXP_QUERY_DEVICE
struct ibv_exp_device_attr exp_device_attr;
#endif /* HAVE_EXP_QUERY_DEVICE */
goto port_error;
}
if (port_attr.state != IBV_PORT_ACTIVE)
- WARN("bad state for port %d: \"%s\" (%d)",
- port, ibv_port_state_str(port_attr.state),
- port_attr.state);
+ DEBUG("port %d is not active: \"%s\" (%d)",
+ port, ibv_port_state_str(port_attr.state),
+ port_attr.state);
/* Allocate protection domain. */
pd = ibv_alloc_pd(ctx);
exp_device_attr.max_rss_tbl_sz);
#endif /* RSS_SUPPORT */
+ priv->hw_csum =
+ ((exp_device_attr.exp_device_cap_flags &
+ IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) &&
+ (exp_device_attr.exp_device_cap_flags &
+ IBV_EXP_DEVICE_RX_CSUM_IP_PKT));
+ DEBUG("checksum offloading is %ssupported",
+ (priv->hw_csum ? "" : "not "));
+
+ priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags &
+ IBV_EXP_DEVICE_VXLAN_SUPPORT);
+ DEBUG("L2 tunnel checksum offloads are %ssupported",
+ (priv->hw_csum_l2tun ? "" : "not "));
+
#ifdef INLINE_RECV
priv->inl_recv_size = mlx4_getenv_int("MLX4_INLINE_RECV_SIZE");
claim_zero(priv_mac_addr_add(priv, 0,
(const uint8_t (*)[ETHER_ADDR_LEN])
mac.addr_bytes));
- claim_zero(priv_mac_addr_add(priv, 1,
+ claim_zero(priv_mac_addr_add(priv, (elemof(priv->mac) - 1),
&(const uint8_t [ETHER_ADDR_LEN])
{ "\xff\xff\xff\xff\xff\xff" }));
#ifndef NDEBUG
goto port_error;
}
- eth_dev->data->dev_private = priv;
+ /* Secondary processes have to use local storage for their
+ * private data as well as a copy of eth_dev->data, but this
+ * pointer must not be modified before burst functions are
+ * actually called. */
+ if (mlx4_is_secondary()) {
+ struct mlx4_secondary_data *sd =
+ &mlx4_secondary_data[eth_dev->data->port_id];
+
+ sd->primary_priv = eth_dev->data->dev_private;
+ if (sd->primary_priv == NULL) {
+ ERROR("no private data for port %u",
+ eth_dev->data->port_id);
+ err = EINVAL;
+ goto port_error;
+ }
+ sd->shared_dev_data = eth_dev->data;
+ rte_spinlock_init(&sd->lock);
+ memcpy(sd->data.name, sd->shared_dev_data->name,
+ sizeof(sd->data.name));
+ sd->data.dev_private = priv;
+ sd->data.rx_mbuf_alloc_failed = 0;
+ sd->data.mtu = ETHER_MTU;
+ sd->data.port_id = sd->shared_dev_data->port_id;
+ sd->data.mac_addrs = priv->mac;
+ eth_dev->tx_pkt_burst = mlx4_tx_burst_secondary_setup;
+ eth_dev->rx_pkt_burst = mlx4_rx_burst_secondary_setup;
+ } else {
+ eth_dev->data->dev_private = priv;
+ eth_dev->data->rx_mbuf_alloc_failed = 0;
+ eth_dev->data->mtu = ETHER_MTU;
+ eth_dev->data->mac_addrs = priv->mac;
+ }
eth_dev->pci_dev = pci_dev;
+
+ rte_eth_copy_pci_info(eth_dev, pci_dev);
+
eth_dev->driver = &mlx4_driver;
- eth_dev->data->rx_mbuf_alloc_failed = 0;
- eth_dev->data->mtu = ETHER_MTU;
priv->dev = eth_dev;
eth_dev->dev_ops = &mlx4_dev_ops;
- eth_dev->data->mac_addrs = priv->mac;
+ TAILQ_INIT(ð_dev->link_intr_cbs);
/* Bring Ethernet device up. */
DEBUG("forcing Ethernet interface up");
claim_zero(ibv_dealloc_pd(pd));
if (ctx)
claim_zero(ibv_close_device(ctx));
+ if (eth_dev)
+ rte_eth_dev_release_port(eth_dev);
break;
}
.name = MLX4_DRIVER_NAME,
.id_table = mlx4_pci_id_map,
.devinit = mlx4_pci_devinit,
+ .drv_flags = RTE_PCI_DRV_INTR_LSC,
},
.dev_private_size = sizeof(struct priv)
};
{
(void)name;
(void)args;
+
+ RTE_BUILD_BUG_ON(sizeof(wr_id_t) != sizeof(uint64_t));
/*
* RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
* huge pages. Calling ibv_fork_init() during init allows
git.droids-corp.org / dpdk.git / blobdiff