* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-/* System headers. */
+/**
+ * @file
+ * mlx4 driver initialization.
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <inttypes.h>
#include <stddef.h>
+#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
-#include <stdint.h>
-#include <inttypes.h>
#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <assert.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_ethdev_pci.h>
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_common.h>
#include <rte_dev.h>
-#include <rte_mbuf.h>
#include <rte_errno.h>
-#include <rte_mempool.h>
-#include <rte_malloc.h>
-#include <rte_memory.h>
-#include <rte_kvargs.h>
+#include <rte_ethdev.h>
+#include <rte_ethdev_pci.h>
+#include <rte_ether.h>
#include <rte_interrupts.h>
-#include <rte_common.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
-/* PMD headers. */
#include "mlx4.h"
#include "mlx4_flow.h"
#include "mlx4_rxtx.h"
NULL,
};
-/* Device configuration. */
-
/**
* DPDK callback for Ethernet device configuration.
*
- * Prepare the driver for a given number of TX and RX queues.
- *
* @param dev
* Pointer to Ethernet device structure.
*
mlx4_dev_configure(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
- unsigned int rxqs_n = dev->data->nb_rx_queues;
- unsigned int txqs_n = dev->data->nb_tx_queues;
-
- priv->rxqs = (void *)dev->data->rx_queues;
- priv->txqs = (void *)dev->data->tx_queues;
- if (txqs_n != priv->txqs_n) {
- INFO("%p: TX queues number update: %u -> %u",
- (void *)dev, priv->txqs_n, txqs_n);
- priv->txqs_n = txqs_n;
- }
- if (rxqs_n != priv->rxqs_n) {
- INFO("%p: Rx queues number update: %u -> %u",
- (void *)dev, priv->rxqs_n, rxqs_n);
- priv->rxqs_n = rxqs_n;
- }
- return 0;
-}
-
-struct mlx4_check_mempool_data {
int ret;
- char *start;
- char *end;
-};
-
-/* Called by mlx4_check_mempool() when iterating the memory chunks. */
-static void mlx4_check_mempool_cb(struct rte_mempool *mp,
- void *opaque, struct rte_mempool_memhdr *memhdr,
- unsigned mem_idx)
-{
- struct mlx4_check_mempool_data *data = opaque;
-
- (void)mp;
- (void)mem_idx;
- /* It already failed, skip the next chunks. */
- if (data->ret != 0)
- return;
- /* It is the first chunk. */
- if (data->start == NULL && data->end == NULL) {
- data->start = memhdr->addr;
- data->end = data->start + memhdr->len;
- return;
- }
- if (data->end == memhdr->addr) {
- data->end += memhdr->len;
- return;
- }
- if (data->start == (char *)memhdr->addr + memhdr->len) {
- data->start -= memhdr->len;
- return;
- }
- /* Error, mempool is not virtually contigous. */
- data->ret = -1;
-}
-
-/**
- * Check if a mempool can be used: it must be virtually contiguous.
- *
- * @param[in] mp
- * Pointer to memory pool.
- * @param[out] start
- * Pointer to the start address of the mempool virtual memory area
- * @param[out] end
- * Pointer to the end address of the mempool virtual memory area
- *
- * @return
- * 0 on success (mempool is virtually contiguous), -1 on error.
- */
-static int mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start,
- uintptr_t *end)
-{
- struct mlx4_check_mempool_data data;
- memset(&data, 0, sizeof(data));
- rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data);
- *start = (uintptr_t)data.start;
- *end = (uintptr_t)data.end;
- return data.ret;
-}
-
-/**
- * Register mempool as a memory region.
- *
- * @param pd
- * Pointer to protection domain.
- * @param mp
- * Pointer to memory pool.
- *
- * @return
- * Memory region pointer, NULL in case of error and rte_errno is set.
- */
-struct ibv_mr *
-mlx4_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp)
-{
- const struct rte_memseg *ms = rte_eal_get_physmem_layout();
- uintptr_t start;
- uintptr_t end;
- unsigned int i;
- struct ibv_mr *mr;
-
- if (mlx4_check_mempool(mp, &start, &end) != 0) {
- rte_errno = EINVAL;
- ERROR("mempool %p: not virtually contiguous",
- (void *)mp);
- return NULL;
- }
- DEBUG("mempool %p area start=%p end=%p size=%zu",
- (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- /* Round start and end to page boundary if found in memory segments. */
- for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
- uintptr_t addr = (uintptr_t)ms[i].addr;
- size_t len = ms[i].len;
- unsigned int align = ms[i].hugepage_sz;
-
- if ((start > addr) && (start < addr + len))
- start = RTE_ALIGN_FLOOR(start, align);
- if ((end > addr) && (end < addr + len))
- end = RTE_ALIGN_CEIL(end, align);
- }
- DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
- (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- mr = ibv_reg_mr(pd,
- (void *)start,
- end - start,
- IBV_ACCESS_LOCAL_WRITE);
- if (!mr)
- rte_errno = errno ? errno : EINVAL;
- return mr;
+ /* Prepare internal flow rules. */
+ ret = mlx4_flow_sync(priv);
+ if (ret)
+ ERROR("cannot set up internal flow rules: %s",
+ strerror(-ret));
+ return ret;
}
/**
return 0;
DEBUG("%p: attaching configured flows to all RX queues", (void *)dev);
priv->started = 1;
- ret = mlx4_mac_addr_add(priv);
- if (ret)
- goto err;
ret = mlx4_intr_install(priv);
if (ret) {
ERROR("%p: interrupt handler installation failed",
(void *)dev);
goto err;
}
- ret = mlx4_priv_flow_start(priv);
+ ret = mlx4_flow_start(priv);
if (ret) {
ERROR("%p: flow start failed: %s",
(void *)dev, strerror(ret));
return 0;
err:
/* Rollback. */
- mlx4_mac_addr_del(priv);
priv->started = 0;
return ret;
}
return;
DEBUG("%p: detaching flows from all RX queues", (void *)dev);
priv->started = 0;
- mlx4_priv_flow_stop(priv);
+ mlx4_flow_stop(priv);
mlx4_intr_uninstall(priv);
- mlx4_mac_addr_del(priv);
}
/**
mlx4_dev_close(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
- void *tmp;
unsigned int i;
if (priv == NULL)
DEBUG("%p: closing device \"%s\"",
(void *)dev,
((priv->ctx != NULL) ? priv->ctx->device->name : ""));
- mlx4_mac_addr_del(priv);
- /*
- * Prevent crashes when queues are still in use. This is unfortunately
- * still required for DPDK 1.3 because some programs (such as testpmd)
- * never release them before closing the device.
- */
+ mlx4_flow_clean(priv);
dev->rx_pkt_burst = mlx4_rx_burst_removed;
dev->tx_pkt_burst = mlx4_tx_burst_removed;
- if (priv->rxqs != NULL) {
- /* XXX race condition if mlx4_rx_burst() is still running. */
- usleep(1000);
- for (i = 0; (i != priv->rxqs_n); ++i) {
- tmp = (*priv->rxqs)[i];
- if (tmp == NULL)
- continue;
- (*priv->rxqs)[i] = NULL;
- mlx4_rxq_cleanup(tmp);
- rte_free(tmp);
- }
- priv->rxqs_n = 0;
- priv->rxqs = NULL;
- }
- if (priv->txqs != NULL) {
- /* XXX race condition if mlx4_tx_burst() is still running. */
- usleep(1000);
- for (i = 0; (i != priv->txqs_n); ++i) {
- tmp = (*priv->txqs)[i];
- if (tmp == NULL)
- continue;
- (*priv->txqs)[i] = NULL;
- mlx4_txq_cleanup(tmp);
- rte_free(tmp);
- }
- priv->txqs_n = 0;
- priv->txqs = NULL;
- }
+ for (i = 0; i != dev->data->nb_rx_queues; ++i)
+ mlx4_rx_queue_release(dev->data->rx_queues[i]);
+ for (i = 0; i != dev->data->nb_tx_queues; ++i)
+ mlx4_tx_queue_release(dev->data->tx_queues[i]);
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(ibv_dealloc_pd(priv->pd));
mac.addr_bytes[4], mac.addr_bytes[5]);
/* Register MAC address. */
priv->mac = mac;
- if (mlx4_mac_addr_add(priv))
- goto port_error;
#ifndef NDEBUG
{
char ifname[IF_NAMESIZE];