* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-/* System headers. */
+/**
+ * @file
+ * mlx4 driver initialization.
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <inttypes.h>
#include <stddef.h>
+#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
-#include <stdint.h>
-#include <inttypes.h>
#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <assert.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_ethdev_pci.h>
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_common.h>
#include <rte_dev.h>
-#include <rte_mbuf.h>
#include <rte_errno.h>
-#include <rte_mempool.h>
-#include <rte_malloc.h>
-#include <rte_memory.h>
+#include <rte_ethdev.h>
+#include <rte_ethdev_pci.h>
+#include <rte_ether.h>
#include <rte_flow.h>
-#include <rte_kvargs.h>
#include <rte_interrupts.h>
-#include <rte_common.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
-/* PMD headers. */
#include "mlx4.h"
#include "mlx4_flow.h"
#include "mlx4_rxtx.h"
NULL,
};
-/* Device configuration. */
-
/**
* DPDK callback for Ethernet device configuration.
*
- * Prepare the driver for a given number of TX and RX queues.
- *
* @param dev
* Pointer to Ethernet device structure.
*
mlx4_dev_configure(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
- unsigned int rxqs_n = dev->data->nb_rx_queues;
- unsigned int txqs_n = dev->data->nb_tx_queues;
-
- priv->rxqs = (void *)dev->data->rx_queues;
- priv->txqs = (void *)dev->data->tx_queues;
- if (txqs_n != priv->txqs_n) {
- INFO("%p: TX queues number update: %u -> %u",
- (void *)dev, priv->txqs_n, txqs_n);
- priv->txqs_n = txqs_n;
- }
- if (rxqs_n != priv->rxqs_n) {
- INFO("%p: Rx queues number update: %u -> %u",
- (void *)dev, priv->rxqs_n, rxqs_n);
- priv->rxqs_n = rxqs_n;
- }
- return 0;
-}
-
-struct mlx4_check_mempool_data {
+ struct rte_flow_error error;
int ret;
- char *start;
- char *end;
-};
-
-/* Called by mlx4_check_mempool() when iterating the memory chunks. */
-static void mlx4_check_mempool_cb(struct rte_mempool *mp,
- void *opaque, struct rte_mempool_memhdr *memhdr,
- unsigned mem_idx)
-{
- struct mlx4_check_mempool_data *data = opaque;
- (void)mp;
- (void)mem_idx;
- /* It already failed, skip the next chunks. */
- if (data->ret != 0)
- return;
- /* It is the first chunk. */
- if (data->start == NULL && data->end == NULL) {
- data->start = memhdr->addr;
- data->end = data->start + memhdr->len;
- return;
- }
- if (data->end == memhdr->addr) {
- data->end += memhdr->len;
- return;
- }
- if (data->start == (char *)memhdr->addr + memhdr->len) {
- data->start -= memhdr->len;
- return;
- }
- /* Error, mempool is not virtually contigous. */
- data->ret = -1;
-}
-
-/**
- * Check if a mempool can be used: it must be virtually contiguous.
- *
- * @param[in] mp
- * Pointer to memory pool.
- * @param[out] start
- * Pointer to the start address of the mempool virtual memory area
- * @param[out] end
- * Pointer to the end address of the mempool virtual memory area
- *
- * @return
- * 0 on success (mempool is virtually contiguous), -1 on error.
- */
-static int mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start,
- uintptr_t *end)
-{
- struct mlx4_check_mempool_data data;
-
- memset(&data, 0, sizeof(data));
- rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data);
- *start = (uintptr_t)data.start;
- *end = (uintptr_t)data.end;
- return data.ret;
-}
-
-/**
- * Register mempool as a memory region.
- *
- * @param pd
- * Pointer to protection domain.
- * @param mp
- * Pointer to memory pool.
- *
- * @return
- * Memory region pointer, NULL in case of error and rte_errno is set.
- */
-struct ibv_mr *
-mlx4_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp)
-{
- const struct rte_memseg *ms = rte_eal_get_physmem_layout();
- uintptr_t start;
- uintptr_t end;
- unsigned int i;
- struct ibv_mr *mr;
-
- if (mlx4_check_mempool(mp, &start, &end) != 0) {
- rte_errno = EINVAL;
- ERROR("mempool %p: not virtually contiguous",
- (void *)mp);
- return NULL;
- }
- DEBUG("mempool %p area start=%p end=%p size=%zu",
- (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- /* Round start and end to page boundary if found in memory segments. */
- for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
- uintptr_t addr = (uintptr_t)ms[i].addr;
- size_t len = ms[i].len;
- unsigned int align = ms[i].hugepage_sz;
-
- if ((start > addr) && (start < addr + len))
- start = RTE_ALIGN_FLOOR(start, align);
- if ((end > addr) && (end < addr + len))
- end = RTE_ALIGN_CEIL(end, align);
+ /* Prepare internal flow rules. */
+ ret = mlx4_flow_sync(priv, &error);
+ if (ret) {
+ ERROR("cannot set up internal flow rules (code %d, \"%s\"),"
+ " flow error type %d, cause %p, message: %s",
+ -ret, strerror(-ret), error.type, error.cause,
+ error.message ? error.message : "(unspecified)");
}
- DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
- (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- mr = ibv_reg_mr(pd,
- (void *)start,
- end - start,
- IBV_ACCESS_LOCAL_WRITE);
- if (!mr)
- rte_errno = errno ? errno : EINVAL;
- return mr;
+ return ret;
}
/**
* DPDK callback to start the device.
*
- * Simulate device start by attaching all configured flows.
+ * Simulate device start by initializing common RSS resources and attaching
+ * all configured flows.
*
* @param dev
* Pointer to Ethernet device structure.
mlx4_dev_start(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
+ struct rte_flow_error error;
int ret;
if (priv->started)
return 0;
DEBUG("%p: attaching configured flows to all RX queues", (void *)dev);
priv->started = 1;
- ret = mlx4_mac_addr_add(priv);
- if (ret)
+ ret = mlx4_rss_init(priv);
+ if (ret) {
+ ERROR("%p: cannot initialize RSS resources: %s",
+ (void *)dev, strerror(-ret));
goto err;
+ }
ret = mlx4_intr_install(priv);
if (ret) {
ERROR("%p: interrupt handler installation failed",
(void *)dev);
goto err;
}
- ret = mlx4_priv_flow_start(priv);
+ ret = mlx4_flow_sync(priv, &error);
if (ret) {
- ERROR("%p: flow start failed: %s",
- (void *)dev, strerror(ret));
+ ERROR("%p: cannot attach flow rules (code %d, \"%s\"),"
+ " flow error type %d, cause %p, message: %s",
+ (void *)dev,
+ -ret, strerror(-ret), error.type, error.cause,
+ error.message ? error.message : "(unspecified)");
goto err;
}
+ rte_wmb();
+ dev->tx_pkt_burst = mlx4_tx_burst;
+ dev->rx_pkt_burst = mlx4_rx_burst;
return 0;
err:
/* Rollback. */
- mlx4_mac_addr_del(priv);
priv->started = 0;
return ret;
}
return;
DEBUG("%p: detaching flows from all RX queues", (void *)dev);
priv->started = 0;
- mlx4_priv_flow_stop(priv);
+ dev->tx_pkt_burst = mlx4_tx_burst_removed;
+ dev->rx_pkt_burst = mlx4_rx_burst_removed;
+ rte_wmb();
+ mlx4_flow_sync(priv, NULL);
mlx4_intr_uninstall(priv);
- mlx4_mac_addr_del(priv);
+ mlx4_rss_deinit(priv);
}
/**
mlx4_dev_close(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
- void *tmp;
unsigned int i;
- if (priv == NULL)
- return;
DEBUG("%p: closing device \"%s\"",
(void *)dev,
((priv->ctx != NULL) ? priv->ctx->device->name : ""));
- mlx4_mac_addr_del(priv);
- /*
- * Prevent crashes when queues are still in use. This is unfortunately
- * still required for DPDK 1.3 because some programs (such as testpmd)
- * never release them before closing the device.
- */
dev->rx_pkt_burst = mlx4_rx_burst_removed;
dev->tx_pkt_burst = mlx4_tx_burst_removed;
- if (priv->rxqs != NULL) {
- /* XXX race condition if mlx4_rx_burst() is still running. */
- usleep(1000);
- for (i = 0; (i != priv->rxqs_n); ++i) {
- tmp = (*priv->rxqs)[i];
- if (tmp == NULL)
- continue;
- (*priv->rxqs)[i] = NULL;
- mlx4_rxq_cleanup(tmp);
- rte_free(tmp);
- }
- priv->rxqs_n = 0;
- priv->rxqs = NULL;
- }
- if (priv->txqs != NULL) {
- /* XXX race condition if mlx4_tx_burst() is still running. */
- usleep(1000);
- for (i = 0; (i != priv->txqs_n); ++i) {
- tmp = (*priv->txqs)[i];
- if (tmp == NULL)
- continue;
- (*priv->txqs)[i] = NULL;
- mlx4_txq_cleanup(tmp);
- rte_free(tmp);
- }
- priv->txqs_n = 0;
- priv->txqs = NULL;
- }
+ rte_wmb();
+ mlx4_flow_clean(priv);
+ for (i = 0; i != dev->data->nb_rx_queues; ++i)
+ mlx4_rx_queue_release(dev->data->rx_queues[i]);
+ for (i = 0; i != dev->data->nb_tx_queues; ++i)
+ mlx4_tx_queue_release(dev->data->tx_queues[i]);
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(ibv_dealloc_pd(priv->pd));
memset(priv, 0, sizeof(*priv));
}
-const struct rte_flow_ops mlx4_flow_ops = {
- .validate = mlx4_flow_validate,
- .create = mlx4_flow_create,
- .destroy = mlx4_flow_destroy,
- .flush = mlx4_flow_flush,
- .query = NULL,
- .isolate = mlx4_flow_isolate,
-};
-
-/**
- * Manage filter operations.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param filter_type
- * Filter type.
- * @param filter_op
- * Operation to perform.
- * @param arg
- * Pointer to operation-specific structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_dev_filter_ctrl(struct rte_eth_dev *dev,
- enum rte_filter_type filter_type,
- enum rte_filter_op filter_op,
- void *arg)
-{
- switch (filter_type) {
- case RTE_ETH_FILTER_GENERIC:
- if (filter_op != RTE_ETH_FILTER_GET)
- break;
- *(const void **)arg = &mlx4_flow_ops;
- return 0;
- default:
- ERROR("%p: filter type (%d) not supported",
- (void *)dev, filter_type);
- break;
- }
- rte_errno = ENOTSUP;
- return -rte_errno;
-}
-
static const struct eth_dev_ops mlx4_dev_ops = {
.dev_configure = mlx4_dev_configure,
.dev_start = mlx4_dev_start,
.dev_set_link_up = mlx4_dev_set_link_up,
.dev_close = mlx4_dev_close,
.link_update = mlx4_link_update,
+ .promiscuous_enable = mlx4_promiscuous_enable,
+ .promiscuous_disable = mlx4_promiscuous_disable,
+ .allmulticast_enable = mlx4_allmulticast_enable,
+ .allmulticast_disable = mlx4_allmulticast_disable,
+ .mac_addr_remove = mlx4_mac_addr_remove,
+ .mac_addr_add = mlx4_mac_addr_add,
+ .mac_addr_set = mlx4_mac_addr_set,
.stats_get = mlx4_stats_get,
.stats_reset = mlx4_stats_reset,
.dev_infos_get = mlx4_dev_infos_get,
+ .vlan_filter_set = mlx4_vlan_filter_set,
.rx_queue_setup = mlx4_rx_queue_setup,
.tx_queue_setup = mlx4_tx_queue_setup,
.rx_queue_release = mlx4_rx_queue_release,
.flow_ctrl_get = mlx4_flow_ctrl_get,
.flow_ctrl_set = mlx4_flow_ctrl_set,
.mtu_set = mlx4_mtu_set,
- .filter_ctrl = mlx4_dev_filter_ctrl,
+ .filter_ctrl = mlx4_filter_ctrl,
.rx_queue_intr_enable = mlx4_rx_intr_enable,
.rx_queue_intr_disable = mlx4_rx_intr_disable,
};
priv->pd = pd;
priv->mtu = ETHER_MTU;
priv->vf = vf;
+ priv->hw_csum = !!(device_attr.device_cap_flags &
+ IBV_DEVICE_RAW_IP_CSUM);
+ DEBUG("checksum offloading is %ssupported",
+ (priv->hw_csum ? "" : "not "));
+ /* Only ConnectX-3 Pro supports tunneling. */
+ priv->hw_csum_l2tun =
+ priv->hw_csum &&
+ (device_attr.vendor_part_id ==
+ PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO);
+ DEBUG("L2 tunnel checksum offloads are %ssupported",
+ (priv->hw_csum_l2tun ? "" : "not "));
/* Configure the first MAC address by default. */
if (mlx4_get_mac(priv, &mac.addr_bytes)) {
ERROR("cannot get MAC address, is mlx4_en loaded?"
mac.addr_bytes[2], mac.addr_bytes[3],
mac.addr_bytes[4], mac.addr_bytes[5]);
/* Register MAC address. */
- priv->mac = mac;
- if (mlx4_mac_addr_add(priv))
- goto port_error;
+ priv->mac[0] = mac;
#ifndef NDEBUG
{
char ifname[IF_NAMESIZE];
goto port_error;
}
eth_dev->data->dev_private = priv;
- eth_dev->data->mac_addrs = &priv->mac;
+ eth_dev->data->mac_addrs = priv->mac;
eth_dev->device = &pci_dev->device;
rte_eth_copy_pci_info(eth_dev, pci_dev);
eth_dev->device->driver = &mlx4_driver.driver;
eth_dev->intr_handle = &priv->intr_handle;
priv->dev = eth_dev;
eth_dev->dev_ops = &mlx4_dev_ops;
- eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
/* Bring Ethernet device up. */
DEBUG("forcing Ethernet interface up");
mlx4_dev_set_link_up(priv->dev);
git.droids-corp.org / dpdk.git / blobdiff