-/*-
- * BSD LICENSE
- *
- * Copyright 2015 6WIND S.A.
- * Copyright 2015 Mellanox.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of 6WIND S.A. nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 6WIND S.A.
+ * Copyright 2015 Mellanox Technologies, Ltd
*/
#include <stddef.h>
#include <errno.h>
#include <string.h>
-/* Verbs header. */
-/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
-#ifdef PEDANTIC
-#pragma GCC diagnostic ignored "-Wpedantic"
-#endif
-#include <infiniband/verbs.h>
-#ifdef PEDANTIC
-#pragma GCC diagnostic error "-Wpedantic"
-#endif
-
-#include <rte_ethdev.h>
+#include <ethdev_driver.h>
+#include <mlx5_glue.h>
#include "mlx5.h"
-#include "mlx5_rxtx.h"
#include "mlx5_utils.h"
-/* Initialization data for special flows. */
-static const struct special_flow_init special_flow_init[] = {
- [HASH_RXQ_FLOW_TYPE_PROMISC] = {
- .dst_mac_val = "\x00\x00\x00\x00\x00\x00",
- .dst_mac_mask = "\x00\x00\x00\x00\x00\x00",
- .hash_types =
- 1 << HASH_RXQ_TCPV4 |
- 1 << HASH_RXQ_UDPV4 |
- 1 << HASH_RXQ_IPV4 |
- 1 << HASH_RXQ_TCPV6 |
- 1 << HASH_RXQ_UDPV6 |
- 1 << HASH_RXQ_IPV6 |
- 1 << HASH_RXQ_ETH |
- 0,
- .per_vlan = 0,
- },
- [HASH_RXQ_FLOW_TYPE_ALLMULTI] = {
- .dst_mac_val = "\x01\x00\x00\x00\x00\x00",
- .dst_mac_mask = "\x01\x00\x00\x00\x00\x00",
- .hash_types =
- 1 << HASH_RXQ_UDPV4 |
- 1 << HASH_RXQ_IPV4 |
- 1 << HASH_RXQ_UDPV6 |
- 1 << HASH_RXQ_IPV6 |
- 1 << HASH_RXQ_ETH |
- 0,
- .per_vlan = 0,
- },
- [HASH_RXQ_FLOW_TYPE_BROADCAST] = {
- .dst_mac_val = "\xff\xff\xff\xff\xff\xff",
- .dst_mac_mask = "\xff\xff\xff\xff\xff\xff",
- .hash_types =
- 1 << HASH_RXQ_UDPV4 |
- 1 << HASH_RXQ_IPV4 |
- 1 << HASH_RXQ_UDPV6 |
- 1 << HASH_RXQ_IPV6 |
- 1 << HASH_RXQ_ETH |
- 0,
- .per_vlan = 1,
- },
- [HASH_RXQ_FLOW_TYPE_IPV6MULTI] = {
- .dst_mac_val = "\x33\x33\x00\x00\x00\x00",
- .dst_mac_mask = "\xff\xff\x00\x00\x00\x00",
- .hash_types =
- 1 << HASH_RXQ_UDPV6 |
- 1 << HASH_RXQ_IPV6 |
- 1 << HASH_RXQ_ETH |
- 0,
- .per_vlan = 1,
- },
-};
-
/**
- * Enable a special flow in a hash RX queue for a given VLAN index.
- *
- * @param hash_rxq
- * Pointer to hash RX queue structure.
- * @param flow_type
- * Special flow type.
- * @param vlan_index
- * VLAN index to use.
- *
- * @return
- * 0 on success, errno value on failure.
- */
-static int
-hash_rxq_special_flow_enable_vlan(struct hash_rxq *hash_rxq,
- enum hash_rxq_flow_type flow_type,
- unsigned int vlan_index)
-{
- struct priv *priv = hash_rxq->priv;
- struct ibv_exp_flow *flow;
- FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, hash_rxq->type));
- struct ibv_exp_flow_attr *attr = &data->attr;
- struct ibv_exp_flow_spec_eth *spec = &data->spec;
- const uint8_t *mac;
- const uint8_t *mask;
- unsigned int vlan_enabled = (priv->vlan_filter_n &&
- special_flow_init[flow_type].per_vlan);
- unsigned int vlan_id = priv->vlan_filter[vlan_index];
-
- /* Check if flow is relevant for this hash_rxq. */
- if (!(special_flow_init[flow_type].hash_types & (1 << hash_rxq->type)))
- return 0;
- /* Check if flow already exists. */
- if (hash_rxq->special_flow[flow_type][vlan_index] != NULL)
- return 0;
-
- /*
- * No padding must be inserted by the compiler between attr and spec.
- * This layout is expected by libibverbs.
- */
- assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec);
- priv_flow_attr(priv, attr, sizeof(data), hash_rxq->type);
- /* The first specification must be Ethernet. */
- assert(spec->type == IBV_EXP_FLOW_SPEC_ETH);
- assert(spec->size == sizeof(*spec));
-
- mac = special_flow_init[flow_type].dst_mac_val;
- mask = special_flow_init[flow_type].dst_mac_mask;
- *spec = (struct ibv_exp_flow_spec_eth){
- .type = IBV_EXP_FLOW_SPEC_ETH,
- .size = sizeof(*spec),
- .val = {
- .dst_mac = {
- mac[0], mac[1], mac[2],
- mac[3], mac[4], mac[5],
- },
- .vlan_tag = (vlan_enabled ? htons(vlan_id) : 0),
- },
- .mask = {
- .dst_mac = {
- mask[0], mask[1], mask[2],
- mask[3], mask[4], mask[5],
- },
- .vlan_tag = (vlan_enabled ? htons(0xfff) : 0),
- },
- };
-
- errno = 0;
- flow = ibv_exp_create_flow(hash_rxq->qp, attr);
- if (flow == NULL) {
- /* It's not clear whether errno is always set in this case. */
- ERROR("%p: flow configuration failed, errno=%d: %s",
- (void *)hash_rxq, errno,
- (errno ? strerror(errno) : "Unknown error"));
- if (errno)
- return errno;
- return EINVAL;
- }
- hash_rxq->special_flow[flow_type][vlan_index] = flow;
- DEBUG("%p: special flow %s (index %d) VLAN %u (index %u) enabled",
- (void *)hash_rxq, hash_rxq_flow_type_str(flow_type), flow_type,
- vlan_id, vlan_index);
- return 0;
-}
-
-/**
- * Disable a special flow in a hash RX queue for a given VLAN index.
- *
- * @param hash_rxq
- * Pointer to hash RX queue structure.
- * @param flow_type
- * Special flow type.
- * @param vlan_index
- * VLAN index to use.
- */
-static void
-hash_rxq_special_flow_disable_vlan(struct hash_rxq *hash_rxq,
- enum hash_rxq_flow_type flow_type,
- unsigned int vlan_index)
-{
- struct ibv_exp_flow *flow =
- hash_rxq->special_flow[flow_type][vlan_index];
-
- if (flow == NULL)
- return;
- claim_zero(ibv_exp_destroy_flow(flow));
- hash_rxq->special_flow[flow_type][vlan_index] = NULL;
- DEBUG("%p: special flow %s (index %d) VLAN %u (index %u) disabled",
- (void *)hash_rxq, hash_rxq_flow_type_str(flow_type), flow_type,
- hash_rxq->priv->vlan_filter[vlan_index], vlan_index);
-}
-
-/**
- * Enable a special flow in a hash RX queue.
- *
- * @param hash_rxq
- * Pointer to hash RX queue structure.
- * @param flow_type
- * Special flow type.
- * @param vlan_index
- * VLAN index to use.
- *
- * @return
- * 0 on success, errno value on failure.
- */
-static int
-hash_rxq_special_flow_enable(struct hash_rxq *hash_rxq,
- enum hash_rxq_flow_type flow_type)
-{
- struct priv *priv = hash_rxq->priv;
- unsigned int i = 0;
- int ret;
-
- assert((unsigned int)flow_type < RTE_DIM(hash_rxq->special_flow));
- assert(RTE_DIM(hash_rxq->special_flow[flow_type]) ==
- RTE_DIM(priv->vlan_filter));
- /* Add a special flow for each VLAN filter when relevant. */
- do {
- ret = hash_rxq_special_flow_enable_vlan(hash_rxq, flow_type, i);
- if (ret) {
- /* Failure, rollback. */
- while (i != 0)
- hash_rxq_special_flow_disable_vlan(hash_rxq,
- flow_type,
- --i);
- return ret;
- }
- } while (special_flow_init[flow_type].per_vlan &&
- ++i < priv->vlan_filter_n);
- return 0;
-}
-
-/**
- * Disable a special flow in a hash RX queue.
- *
- * @param hash_rxq
- * Pointer to hash RX queue structure.
- * @param flow_type
- * Special flow type.
- */
-static void
-hash_rxq_special_flow_disable(struct hash_rxq *hash_rxq,
- enum hash_rxq_flow_type flow_type)
-{
- unsigned int i;
-
- assert((unsigned int)flow_type < RTE_DIM(hash_rxq->special_flow));
- for (i = 0; (i != RTE_DIM(hash_rxq->special_flow[flow_type])); ++i)
- hash_rxq_special_flow_disable_vlan(hash_rxq, flow_type, i);
-}
-
-/**
- * Enable a special flow in all hash RX queues.
+ * DPDK callback to enable promiscuous mode.
*
- * @param priv
- * Private structure.
- * @param flow_type
- * Special flow type.
+ * @param dev
+ * Pointer to Ethernet device structure.
*
* @return
- * 0 on success, errno value on failure.
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
*/
int
-priv_special_flow_enable(struct priv *priv, enum hash_rxq_flow_type flow_type)
+mlx5_promiscuous_enable(struct rte_eth_dev *dev)
{
- unsigned int i;
+ struct mlx5_priv *priv = dev->data->dev_private;
+ int ret;
- if (!priv_allow_flow_type(priv, flow_type))
+ dev->data->promiscuous = 1;
+ if (priv->isolated) {
+ DRV_LOG(WARNING,
+ "port %u cannot enable promiscuous mode"
+ " in flow isolation mode",
+ dev->data->port_id);
return 0;
- for (i = 0; (i != priv->hash_rxqs_n); ++i) {
- struct hash_rxq *hash_rxq = &(*priv->hash_rxqs)[i];
- int ret;
-
- ret = hash_rxq_special_flow_enable(hash_rxq, flow_type);
- if (!ret)
- continue;
- /* Failure, rollback. */
- while (i != 0) {
- hash_rxq = &(*priv->hash_rxqs)[--i];
- hash_rxq_special_flow_disable(hash_rxq, flow_type);
- }
- return ret;
- }
- return 0;
-}
-
-/**
- * Disable a special flow in all hash RX queues.
- *
- * @param priv
- * Private structure.
- * @param flow_type
- * Special flow type.
- */
-void
-priv_special_flow_disable(struct priv *priv, enum hash_rxq_flow_type flow_type)
-{
- unsigned int i;
-
- for (i = 0; (i != priv->hash_rxqs_n); ++i) {
- struct hash_rxq *hash_rxq = &(*priv->hash_rxqs)[i];
-
- hash_rxq_special_flow_disable(hash_rxq, flow_type);
}
-}
-
-/**
- * Enable all special flows in all hash RX queues.
- *
- * @param priv
- * Private structure.
- */
-int
-priv_special_flow_enable_all(struct priv *priv)
-{
- enum hash_rxq_flow_type flow_type;
-
- if (priv->isolated)
- return 0;
- for (flow_type = HASH_RXQ_FLOW_TYPE_PROMISC;
- flow_type != HASH_RXQ_FLOW_TYPE_MAC;
- ++flow_type) {
- int ret;
-
- ret = priv_special_flow_enable(priv, flow_type);
- if (!ret)
- continue;
- /* Failure, rollback. */
- while (flow_type)
- priv_special_flow_disable(priv, --flow_type);
- return ret;
+ if (priv->sh->dev_cap.vf || priv->sh->dev_cap.sf) {
+ ret = mlx5_os_set_promisc(dev, 1);
+ if (ret)
+ return ret;
}
- return 0;
-}
-
-/**
- * Disable all special flows in all hash RX queues.
- *
- * @param priv
- * Private structure.
- */
-void
-priv_special_flow_disable_all(struct priv *priv)
-{
- enum hash_rxq_flow_type flow_type;
-
- for (flow_type = HASH_RXQ_FLOW_TYPE_PROMISC;
- flow_type != HASH_RXQ_FLOW_TYPE_MAC;
- ++flow_type)
- priv_special_flow_disable(priv, flow_type);
-}
-
-/**
- * DPDK callback to enable promiscuous mode.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- */
-void
-mlx5_promiscuous_enable(struct rte_eth_dev *dev)
-{
- struct priv *priv = dev->data->dev_private;
- int ret;
-
- if (mlx5_is_secondary())
- return;
-
- priv_lock(priv);
- priv->promisc_req = 1;
- ret = priv_rehash_flows(priv);
+ ret = mlx5_traffic_restart(dev);
if (ret)
- ERROR("error while enabling promiscuous mode: %s",
- strerror(ret));
- priv_unlock(priv);
+ DRV_LOG(ERR, "port %u cannot enable promiscuous mode: %s",
+ dev->data->port_id, strerror(rte_errno));
+
+ /*
+ * rte_eth_dev_promiscuous_enable() rollback
+ * dev->data->promiscuous in the case of failure.
+ */
+ return ret;
}
/**
*
* @param dev
* Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-void
+int
mlx5_promiscuous_disable(struct rte_eth_dev *dev)
{
- struct priv *priv = dev->data->dev_private;
+ struct mlx5_priv *priv = dev->data->dev_private;
int ret;
- if (mlx5_is_secondary())
- return;
-
- priv_lock(priv);
- priv->promisc_req = 0;
- ret = priv_rehash_flows(priv);
+ dev->data->promiscuous = 0;
+ if (priv->sh->dev_cap.vf || priv->sh->dev_cap.sf) {
+ ret = mlx5_os_set_promisc(dev, 0);
+ if (ret)
+ return ret;
+ }
+ ret = mlx5_traffic_restart(dev);
if (ret)
- ERROR("error while disabling promiscuous mode: %s",
- strerror(ret));
- priv_unlock(priv);
+ DRV_LOG(ERR, "port %u cannot disable promiscuous mode: %s",
+ dev->data->port_id, strerror(rte_errno));
+
+ /*
+ * rte_eth_dev_promiscuous_disable() rollback
+ * dev->data->promiscuous in the case of failure.
+ */
+ return ret;
}
/**
*
* @param dev
* Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-void
+int
mlx5_allmulticast_enable(struct rte_eth_dev *dev)
{
- struct priv *priv = dev->data->dev_private;
+ struct mlx5_priv *priv = dev->data->dev_private;
int ret;
- if (mlx5_is_secondary())
- return;
-
- priv_lock(priv);
- priv->allmulti_req = 1;
- ret = priv_rehash_flows(priv);
+ dev->data->all_multicast = 1;
+ if (priv->isolated) {
+ DRV_LOG(WARNING,
+ "port %u cannot enable allmulticast mode"
+ " in flow isolation mode",
+ dev->data->port_id);
+ return 0;
+ }
+ if (priv->sh->dev_cap.vf || priv->sh->dev_cap.sf) {
+ ret = mlx5_os_set_allmulti(dev, 1);
+ if (ret)
+ goto error;
+ }
+ ret = mlx5_traffic_restart(dev);
if (ret)
- ERROR("error while enabling allmulticast mode: %s",
- strerror(ret));
- priv_unlock(priv);
+ DRV_LOG(ERR, "port %u cannot enable allmulicast mode: %s",
+ dev->data->port_id, strerror(rte_errno));
+error:
+ /*
+ * rte_eth_allmulticast_enable() rollback
+ * dev->data->all_multicast in the case of failure.
+ */
+ return ret;
}
/**
*
* @param dev
* Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-void
+int
mlx5_allmulticast_disable(struct rte_eth_dev *dev)
{
- struct priv *priv = dev->data->dev_private;
+ struct mlx5_priv *priv = dev->data->dev_private;
int ret;
- if (mlx5_is_secondary())
- return;
-
- priv_lock(priv);
- priv->allmulti_req = 0;
- ret = priv_rehash_flows(priv);
+ dev->data->all_multicast = 0;
+ if (priv->sh->dev_cap.vf || priv->sh->dev_cap.sf) {
+ ret = mlx5_os_set_allmulti(dev, 0);
+ if (ret)
+ goto error;
+ }
+ ret = mlx5_traffic_restart(dev);
if (ret)
- ERROR("error while disabling allmulticast mode: %s",
- strerror(ret));
- priv_unlock(priv);
+ DRV_LOG(ERR, "port %u cannot disable allmulicast mode: %s",
+ dev->data->port_id, strerror(rte_errno));
+error:
+ /*
+ * rte_eth_allmulticast_disable() rollback
+ * dev->data->all_multicast in the case of failure.
+ */
+ return ret;
}