+/**
+ * Generate internal flow rules.
+ *
+ * Various flow rules are created depending on the mode the device is in:
+ *
+ * 1. Promiscuous:
+ * port MAC + broadcast + catch-all (VLAN filtering is ignored).
+ * 2. All multicast:
+ * port MAC/VLAN + broadcast + catch-all multicast.
+ * 3. Otherwise:
+ * port MAC/VLAN + broadcast MAC/VLAN.
+ *
+ * About MAC flow rules:
+ *
+ * - MAC flow rules are generated from @p dev->data->mac_addrs
+ * (@p priv->mac array).
+ * - An additional flow rule for Ethernet broadcasts is also generated.
+ * - All these are per-VLAN if @p DEV_RX_OFFLOAD_VLAN_FILTER
+ * is enabled and VLAN filters are configured.
+ *
+ * @param priv
+ * Pointer to private structure.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx4_flow_internal(struct mlx4_priv *priv, struct rte_flow_error *error)
+{
+ struct rte_flow_attr attr = {
+ .priority = MLX4_FLOW_PRIORITY_LAST,
+ .ingress = 1,
+ };
+ struct rte_flow_item_eth eth_spec;
+ const struct rte_flow_item_eth eth_mask = {
+ .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+ };
+ const struct rte_flow_item_eth eth_allmulti = {
+ .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
+ };
+ struct rte_flow_item_vlan vlan_spec;
+ const struct rte_flow_item_vlan vlan_mask = {
+ .tci = RTE_BE16(0x0fff),
+ };
+ struct rte_flow_item pattern[] = {
+ {
+ .type = MLX4_FLOW_ITEM_TYPE_INTERNAL,
+ },
+ {
+ .type = RTE_FLOW_ITEM_TYPE_ETH,
+ .spec = ð_spec,
+ .mask = ð_mask,
+ },
+ {
+ /* Replaced with VLAN if filtering is enabled. */
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ },
+ {
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ },
+ };
+ /*
+ * Round number of queues down to their previous power of 2 to
+ * comply with RSS context limitations. Extra queues silently do not
+ * get RSS by default.
+ */
+ uint32_t queues =
+ rte_align32pow2(ETH_DEV(priv)->data->nb_rx_queues + 1) >> 1;
+ uint16_t queue[queues];
+ struct rte_flow_action_rss action_rss = {
+ .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
+ .level = 0,
+ .types = 0,
+ .key_len = MLX4_RSS_HASH_KEY_SIZE,
+ .queue_num = queues,
+ .key = mlx4_rss_hash_key_default,
+ .queue = queue,
+ };
+ struct rte_flow_action actions[] = {
+ {
+ .type = RTE_FLOW_ACTION_TYPE_RSS,
+ .conf = &action_rss,
+ },
+ {
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ },
+ };
+ struct rte_ether_addr *rule_mac = ð_spec.dst;
+ rte_be16_t *rule_vlan =
+ (ETH_DEV(priv)->data->dev_conf.rxmode.offloads &
+ DEV_RX_OFFLOAD_VLAN_FILTER) &&
+ !ETH_DEV(priv)->data->promiscuous ?
+ &vlan_spec.tci :
+ NULL;
+ uint16_t vlan = 0;
+ struct rte_flow *flow;
+ unsigned int i;
+ int err = 0;
+
+ /* Nothing to be done if there are no Rx queues. */
+ if (!queues)
+ goto error;
+ /* Prepare default RSS configuration. */
+ for (i = 0; i != queues; ++i)
+ queue[i] = i;
+ /*
+ * Set up VLAN item if filtering is enabled and at least one VLAN
+ * filter is configured.
+ */
+ if (rule_vlan) {
+ vlan = mlx4_flow_internal_next_vlan(priv, 0);
+ if (vlan < 4096) {
+ pattern[2] = (struct rte_flow_item){
+ .type = RTE_FLOW_ITEM_TYPE_VLAN,
+ .spec = &vlan_spec,
+ .mask = &vlan_mask,
+ };
+next_vlan:
+ *rule_vlan = rte_cpu_to_be_16(vlan);
+ } else {
+ rule_vlan = NULL;
+ }
+ }
+ for (i = 0; i != RTE_DIM(priv->mac) + 1; ++i) {
+ const struct rte_ether_addr *mac;
+
+ /* Broadcasts are handled by an extra iteration. */
+ if (i < RTE_DIM(priv->mac))
+ mac = &priv->mac[i];
+ else
+ mac = ð_mask.dst;
+ if (rte_is_zero_ether_addr(mac))
+ continue;
+ /* Check if MAC flow rule is already present. */
+ for (flow = LIST_FIRST(&priv->flows);
+ flow && flow->internal;
+ flow = LIST_NEXT(flow, next)) {
+ const struct ibv_flow_spec_eth *eth =
+ (const void *)((uintptr_t)flow->ibv_attr +
+ sizeof(*flow->ibv_attr));
+ unsigned int j;
+
+ if (!flow->mac)
+ continue;
+ MLX4_ASSERT(flow->ibv_attr->type ==
+ IBV_FLOW_ATTR_NORMAL);
+ MLX4_ASSERT(flow->ibv_attr->num_of_specs == 1);
+ MLX4_ASSERT(eth->type == IBV_FLOW_SPEC_ETH);
+ MLX4_ASSERT(flow->rss);
+ if (rule_vlan &&
+ (eth->val.vlan_tag != *rule_vlan ||
+ eth->mask.vlan_tag != RTE_BE16(0x0fff)))
+ continue;
+ if (!rule_vlan && eth->mask.vlan_tag)
+ continue;
+ for (j = 0; j != sizeof(mac->addr_bytes); ++j)
+ if (eth->val.dst_mac[j] != mac->addr_bytes[j] ||
+ eth->mask.dst_mac[j] != UINT8_C(0xff) ||
+ eth->val.src_mac[j] != UINT8_C(0x00) ||
+ eth->mask.src_mac[j] != UINT8_C(0x00))
+ break;
+ if (j != sizeof(mac->addr_bytes))
+ continue;
+ if (flow->rss->queues != queues ||
+ memcmp(flow->rss->queue_id, action_rss.queue,
+ queues * sizeof(flow->rss->queue_id[0])))
+ continue;
+ break;
+ }
+ if (!flow || !flow->internal) {
+ /* Not found, create a new flow rule. */
+ memcpy(rule_mac, mac, sizeof(*mac));
+ flow = mlx4_flow_create(ETH_DEV(priv), &attr, pattern,
+ actions, error);
+ if (!flow) {
+ err = -rte_errno;
+ goto error;
+ }
+ }
+ flow->select = 1;
+ flow->mac = 1;