net/mlx5: reduce PCI dependency
[dpdk.git] / drivers / net / mlx5 / linux / mlx5_os.c
index 17d0533..4ab30fd 100644 (file)
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -69,6 +67,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+       {
+               .size = sizeof(struct rte_flow),
+               .trunk_size = 64,
+               .need_lock = 1,
+               .release_mem_en = 0,
+               .malloc = mlx5_malloc,
+               .free = mlx5_free,
+               .per_core_cache = 0,
+               .type = "ctl_flow_ipool",
+       },
+       {
+               .size = sizeof(struct rte_flow),
+               .trunk_size = 64,
+               .grow_trunk = 3,
+               .grow_shift = 2,
+               .need_lock = 1,
+               .release_mem_en = 0,
+               .malloc = mlx5_malloc,
+               .free = mlx5_free,
+               .per_core_cache = 1 << 14,
+               .type = "rte_flow_ipool",
+       },
+       {
+               .size = sizeof(struct rte_flow),
+               .trunk_size = 64,
+               .grow_trunk = 3,
+               .grow_shift = 2,
+               .need_lock = 1,
+               .release_mem_en = 0,
+               .malloc = mlx5_malloc,
+               .free = mlx5_free,
+               .per_core_cache = 0,
+               .type = "mcp_flow_ipool",
+       },
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -154,6 +190,8 @@ mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr)
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
        device_attr->tunnel_offloads_caps = dv_attr.tunnel_offloads_caps;
 #endif
+       strlcpy(device_attr->fw_ver, attr_ex.orig_attr.fw_ver,
+               sizeof(device_attr->fw_ver));
 
        return err;
 }
@@ -191,6 +229,79 @@ mlx5_alloc_verbs_buf(size_t size, void *data)
        return ret;
 }
 
+/**
+ * Detect misc5 support or not
+ *
+ * @param[in] priv
+ *   Device private data pointer
+ */
+#ifdef HAVE_MLX5DV_DR
+static void
+__mlx5_discovery_misc5_cap(struct mlx5_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+       /* Dummy VxLAN matcher to detect rdma-core misc5 cap
+        * Case: IPv4--->UDP--->VxLAN--->vni
+        */
+       void *tbl;
+       struct mlx5_flow_dv_match_params matcher_mask;
+       void *match_m;
+       void *matcher;
+       void *headers_m;
+       void *misc5_m;
+       uint32_t *tunnel_header_m;
+       struct mlx5dv_flow_matcher_attr dv_attr;
+
+       memset(&matcher_mask, 0, sizeof(matcher_mask));
+       matcher_mask.size = sizeof(matcher_mask.buf);
+       match_m = matcher_mask.buf;
+       headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers);
+       misc5_m = MLX5_ADDR_OF(fte_match_param,
+                              match_m, misc_parameters_5);
+       tunnel_header_m = (uint32_t *)
+                               MLX5_ADDR_OF(fte_match_set_misc5,
+                               misc5_m, tunnel_header_1);
+       MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
+       MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 4);
+       MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
+       *tunnel_header_m = 0xffffff;
+
+       tbl = mlx5_glue->dr_create_flow_tbl(priv->sh->rx_domain, 1);
+       if (!tbl) {
+               DRV_LOG(INFO, "No SW steering support");
+               return;
+       }
+       dv_attr.type = IBV_FLOW_ATTR_NORMAL,
+       dv_attr.match_mask = (void *)&matcher_mask,
+       dv_attr.match_criteria_enable =
+                       (1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT) |
+                       (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT);
+       dv_attr.priority = 3;
+#ifdef HAVE_MLX5DV_DR_ESWITCH
+       void *misc2_m;
+       if (priv->config.dv_esw_en) {
+               /* FDB enabled reg_c_0 */
+               dv_attr.match_criteria_enable |=
+                               (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT);
+               misc2_m = MLX5_ADDR_OF(fte_match_param,
+                                      match_m, misc_parameters_2);
+               MLX5_SET(fte_match_set_misc2, misc2_m,
+                        metadata_reg_c_0, 0xffff);
+       }
+#endif
+       matcher = mlx5_glue->dv_create_flow_matcher(priv->sh->ctx,
+                                                   &dv_attr, tbl);
+       if (matcher) {
+               priv->sh->misc5_cap = 1;
+               mlx5_glue->dv_destroy_flow_matcher(matcher);
+       }
+       mlx5_glue->dr_destroy_flow_tbl(tbl);
+#else
+       RTE_SET_USED(priv);
+#endif
+}
+#endif
+
 /**
  * Verbs callback to free a memory.
  *
@@ -221,7 +332,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
        struct mlx5_dev_ctx_shared *sh = priv->sh;
-       char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+       char s[MLX5_NAME_SIZE] __rte_unused;
        int err;
 
        MLX5_ASSERT(sh && sh->refcnt);
@@ -232,70 +343,46 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
                goto error;
        /* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-       /* Init port id action cache list. */
-       snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-       mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
-                            flow_dv_port_id_create_cb,
-                            flow_dv_port_id_match_cb,
-                            flow_dv_port_id_remove_cb);
-       /* Init push vlan action cache list. */
-       snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-       mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
-                            flow_dv_push_vlan_create_cb,
-                            flow_dv_push_vlan_match_cb,
-                            flow_dv_push_vlan_remove_cb);
-       /* Init sample action cache list. */
-       snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-       mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
-                            flow_dv_sample_create_cb,
-                            flow_dv_sample_match_cb,
-                            flow_dv_sample_remove_cb);
-       /* Init dest array action cache list. */
-       snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-       mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
-                            flow_dv_dest_array_create_cb,
-                            flow_dv_dest_array_match_cb,
-                            flow_dv_dest_array_remove_cb);
-       /* Create tags hash list table. */
-       snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-       sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-                                         MLX5_HLIST_WRITE_MOST,
-                                         flow_dv_tag_create_cb,
-                                         flow_dv_tag_match_cb,
-                                         flow_dv_tag_remove_cb);
-       if (!sh->tag_table) {
-               DRV_LOG(ERR, "tags with hash creation failed.");
-               err = ENOMEM;
+       /* Init port id action list. */
+       snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+       sh->port_id_action_list = mlx5_list_create(s, sh, true,
+                                                  flow_dv_port_id_create_cb,
+                                                  flow_dv_port_id_match_cb,
+                                                  flow_dv_port_id_remove_cb,
+                                                  flow_dv_port_id_clone_cb,
+                                                flow_dv_port_id_clone_free_cb);
+       if (!sh->port_id_action_list)
                goto error;
-       }
-       sh->tag_table->ctx = sh;
-       snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-       sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-                                           0, MLX5_HLIST_WRITE_MOST |
-                                           MLX5_HLIST_DIRECT_KEY,
-                                           flow_dv_modify_create_cb,
-                                           flow_dv_modify_match_cb,
-                                           flow_dv_modify_remove_cb);
-       if (!sh->modify_cmds) {
-               DRV_LOG(ERR, "hdr modify hash creation failed");
-               err = ENOMEM;
+       /* Init push vlan action list. */
+       snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+       sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
+                                                   flow_dv_push_vlan_create_cb,
+                                                   flow_dv_push_vlan_match_cb,
+                                                   flow_dv_push_vlan_remove_cb,
+                                                   flow_dv_push_vlan_clone_cb,
+                                              flow_dv_push_vlan_clone_free_cb);
+       if (!sh->push_vlan_action_list)
                goto error;
-       }
-       sh->modify_cmds->ctx = sh;
-       snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-       sh->encaps_decaps = mlx5_hlist_create(s,
-                                             MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-                                             0, MLX5_HLIST_DIRECT_KEY |
-                                             MLX5_HLIST_WRITE_MOST,
-                                             flow_dv_encap_decap_create_cb,
-                                             flow_dv_encap_decap_match_cb,
-                                             flow_dv_encap_decap_remove_cb);
-       if (!sh->encaps_decaps) {
-               DRV_LOG(ERR, "encap decap hash creation failed");
-               err = ENOMEM;
+       /* Init sample action list. */
+       snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+       sh->sample_action_list = mlx5_list_create(s, sh, true,
+                                                 flow_dv_sample_create_cb,
+                                                 flow_dv_sample_match_cb,
+                                                 flow_dv_sample_remove_cb,
+                                                 flow_dv_sample_clone_cb,
+                                                 flow_dv_sample_clone_free_cb);
+       if (!sh->sample_action_list)
+               goto error;
+       /* Init dest array action list. */
+       snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+       sh->dest_array_list = mlx5_list_create(s, sh, true,
+                                              flow_dv_dest_array_create_cb,
+                                              flow_dv_dest_array_match_cb,
+                                              flow_dv_dest_array_remove_cb,
+                                              flow_dv_dest_array_clone_cb,
+                                             flow_dv_dest_array_clone_free_cb);
+       if (!sh->dest_array_list)
                goto error;
-       }
-       sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
        void *domain;
@@ -340,7 +427,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
                goto error;
        }
 #endif
-       if (!sh->tunnel_hub)
+       if (!sh->tunnel_hub && priv->config.dv_miss_info)
                err = mlx5_alloc_tunnel_hub(sh);
        if (err) {
                DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
@@ -353,6 +440,17 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
                        mlx5_glue->dr_reclaim_domain_memory(sh->fdb_domain, 1);
        }
        sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan();
+       if (!priv->config.allow_duplicate_pattern) {
+#ifndef HAVE_MLX5_DR_ALLOW_DUPLICATE
+               DRV_LOG(WARNING, "Disallow duplicate pattern is not supported - maybe old rdma-core version?");
+#endif
+               mlx5_glue->dr_allow_duplicate_rules(sh->rx_domain, 0);
+               mlx5_glue->dr_allow_duplicate_rules(sh->tx_domain, 0);
+               if (sh->fdb_domain)
+                       mlx5_glue->dr_allow_duplicate_rules(sh->fdb_domain, 0);
+       }
+
+       __mlx5_discovery_misc5_cap(priv);
 #endif /* HAVE_MLX5DV_DR */
        sh->default_miss_action =
                        mlx5_glue->dr_create_flow_action_default_miss();
@@ -399,6 +497,22 @@ error:
                sh->tunnel_hub = NULL;
        }
        mlx5_free_table_hash_list(priv);
+       if (sh->port_id_action_list) {
+               mlx5_list_destroy(sh->port_id_action_list);
+               sh->port_id_action_list = NULL;
+       }
+       if (sh->push_vlan_action_list) {
+               mlx5_list_destroy(sh->push_vlan_action_list);
+               sh->push_vlan_action_list = NULL;
+       }
+       if (sh->sample_action_list) {
+               mlx5_list_destroy(sh->sample_action_list);
+               sh->sample_action_list = NULL;
+       }
+       if (sh->dest_array_list) {
+               mlx5_list_destroy(sh->dest_array_list);
+               sh->dest_array_list = NULL;
+       }
        return err;
 }
 
@@ -460,9 +574,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
                mlx5_release_tunnel_hub(sh, priv->dev_port);
                sh->tunnel_hub = NULL;
        }
-       mlx5_cache_list_destroy(&sh->port_id_action_list);
-       mlx5_cache_list_destroy(&sh->push_vlan_action_list);
        mlx5_free_table_hash_list(priv);
+       if (sh->port_id_action_list) {
+               mlx5_list_destroy(sh->port_id_action_list);
+               sh->port_id_action_list = NULL;
+       }
+       if (sh->push_vlan_action_list) {
+               mlx5_list_destroy(sh->push_vlan_action_list);
+               sh->push_vlan_action_list = NULL;
+       }
+       if (sh->sample_action_list) {
+               mlx5_list_destroy(sh->sample_action_list);
+               sh->sample_action_list = NULL;
+       }
+       if (sh->dest_array_list) {
+               mlx5_list_destroy(sh->dest_array_list);
+               sh->dest_array_list = NULL;
+       }
 }
 
 /**
@@ -820,9 +948,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
        char name[RTE_ETH_NAME_MAX_LEN];
        int own_domain_id = 0;
        uint16_t port_id;
-#ifdef HAVE_MLX5DV_DR_DEVX_PORT
-       struct mlx5dv_devx_port devx_port = { .comp_mask = 0 };
-#endif
+       struct mlx5_port_info vport_info = { .query_flags = 0 };
+       int i;
 
        /* Determine if this port representor is supposed to be spawned. */
        if (switch_info->representor && dpdk_dev->devargs &&
@@ -1053,29 +1180,27 @@ err_secondary:
        priv->vport_meta_tag = 0;
        priv->vport_meta_mask = 0;
        priv->pf_bond = spawn->pf_bond;
-#ifdef HAVE_MLX5DV_DR_DEVX_PORT
        /*
-        * The DevX port query API is implemented. E-Switch may use
-        * either vport or reg_c[0] metadata register to match on
-        * vport index. The engaged part of metadata register is
-        * defined by mask.
+        * If we have E-Switch we should determine the vport attributes.
+        * E-Switch may use either source vport field or reg_c[0] metadata
+        * register to match on vport index. The engaged part of metadata
+        * register is defined by mask.
         */
        if (switch_info->representor || switch_info->master) {
-               devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT |
-                                     MLX5DV_DEVX_PORT_MATCH_REG_C_0;
-               err = mlx5_glue->devx_port_query(sh->ctx, spawn->phys_port,
-                                                &devx_port);
+               err = mlx5_glue->devx_port_query(sh->ctx,
+                                                spawn->phys_port,
+                                                &vport_info);
                if (err) {
                        DRV_LOG(WARNING,
                                "can't query devx port %d on device %s",
                                spawn->phys_port,
                                mlx5_os_get_dev_device_name(spawn->phys_dev));
-                       devx_port.comp_mask = 0;
+                       vport_info.query_flags = 0;
                }
        }
-       if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) {
-               priv->vport_meta_tag = devx_port.reg_c_0.value;
-               priv->vport_meta_mask = devx_port.reg_c_0.mask;
+       if (vport_info.query_flags & MLX5_PORT_QUERY_REG_C0) {
+               priv->vport_meta_tag = vport_info.vport_meta_tag;
+               priv->vport_meta_mask = vport_info.vport_meta_mask;
                if (!priv->vport_meta_mask) {
                        DRV_LOG(ERR, "vport zero mask for port %d"
                                     " on bonding device %s",
@@ -1095,9 +1220,10 @@ err_secondary:
                        goto error;
                }
        }
-       if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) {
-               priv->vport_id = devx_port.vport_num;
-       } else if (spawn->pf_bond >= 0) {
+       if (vport_info.query_flags & MLX5_PORT_QUERY_VPORT) {
+               priv->vport_id = vport_info.vport_id;
+       } else if (spawn->pf_bond >= 0 &&
+                  (switch_info->representor || switch_info->master)) {
                DRV_LOG(ERR, "can't deduce vport index for port %d"
                             " on bonding device %s",
                             spawn->phys_port,
@@ -1105,32 +1231,28 @@ err_secondary:
                err = ENOTSUP;
                goto error;
        } else {
-               /* Suppose vport index in compatible way. */
+               /*
+                * Suppose vport index in compatible way. Kernel/rdma_core
+                * support single E-Switch per PF configurations only and
+                * vport_id field contains the vport index for associated VF,
+                * which is deduced from representor port name.
+                * For example, let's have the IB device port 10, it has
+                * attached network device eth0, which has port name attribute
+                * pf0vf2, we can deduce the VF number as 2, and set vport index
+                * as 3 (2+1). This assigning schema should be changed if the
+                * multiple E-Switch instances per PF configurations or/and PCI
+                * subfunctions are added.
+                */
                priv->vport_id = switch_info->representor ?
                                 switch_info->port_name + 1 : -1;
        }
-#else
-       /*
-        * Kernel/rdma_core support single E-Switch per PF configurations
-        * only and vport_id field contains the vport index for
-        * associated VF, which is deduced from representor port name.
-        * For example, let's have the IB device port 10, it has
-        * attached network device eth0, which has port name attribute
-        * pf0vf2, we can deduce the VF number as 2, and set vport index
-        * as 3 (2+1). This assigning schema should be changed if the
-        * multiple E-Switch instances per PF configurations or/and PCI
-        * subfunctions are added.
-        */
-       priv->vport_id = switch_info->representor ?
-                        switch_info->port_name + 1 : -1;
-#endif
        priv->representor_id = mlx5_representor_id_encode(switch_info,
                                                          eth_da->type);
        /*
         * Look for sibling devices in order to reuse their switch domain
         * if any, otherwise allocate one.
         */
-       MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+       MLX5_ETH_FOREACH_DEV(port_id, NULL) {
                const struct mlx5_priv *opriv =
                        rte_eth_devices[port_id].data->dev_private;
 
@@ -1309,6 +1431,8 @@ err_secondary:
                                goto error;
                        }
                }
+               if (config->hca_attr.flow.tunnel_header_0_1)
+                       sh->tunnel_header_0_1 = 1;
 #endif
 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
                if (config->hca_attr.flow_hit_aso &&
@@ -1322,6 +1446,19 @@ err_secondary:
                        DRV_LOG(DEBUG, "Flow Hit ASO is supported.");
                }
 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
+#if defined(HAVE_MLX5_DR_CREATE_ACTION_ASO) && \
+       defined(HAVE_MLX5_DR_ACTION_ASO_CT)
+               if (config->hca_attr.ct_offload &&
+                   priv->mtr_color_reg == REG_C_3) {
+                       err = mlx5_flow_aso_ct_mng_init(sh);
+                       if (err) {
+                               err = -err;
+                               goto error;
+                       }
+                       DRV_LOG(DEBUG, "CT ASO is supported.");
+                       sh->ct_aso_en = 1;
+               }
+#endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO && HAVE_MLX5_DR_ACTION_ASO_CT */
 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_SAMPLE)
                if (config->hca_attr.log_max_ft_sampler_num > 0  &&
                    config->dv_flow_en) {
@@ -1558,11 +1695,12 @@ err_secondary:
                                      mlx5_ifindex(eth_dev),
                                      eth_dev->data->mac_addrs,
                                      MLX5_MAX_MAC_ADDRESSES);
-       priv->flows = 0;
        priv->ctrl_flows = 0;
        rte_spinlock_init(&priv->flow_list_lock);
        TAILQ_INIT(&priv->flow_meters);
-       TAILQ_INIT(&priv->flow_meter_profiles);
+       priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
+       if (!priv->mtr_profile_tbl)
+               goto error;
        /* Hint libmlx5 to use PMD allocator for data plane resources */
        mlx5_glue->dv_set_context_attr(sh->ctx,
                        MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
@@ -1592,6 +1730,14 @@ err_secondary:
        mlx5_set_min_inline(spawn, config);
        /* Store device configuration on private structure. */
        priv->config = *config;
+       for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+               icfg[i].release_mem_en = !!config->reclaim_mode;
+               if (config->reclaim_mode)
+                       icfg[i].per_core_cache = 0;
+               priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+               if (!priv->flows[i])
+                       goto error;
+       }
        /* Create context for virtual machine VLAN workaround. */
        priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
        if (config->dv_flow_en) {
@@ -1616,7 +1762,10 @@ err_secondary:
                priv->obj_ops.txq_obj_new = mlx5_os_txq_obj_new;
                priv->obj_ops.txq_obj_release = mlx5_os_txq_obj_release;
                mlx5_queue_counter_id_prepare(eth_dev);
-
+               priv->obj_ops.lb_dummy_queue_create =
+                                       mlx5_rxq_ibv_obj_dummy_lb_create;
+               priv->obj_ops.lb_dummy_queue_release =
+                                       mlx5_rxq_ibv_obj_dummy_lb_release;
        } else {
                priv->obj_ops = ibv_obj_ops;
        }
@@ -1645,10 +1794,15 @@ err_secondary:
                        err = ENOTSUP;
                        goto error;
        }
-       mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-                            mlx5_hrxq_create_cb,
-                            mlx5_hrxq_match_cb,
-                            mlx5_hrxq_remove_cb);
+       priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+                                      mlx5_hrxq_create_cb,
+                                      mlx5_hrxq_match_cb,
+                                      mlx5_hrxq_remove_cb,
+                                      mlx5_hrxq_clone_cb,
+                                      mlx5_hrxq_clone_free_cb);
+       if (!priv->hrxqs)
+               goto error;
+       rte_rwlock_init(&priv->ind_tbls_lock);
        /* Query availability of metadata reg_c's. */
        err = mlx5_flow_discover_mreg_c(eth_dev);
        if (err < 0) {
@@ -1673,15 +1827,16 @@ err_secondary:
            priv->sh->dv_regc0_mask) {
                priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
                                                      MLX5_FLOW_MREG_HTABLE_SZ,
-                                                     0, 0,
+                                                     false, true, eth_dev,
                                                      flow_dv_mreg_create_cb,
                                                      flow_dv_mreg_match_cb,
-                                                     flow_dv_mreg_remove_cb);
+                                                     flow_dv_mreg_remove_cb,
+                                                     flow_dv_mreg_clone_cb,
+                                                   flow_dv_mreg_clone_free_cb);
                if (!priv->mreg_cp_tbl) {
                        err = ENOMEM;
                        goto error;
                }
-               priv->mreg_cp_tbl->ctx = eth_dev;
        }
        rte_spinlock_init(&priv->shared_act_sl);
        mlx5_flow_counter_mode_config(eth_dev);
@@ -1702,9 +1857,12 @@ error:
                        mlx5_vlan_vmwa_exit(priv->vmwa_context);
                if (eth_dev && priv->drop_queue.hrxq)
                        mlx5_drop_action_destroy(eth_dev);
+               if (priv->mtr_profile_tbl)
+                       mlx5_l3t_destroy(priv->mtr_profile_tbl);
                if (own_domain_id)
                        claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-               mlx5_cache_list_destroy(&priv->hrxqs);
+               if (priv->hrxqs)
+                       mlx5_list_destroy(priv->hrxqs);
                mlx5_free(priv);
                if (eth_dev != NULL)
                        eth_dev->data->dev_private = NULL;
@@ -1837,7 +1995,7 @@ mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev,
                /* Process slave interface names in the loop. */
                snprintf(tmp_str, sizeof(tmp_str),
                         "/sys/class/net/%s", ifname);
-               if (mlx5_dev_to_pci_addr(tmp_str, &pci_addr)) {
+               if (mlx5_get_pci_addr(tmp_str, &pci_addr)) {
                        DRV_LOG(WARNING, "can not get PCI address"
                                         " for netdev \"%s\"", ifname);
                        continue;
@@ -1863,11 +2021,14 @@ mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev,
                                tmp_str);
                        break;
                }
-               /* Match PCI address. */
+               /* Match PCI address, allows BDF0+pfx or BDFx+pfx. */
                if (pci_dev->domain == pci_addr.domain &&
                    pci_dev->bus == pci_addr.bus &&
                    pci_dev->devid == pci_addr.devid &&
-                   pci_dev->function + owner == pci_addr.function)
+                   ((pci_dev->function == 0 &&
+                     pci_dev->function + owner == pci_addr.function) ||
+                    (pci_dev->function == owner &&
+                     pci_addr.function == owner)))
                        pf = info.port_name;
                /* Get ifindex. */
                snprintf(tmp_str, sizeof(tmp_str),
@@ -2011,8 +2172,8 @@ mlx5_os_pci_probe_pf(struct rte_pci_device *pci_dev,
                        break;
                } else {
                        /* Bonding device not found. */
-                       if (mlx5_dev_to_pci_addr
-                               (ibv_list[ret]->ibdev_path, &pci_addr))
+                       if (mlx5_get_pci_addr(ibv_list[ret]->ibdev_path,
+                                             &pci_addr))
                                continue;
                        if (owner_pci.domain != pci_addr.domain ||
                            owner_pci.bus != pci_addr.bus ||
@@ -2276,6 +2437,18 @@ mlx5_os_pci_probe_pf(struct rte_pci_device *pci_dev,
                        ret = -rte_errno;
                        goto exit;
                }
+               /*
+                * New kernels may add the switch_id attribute for the case
+                * there is no E-Switch and we wrongly recognized the
+                * only device as master. Override this if there is the
+                * single device with single port and new device name
+                * format present.
+                */
+               if (nd == 1 &&
+                   list[0].info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK) {
+                       list[0].info.master = 0;
+                       list[0].info.representor = 0;
+               }
        }
        MLX5_ASSERT(ns);
        /*
@@ -2337,6 +2510,8 @@ mlx5_os_pci_probe_pf(struct rte_pci_device *pci_dev,
                dev_config.dv_flow_en = 1;
                dev_config.decap_en = 1;
                dev_config.log_hp_size = MLX5_ARG_UNSET;
+               dev_config.allow_duplicate_pattern = 1;
+               list[i].numa_node = pci_dev->device.numa_node;
                list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device,
                                                 &list[i],
                                                 &dev_config,
@@ -2534,7 +2709,6 @@ mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
        int dbmap_env;
        int err = 0;
 
-       sh->numa_node = spawn->pci_dev->device.numa_node;
        pthread_mutex_init(&sh->txpp.mutex, NULL);
        /*
         * Configure environment variable "MLX5_BF_SHUT_UP"
@@ -2683,8 +2857,8 @@ mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name,
        if (priv->sh) {
                if (priv->q_counters != NULL &&
                    strcmp(ctr_name, "out_of_buffer") == 0)
-                       return mlx5_devx_cmd_queue_counter_query(priv->sh->ctx,
-                                                          0, (uint32_t *)stat);
+                       return mlx5_devx_cmd_queue_counter_query
+                                       (priv->q_counters, 0, (uint32_t *)stat);
                MKSTR(path, "%s/ports/%d/hw_counters/%s",
                      priv->sh->ibdev_path,
                      priv->dev_port,