+ * @param dev
+ * Pointer to Ethernet device.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
+ *
+ * @return
+ * 1 while a reference on it exists, 0 when freed.
+ */
+static int
+flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
+ struct mlx5_flow_handle *handle)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t idx = handle->dvh.rix_push_vlan;
+ struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
+
+ cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
+ idx);
+ if (!cache_resource)
+ return 0;
+ MLX5_ASSERT(cache_resource->action);
+ DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
+ (void *)cache_resource,
+ __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+ if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+ __ATOMIC_RELAXED) == 0) {
+ claim_zero(mlx5_flow_os_destroy_flow_action
+ (cache_resource->action));
+ ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
+ &priv->sh->push_vlan_action_list, idx,
+ cache_resource, next);
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
+ DRV_LOG(DEBUG, "push vlan action resource %p: removed",
+ (void *)cache_resource);
+ return 0;
+ }
+ return 1;
+}
+
+/**
+ * Release the fate resource.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
+ */
+static void
+flow_dv_fate_resource_release(struct rte_eth_dev *dev,
+ struct mlx5_flow_handle *handle)
+{
+ if (!handle->rix_fate)
+ return;
+ switch (handle->fate_action) {
+ case MLX5_FLOW_FATE_DROP:
+ mlx5_drop_action_destroy(dev);
+ break;
+ case MLX5_FLOW_FATE_QUEUE:
+ mlx5_hrxq_release(dev, handle->rix_hrxq);
+ break;
+ case MLX5_FLOW_FATE_JUMP:
+ flow_dv_jump_tbl_resource_release(dev, handle);
+ break;
+ case MLX5_FLOW_FATE_PORT_ID:
+ flow_dv_port_id_action_resource_release(dev,
+ handle->rix_port_id_action);
+ break;
+ case MLX5_FLOW_FATE_DEFAULT_MISS:
+ flow_dv_default_miss_resource_release(dev);
+ break;
+ default:
+ DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
+ break;
+ }
+ handle->rix_fate = 0;
+}
+
+/**
+ * Release an sample resource.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
+ *
+ * @return
+ * 1 while a reference on it exists, 0 when freed.
+ */
+static int
+flow_dv_sample_resource_release(struct rte_eth_dev *dev,
+ struct mlx5_flow_handle *handle)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t idx = handle->dvh.rix_sample;
+ struct mlx5_flow_dv_sample_resource *cache_resource;
+
+ cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+ idx);
+ if (!cache_resource)
+ return 0;
+ MLX5_ASSERT(cache_resource->verbs_action);
+ DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
+ (void *)cache_resource,
+ __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+ if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+ __ATOMIC_RELAXED) == 0) {
+ if (cache_resource->verbs_action)
+ claim_zero(mlx5_glue->destroy_flow_action
+ (cache_resource->verbs_action));
+ if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
+ if (cache_resource->default_miss)
+ claim_zero(mlx5_glue->destroy_flow_action
+ (cache_resource->default_miss));
+ }
+ if (cache_resource->normal_path_tbl)
+ flow_dv_tbl_resource_release(dev,
+ cache_resource->normal_path_tbl);
+ }
+ if (cache_resource->sample_idx.rix_hrxq &&
+ !mlx5_hrxq_release(dev,
+ cache_resource->sample_idx.rix_hrxq))
+ cache_resource->sample_idx.rix_hrxq = 0;
+ if (cache_resource->sample_idx.rix_tag &&
+ !flow_dv_tag_release(dev,
+ cache_resource->sample_idx.rix_tag))
+ cache_resource->sample_idx.rix_tag = 0;
+ if (cache_resource->sample_idx.cnt) {
+ flow_dv_counter_release(dev,
+ cache_resource->sample_idx.cnt);
+ cache_resource->sample_idx.cnt = 0;
+ }
+ if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
+ ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+ &priv->sh->sample_action_list, idx,
+ cache_resource, next);
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
+ DRV_LOG(DEBUG, "sample resource %p: removed",
+ (void *)cache_resource);
+ return 0;
+ }
+ return 1;
+}
+
+/**
+ * Release an destination array resource.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
+ *
+ * @return
+ * 1 while a reference on it exists, 0 when freed.
+ */
+static int
+flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
+ struct mlx5_flow_handle *handle)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_dv_dest_array_resource *cache_resource;
+ struct mlx5_flow_sub_actions_idx *mdest_act_res;
+ uint32_t idx = handle->dvh.rix_dest_array;
+ uint32_t i = 0;
+
+ cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+ idx);
+ if (!cache_resource)
+ return 0;
+ MLX5_ASSERT(cache_resource->action);
+ DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
+ (void *)cache_resource,
+ __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+ if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+ __ATOMIC_RELAXED) == 0) {
+ if (cache_resource->action)
+ claim_zero(mlx5_glue->destroy_flow_action
+ (cache_resource->action));
+ for (; i < cache_resource->num_of_dest; i++) {
+ mdest_act_res = &cache_resource->sample_idx[i];
+ if (mdest_act_res->rix_hrxq) {
+ mlx5_hrxq_release(dev,
+ mdest_act_res->rix_hrxq);
+ mdest_act_res->rix_hrxq = 0;
+ }
+ if (mdest_act_res->rix_encap_decap) {
+ flow_dv_encap_decap_resource_release(dev,
+ mdest_act_res->rix_encap_decap);
+ mdest_act_res->rix_encap_decap = 0;
+ }
+ if (mdest_act_res->rix_port_id_action) {
+ flow_dv_port_id_action_resource_release(dev,
+ mdest_act_res->rix_port_id_action);
+ mdest_act_res->rix_port_id_action = 0;
+ }
+ if (mdest_act_res->rix_tag) {
+ flow_dv_tag_release(dev,
+ mdest_act_res->rix_tag);
+ mdest_act_res->rix_tag = 0;
+ }
+ }
+ ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+ &priv->sh->dest_array_list, idx,
+ cache_resource, next);
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
+ DRV_LOG(DEBUG, "destination array resource %p: removed",
+ (void *)cache_resource);
+ return 0;
+ }
+ return 1;
+}
+
+/**
+ * Remove the flow from the NIC but keeps it in memory.
+ * Lock free, (mutex should be acquired by caller).
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in, out] flow
+ * Pointer to flow structure.
+ */
+static void
+__flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+ struct mlx5_flow_handle *dh;
+ uint32_t handle_idx;
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ if (!flow)
+ return;
+ handle_idx = flow->dev_handles;
+ while (handle_idx) {
+ dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
+ handle_idx);
+ if (!dh)
+ return;
+ if (dh->drv_flow) {
+ claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
+ dh->drv_flow = NULL;
+ }
+ if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
+ dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
+ dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
+ flow_dv_fate_resource_release(dev, dh);
+ if (dh->vf_vlan.tag && dh->vf_vlan.created)
+ mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
+ handle_idx = dh->next.next;
+ }
+}
+
+/**
+ * Remove the flow from the NIC and the memory.
+ * Lock free, (mutex should be acquired by caller).
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in, out] flow
+ * Pointer to flow structure.
+ */
+static void
+__flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+ struct rte_flow_shared_action *shared;
+ struct mlx5_flow_handle *dev_handle;
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ if (!flow)
+ return;
+ __flow_dv_remove(dev, flow);
+ shared = mlx5_flow_get_shared_rss(flow);
+ if (shared)
+ __atomic_sub_fetch(&shared->refcnt, 1, __ATOMIC_RELAXED);
+ if (flow->counter) {
+ flow_dv_counter_release(dev, flow->counter);
+ flow->counter = 0;
+ }
+ if (flow->meter) {
+ struct mlx5_flow_meter *fm;
+
+ fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
+ flow->meter);
+ if (fm)
+ mlx5_flow_meter_detach(fm);
+ flow->meter = 0;
+ }
+ while (flow->dev_handles) {
+ uint32_t tmp_idx = flow->dev_handles;
+
+ dev_handle = mlx5_ipool_get(priv->sh->ipool
+ [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
+ if (!dev_handle)
+ return;
+ flow->dev_handles = dev_handle->next.next;
+ if (dev_handle->dvh.matcher)
+ flow_dv_matcher_release(dev, dev_handle);
+ if (dev_handle->dvh.rix_sample)
+ flow_dv_sample_resource_release(dev, dev_handle);
+ if (dev_handle->dvh.rix_dest_array)
+ flow_dv_dest_array_resource_release(dev, dev_handle);
+ if (dev_handle->dvh.rix_encap_decap)
+ flow_dv_encap_decap_resource_release(dev,
+ dev_handle->dvh.rix_encap_decap);
+ if (dev_handle->dvh.modify_hdr)
+ flow_dv_modify_hdr_resource_release(dev, dev_handle);
+ if (dev_handle->dvh.rix_push_vlan)
+ flow_dv_push_vlan_action_resource_release(dev,
+ dev_handle);
+ if (dev_handle->dvh.rix_tag)
+ flow_dv_tag_release(dev,
+ dev_handle->dvh.rix_tag);
+ flow_dv_fate_resource_release(dev, dev_handle);
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
+ tmp_idx);
+ }
+}
+
+/**
+ * Release array of hash RX queue objects.
+ * Helper function.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in, out] hrxqs
+ * Array of hash RX queue objects.
+ *
+ * @return
+ * Total number of references to hash RX queue objects in *hrxqs* array
+ * after this operation.
+ */
+static int
+__flow_dv_hrxqs_release(struct rte_eth_dev *dev,
+ uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
+{
+ size_t i;
+ int remaining = 0;
+
+ for (i = 0; i < RTE_DIM(*hrxqs); i++) {
+ int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
+
+ if (!ret)
+ (*hrxqs)[i] = 0;
+ remaining += ret;
+ }
+ return remaining;
+}
+
+/**
+ * Release all hash RX queue objects representing shared RSS action.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in, out] action
+ * Shared RSS action to remove hash RX queue objects from.
+ *
+ * @return
+ * Total number of references to hash RX queue objects stored in *action*
+ * after this operation.
+ * Expected to be 0 if no external references held.
+ */
+static int
+__flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
+ struct mlx5_shared_action_rss *action)
+{
+ return __flow_dv_hrxqs_release(dev, &action->hrxq) +
+ __flow_dv_hrxqs_release(dev, &action->hrxq_tunnel);
+}
+
+/**
+ * Setup shared RSS action.
+ * Prepare set of hash RX queue objects sufficient to handle all valid
+ * hash_fields combinations (see enum ibv_rx_hash_fields).
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in, out] action
+ * Partially initialized shared RSS action.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL. Initialized in case of
+ * error only.
+ *
+ * @return
+ * 0 on success, otherwise negative errno value.
+ */
+static int
+__flow_dv_action_rss_setup(struct rte_eth_dev *dev,
+ struct mlx5_shared_action_rss *action,
+ struct rte_flow_error *error)
+{
+ size_t i;
+ int err;
+
+ for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
+ uint32_t hrxq_idx;
+ uint64_t hash_fields = mlx5_rss_hash_fields[i];
+ int tunnel;
+
+ for (tunnel = 0; tunnel < 2; tunnel++) {
+ hrxq_idx = mlx5_hrxq_new(dev, action->origin.key,
+ MLX5_RSS_HASH_KEY_LEN,
+ hash_fields,
+ action->origin.queue,
+ action->origin.queue_num,
+ tunnel, true);
+ if (!hrxq_idx) {
+ rte_flow_error_set
+ (error, rte_errno,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "cannot get hash queue");
+ goto error_hrxq_new;
+ }
+ err = __flow_dv_action_rss_hrxq_set
+ (action, hash_fields, tunnel, hrxq_idx);
+ MLX5_ASSERT(!err);
+ }
+ }
+ return 0;
+error_hrxq_new:
+ err = rte_errno;
+ __flow_dv_action_rss_hrxqs_release(dev, action);
+ rte_errno = err;
+ return -rte_errno;
+}
+
+/**
+ * Create shared RSS action.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in] conf
+ * Shared action configuration.
+ * @param[in] rss
+ * RSS action specification used to create shared action.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL. Initialized in case of
+ * error only.
+ *
+ * @return
+ * A valid shared action handle in case of success, NULL otherwise and
+ * rte_errno is set.
+ */
+static struct rte_flow_shared_action *
+__flow_dv_action_rss_create(struct rte_eth_dev *dev,
+ const struct rte_flow_shared_action_conf *conf,
+ const struct rte_flow_action_rss *rss,
+ struct rte_flow_error *error)
+{
+ struct rte_flow_shared_action *shared_action = NULL;
+ void *queue = NULL;
+ struct mlx5_shared_action_rss *shared_rss;
+ struct rte_flow_action_rss *origin;
+ const uint8_t *rss_key;
+ uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
+
+ RTE_SET_USED(conf);
+ queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
+ 0, SOCKET_ID_ANY);
+ shared_action = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*shared_action), 0,
+ SOCKET_ID_ANY);
+ if (!shared_action || !queue) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "cannot allocate resource memory");
+ goto error_rss_init;
+ }
+ shared_rss = &shared_action->rss;
+ shared_rss->queue = queue;
+ origin = &shared_rss->origin;
+ origin->func = rss->func;
+ origin->level = rss->level;
+ /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
+ origin->types = !rss->types ? ETH_RSS_IP : rss->types;
+ /* NULL RSS key indicates default RSS key. */
+ rss_key = !rss->key ? rss_hash_default_key : rss->key;
+ memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
+ origin->key = &shared_rss->key[0];
+ origin->key_len = MLX5_RSS_HASH_KEY_LEN;
+ memcpy(shared_rss->queue, rss->queue, queue_size);
+ origin->queue = shared_rss->queue;
+ origin->queue_num = rss->queue_num;
+ if (__flow_dv_action_rss_setup(dev, shared_rss, error))
+ goto error_rss_init;
+ shared_action->type = MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS;
+ return shared_action;
+error_rss_init:
+ mlx5_free(shared_action);
+ mlx5_free(queue);
+ return NULL;
+}
+
+/**
+ * Destroy the shared RSS action.
+ * Release related hash RX queue objects.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in] shared_rss
+ * The shared RSS action object to be removed.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL. Initialized in case of
+ * error only.
+ *
+ * @return
+ * 0 on success, otherwise negative errno value.
+ */
+static int
+__flow_dv_action_rss_release(struct rte_eth_dev *dev,
+ struct mlx5_shared_action_rss *shared_rss,
+ struct rte_flow_error *error)
+{
+ struct rte_flow_shared_action *shared_action = NULL;
+ uint32_t old_refcnt = 1;
+ int remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
+
+ if (remaining) {
+ return rte_flow_error_set(error, ETOOMANYREFS,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ NULL,
+ "shared rss hrxq has references");
+ }
+ shared_action = container_of(shared_rss,
+ struct rte_flow_shared_action, rss);
+ if (!__atomic_compare_exchange_n(&shared_action->refcnt, &old_refcnt,
+ 0, 0,
+ __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+ return rte_flow_error_set(error, ETOOMANYREFS,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ NULL,
+ "shared rss has references");
+ }
+ rte_free(shared_rss->queue);
+ return 0;
+}
+
+/**
+ * Create shared action, lock free,
+ * (mutex should be acquired by caller).
+ * Dispatcher for action type specific call.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in] conf
+ * Shared action configuration.
+ * @param[in] action
+ * Action specification used to create shared action.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL. Initialized in case of
+ * error only.
+ *
+ * @return
+ * A valid shared action handle in case of success, NULL otherwise and
+ * rte_errno is set.
+ */
+static struct rte_flow_shared_action *
+__flow_dv_action_create(struct rte_eth_dev *dev,
+ const struct rte_flow_shared_action_conf *conf,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error)
+{
+ struct rte_flow_shared_action *shared_action = NULL;
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ switch (action->type) {
+ case RTE_FLOW_ACTION_TYPE_RSS:
+ shared_action = __flow_dv_action_rss_create(dev, conf,
+ action->conf,
+ error);
+ break;
+ default:
+ rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
+ NULL, "action type not supported");
+ break;
+ }
+ if (shared_action) {
+ __atomic_add_fetch(&shared_action->refcnt, 1,
+ __ATOMIC_RELAXED);
+ LIST_INSERT_HEAD(&priv->shared_actions, shared_action, next);
+ }
+ return shared_action;
+}
+
+/**
+ * Destroy the shared action.
+ * Release action related resources on the NIC and the memory.
+ * Lock free, (mutex should be acquired by caller).
+ * Dispatcher for action type specific call.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in] action
+ * The shared action object to be removed.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL. Initialized in case of
+ * error only.