* The hash fields that should be used.
*/
uint64_t
-mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow,
+mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
int tunnel __rte_unused, uint64_t layer_types,
uint64_t hash_fields)
{
- struct rte_flow *flow = dev_flow->flow;
#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
- int rss_request_inner = flow->rss.level >= 2;
+ int rss_request_inner = rss_desc->level >= 2;
/* Check RSS hash level for tunnel. */
if (tunnel && rss_request_inner)
return 0;
#endif
/* Check if requested layer matches RSS hash fields. */
- if (!(flow->rss.types & layer_types))
+ if (!(rss_desc->types & layer_types))
return 0;
return hash_fields;
}
*
* @param[in] dev
* Pointer to the Ethernet device structure.
- * @param[in] flow
- * Pointer to flow structure.
* @param[in] dev_handle
* Pointer to device flow handle structure.
*/
static void
-flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow,
+flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
struct mlx5_flow_handle *dev_handle)
{
struct mlx5_priv *priv = dev->data->dev_private;
- const int mark = !!(dev_handle->act_flags &
- (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
+ const int mark = dev_handle->mark;
const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
+ struct mlx5_hrxq *hrxq;
unsigned int i;
- for (i = 0; i != flow->rss.queue_num; ++i) {
- int idx = (*flow->rss.queue)[i];
+ if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
+ return;
+ hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
+ dev_handle->rix_hrxq);
+ if (!hrxq)
+ return;
+ for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
+ int idx = hrxq->ind_table->queues[i];
struct mlx5_rxq_ctrl *rxq_ctrl =
container_of((*priv->rxqs)[idx],
struct mlx5_rxq_ctrl, rxq);
static void
flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t handle_idx;
struct mlx5_flow_handle *dev_handle;
- LIST_FOREACH(dev_handle, &flow->dev_handles, next)
- flow_drv_rxq_flags_set(dev, flow, dev_handle);
+ SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
+ handle_idx, dev_handle, next)
+ flow_drv_rxq_flags_set(dev, dev_handle);
}
/**
*
* @param dev
* Pointer to Ethernet device.
- * @param[in] flow
- * Pointer to flow structure.
* @param[in] dev_handle
* Pointer to the device flow handle structure.
*/
static void
-flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow,
+flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
struct mlx5_flow_handle *dev_handle)
{
struct mlx5_priv *priv = dev->data->dev_private;
- const int mark = !!(dev_handle->act_flags &
- (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
+ const int mark = dev_handle->mark;
const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
+ struct mlx5_hrxq *hrxq;
unsigned int i;
+ if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
+ return;
+ hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
+ dev_handle->rix_hrxq);
+ if (!hrxq)
+ return;
MLX5_ASSERT(dev->data->dev_started);
- for (i = 0; i != flow->rss.queue_num; ++i) {
- int idx = (*flow->rss.queue)[i];
+ for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
+ int idx = hrxq->ind_table->queues[i];
struct mlx5_rxq_ctrl *rxq_ctrl =
container_of((*priv->rxqs)[idx],
struct mlx5_rxq_ctrl, rxq);
static void
flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t handle_idx;
struct mlx5_flow_handle *dev_handle;
- LIST_FOREACH(dev_handle, &flow->dev_handles, next)
- flow_drv_rxq_flags_trim(dev, flow, dev_handle);
+ SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
+ handle_idx, dev_handle, next)
+ flow_drv_rxq_flags_trim(dev, dev_handle);
}
/**
}
}
+/**
+ * Set the Rx queue dynamic metadata (mask and offset) for a flow
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ */
+void
+mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_rxq_data *data;
+ unsigned int i;
+
+ for (i = 0; i != priv->rxqs_n; ++i) {
+ if (!(*priv->rxqs)[i])
+ continue;
+ data = (*priv->rxqs)[i];
+ if (!rte_flow_dynf_metadata_avail()) {
+ data->dynf_meta = 0;
+ data->flow_meta_mask = 0;
+ data->flow_meta_offset = -1;
+ } else {
+ data->dynf_meta = 1;
+ data->flow_meta_mask = rte_flow_dynf_metadata_mask;
+ data->flow_meta_offset = rte_flow_dynf_metadata_offs;
+ }
+ }
+}
+
/*
* return a pointer to the desired action in the list of actions.
*
flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
struct rte_flow *flow)
{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t handle_idx;
struct mlx5_flow_handle *dev_handle;
- LIST_FOREACH(dev_handle, &flow->dev_handles, next)
- if (dev_handle->qrss_id)
- flow_qrss_free_id(dev, dev_handle->qrss_id);
+ SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
+ handle_idx, dev_handle, next)
+ if (dev_handle->split_flow_id)
+ flow_qrss_free_id(dev, dev_handle->split_flow_id);
}
static int
const struct rte_flow_item items[] __rte_unused,
const struct rte_flow_action actions[] __rte_unused,
bool external __rte_unused,
+ int hairpin __rte_unused,
struct rte_flow_error *error)
{
return rte_flow_error_set(error, ENOTSUP,
* Pointer to the list of actions.
* @param[in] external
* This flow rule is created by request external to PMD.
+ * @param[in] hairpin
+ * Number of hairpin TX actions, 0 means classic flow.
* @param[out] error
* Pointer to the error structure.
*
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
- bool external, struct rte_flow_error *error)
+ bool external, int hairpin, struct rte_flow_error *error)
{
const struct mlx5_flow_driver_ops *fops;
enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
fops = flow_get_drv_ops(type);
- return fops->validate(dev, attr, items, actions, external, error);
+ return fops->validate(dev, attr, items, actions, external,
+ hairpin, error);
}
/**
fops->destroy(dev, flow);
}
-/**
- * Validate a flow supported by the NIC.
- *
- * @see rte_flow_validate()
- * @see rte_flow_ops
- */
-int
-mlx5_flow_validate(struct rte_eth_dev *dev,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- struct rte_flow_error *error)
-{
- int ret;
-
- ret = flow_drv_validate(dev, attr, items, actions, true, error);
- if (ret < 0)
- return ret;
- return 0;
-}
-
/**
* Get RSS action from the action list.
*
* help to do the optimization work for source code.
* If no decap actions, use the layers directly.
*/
- if (!(dev_flow->handle->act_flags & MLX5_FLOW_ACTION_DECAP))
+ if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
return dev_flow->handle->layers;
/* Convert L3 layers with decap action. */
if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
}
/* Declare flow create/destroy prototype in advance. */
-static struct rte_flow *
-flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list,
+static uint32_t
+flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
bool external, struct rte_flow_error *error);
static void
-flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
- struct rte_flow *flow);
+flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+ uint32_t flow_idx);
/**
* Add a flow of copying flow metadata registers in RX_CP_TBL.
[3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
};
struct mlx5_flow_mreg_copy_resource *mcp_res;
+ uint32_t idx = 0;
int ret;
/* Fill the register fileds in the flow. */
};
}
/* Build a new entry. */
- mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0);
+ mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
if (!mcp_res) {
rte_errno = ENOMEM;
return NULL;
}
+ mcp_res->idx = idx;
/*
* The copy Flows are not included in any list. There
* ones are referenced from other Flows and can not
* be applied, removed, deleted in ardbitrary order
* by list traversing.
*/
- mcp_res->flow = flow_list_create(dev, NULL, &attr, items,
+ mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
actions, false, error);
- if (!mcp_res->flow)
+ if (!mcp_res->rix_flow)
goto error;
mcp_res->refcnt++;
mcp_res->hlist_ent.key = mark_id;
goto error;
return mcp_res;
error:
- if (mcp_res->flow)
- flow_list_destroy(dev, NULL, mcp_res->flow);
- rte_free(mcp_res);
+ if (mcp_res->rix_flow)
+ flow_list_destroy(dev, NULL, mcp_res->rix_flow);
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
return NULL;
}
flow_mreg_del_copy_action(struct rte_eth_dev *dev,
struct rte_flow *flow)
{
- struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+ struct mlx5_flow_mreg_copy_resource *mcp_res;
struct mlx5_priv *priv = dev->data->dev_private;
+ if (!flow->rix_mreg_copy)
+ return;
+ mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
+ flow->rix_mreg_copy);
if (!mcp_res || !priv->mreg_cp_tbl)
return;
if (flow->copy_applied) {
MLX5_ASSERT(mcp_res->appcnt);
flow->copy_applied = 0;
--mcp_res->appcnt;
- if (!mcp_res->appcnt)
- flow_drv_remove(dev, mcp_res->flow);
+ if (!mcp_res->appcnt) {
+ struct rte_flow *mcp_flow = mlx5_ipool_get
+ (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+ mcp_res->rix_flow);
+
+ if (mcp_flow)
+ flow_drv_remove(dev, mcp_flow);
+ }
}
/*
* We do not check availability of metadata registers here,
*/
if (--mcp_res->refcnt)
return;
- MLX5_ASSERT(mcp_res->flow);
- flow_list_destroy(dev, NULL, mcp_res->flow);
+ MLX5_ASSERT(mcp_res->rix_flow);
+ flow_list_destroy(dev, NULL, mcp_res->rix_flow);
mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
- rte_free(mcp_res);
- flow->mreg_copy = NULL;
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
+ flow->rix_mreg_copy = 0;
}
/**
flow_mreg_start_copy_action(struct rte_eth_dev *dev,
struct rte_flow *flow)
{
- struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+ struct mlx5_flow_mreg_copy_resource *mcp_res;
+ struct mlx5_priv *priv = dev->data->dev_private;
int ret;
- if (!mcp_res || flow->copy_applied)
+ if (!flow->rix_mreg_copy || flow->copy_applied)
+ return 0;
+ mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
+ flow->rix_mreg_copy);
+ if (!mcp_res)
return 0;
if (!mcp_res->appcnt) {
- ret = flow_drv_apply(dev, mcp_res->flow, NULL);
- if (ret)
- return ret;
+ struct rte_flow *mcp_flow = mlx5_ipool_get
+ (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+ mcp_res->rix_flow);
+
+ if (mcp_flow) {
+ ret = flow_drv_apply(dev, mcp_flow, NULL);
+ if (ret)
+ return ret;
+ }
}
++mcp_res->appcnt;
flow->copy_applied = 1;
flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
struct rte_flow *flow)
{
- struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+ struct mlx5_flow_mreg_copy_resource *mcp_res;
+ struct mlx5_priv *priv = dev->data->dev_private;
- if (!mcp_res || !flow->copy_applied)
+ if (!flow->rix_mreg_copy || !flow->copy_applied)
+ return;
+ mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
+ flow->rix_mreg_copy);
+ if (!mcp_res)
return;
MLX5_ASSERT(mcp_res->appcnt);
--mcp_res->appcnt;
flow->copy_applied = 0;
- if (!mcp_res->appcnt)
- flow_drv_remove(dev, mcp_res->flow);
+ if (!mcp_res->appcnt) {
+ struct rte_flow *mcp_flow = mlx5_ipool_get
+ (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+ mcp_res->rix_flow);
+
+ if (mcp_flow)
+ flow_drv_remove(dev, mcp_flow);
+ }
}
/**
MLX5_DEFAULT_COPY_ID);
if (!mcp_res)
return;
- MLX5_ASSERT(mcp_res->flow);
- flow_list_destroy(dev, NULL, mcp_res->flow);
+ MLX5_ASSERT(mcp_res->rix_flow);
+ flow_list_destroy(dev, NULL, mcp_res->rix_flow);
mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
- rte_free(mcp_res);
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
}
/**
(dev, MLX5_FLOW_MARK_DEFAULT, error);
if (!mcp_res)
return -rte_errno;
- flow->mreg_copy = mcp_res;
+ flow->rix_mreg_copy = mcp_res->idx;
if (dev->data->dev_started) {
mcp_res->appcnt++;
flow->copy_applied = 1;
flow_mreg_add_copy_action(dev, mark->id, error);
if (!mcp_res)
return -rte_errno;
- flow->mreg_copy = mcp_res;
+ flow->rix_mreg_copy = mcp_res->idx;
if (dev->data->dev_started) {
mcp_res->appcnt++;
flow->copy_applied = 1;
dev_flow->flow = flow;
dev_flow->external = external;
/* Subflow object was created, we must include one in the list. */
- LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next);
+ SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
+ dev_flow->handle, next);
/*
* If dev_flow is as one of the suffix flow, some actions in suffix
* flow may need some user defined item layer flags.
* reallocation becomes possible (for example, for
* other flows in other threads).
*/
- dev_flow->handle->qrss_id = qrss_id;
+ dev_flow->handle->split_flow_id = qrss_id;
ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
error);
if (ret < 0)
ret = -rte_errno;
goto exit;
}
- dev_flow->handle->mtr_flow_id = mtr_tag_id;
+ dev_flow->handle->split_flow_id = mtr_tag_id;
/* Setting the sfx group atrr. */
sfx_attr.group = sfx_attr.transfer ?
(MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
* Perform verbose error reporting if not NULL.
*
* @return
- * A flow on success, NULL otherwise and rte_errno is set.
+ * A flow index on success, 0 otherwise and rte_errno is set.
*/
-static struct rte_flow *
-flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list,
+static uint32_t
+flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
uint8_t buffer[2048];
} items_tx;
struct rte_flow_expand_rss *buf = &expand_buffer.buf;
+ struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
+ priv->rss_desc)[!!priv->flow_idx];
const struct rte_flow_action *p_actions_rx = actions;
uint32_t i;
- uint32_t flow_size;
- int hairpin_flow = 0;
+ uint32_t idx = 0;
+ int hairpin_flow;
uint32_t hairpin_id = 0;
struct rte_flow_attr attr_tx = { .priority = 0 };
- int ret = flow_drv_validate(dev, attr, items, p_actions_rx, external,
- error);
+ int ret;
- if (ret < 0)
- return NULL;
hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
+ ret = flow_drv_validate(dev, attr, items, p_actions_rx,
+ external, hairpin_flow, error);
+ if (ret < 0)
+ return 0;
if (hairpin_flow > 0) {
if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
rte_errno = EINVAL;
- return NULL;
+ return 0;
}
flow_hairpin_split(dev, actions, actions_rx.actions,
actions_hairpin_tx.actions, items_tx.items,
&hairpin_id);
p_actions_rx = actions_rx.actions;
}
- flow_size = sizeof(struct rte_flow);
- rss = flow_get_rss_action(p_actions_rx);
- if (rss)
- flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t),
- sizeof(void *));
- else
- flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *));
- flow = rte_calloc(__func__, 1, flow_size, 0);
+ flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
if (!flow) {
rte_errno = ENOMEM;
goto error_before_flow;
flow->hairpin_flow_id = hairpin_id;
MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
flow->drv_type < MLX5_FLOW_TYPE_MAX);
- flow->rss.queue = (void *)(flow + 1);
+ memset(rss_desc, 0, sizeof(*rss_desc));
+ rss = flow_get_rss_action(p_actions_rx);
if (rss) {
/*
* The following information is required by
* mlx5_flow_hashfields_adjust() in advance.
*/
- flow->rss.level = rss->level;
+ rss_desc->level = rss->level;
/* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
- flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
+ rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
}
- LIST_INIT(&flow->dev_handles);
+ flow->dev_handles = 0;
if (rss && rss->types) {
unsigned int graph_root;
buf->entries = 1;
buf->entry[0].pattern = (void *)(uintptr_t)items;
}
- /* Reset device flow index to 0. */
- priv->flow_idx = 0;
+ /*
+ * Record the start index when there is a nested call. All sub-flows
+ * need to be translated before another calling.
+ * No need to use ping-pong buffer to save memory here.
+ */
+ if (priv->flow_idx) {
+ MLX5_ASSERT(!priv->flow_nested_idx);
+ priv->flow_nested_idx = priv->flow_idx;
+ }
for (i = 0; i < buf->entries; ++i) {
/*
* The splitter may create multiple dev_flows,
goto error;
dev_flow->flow = flow;
dev_flow->external = 0;
- LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next);
+ SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
+ dev_flow->handle, next);
ret = flow_drv_translate(dev, dev_flow, &attr_tx,
items_tx.items,
actions_hairpin_tx.actions, error);
if (ret)
goto error;
}
- if (dev->data->dev_started) {
+ /*
+ * If the flow is external (from application) OR device is started, then
+ * the flow will be applied immediately.
+ */
+ if (external || dev->data->dev_started) {
ret = flow_drv_apply(dev, flow, error);
if (ret < 0)
goto error;
}
if (list)
- TAILQ_INSERT_TAIL(list, flow, next);
+ ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
+ flow, next);
flow_rxq_flags_set(dev, flow);
- return flow;
-error_before_flow:
- if (hairpin_id)
- mlx5_flow_id_release(priv->sh->flow_id_pool,
- hairpin_id);
- return NULL;
+ /* Nested flow creation index recovery. */
+ priv->flow_idx = priv->flow_nested_idx;
+ if (priv->flow_nested_idx)
+ priv->flow_nested_idx = 0;
+ return idx;
error:
MLX5_ASSERT(flow);
- flow_mreg_del_copy_action(dev, flow);
ret = rte_errno; /* Save rte_errno before cleanup. */
- if (flow->hairpin_flow_id)
- mlx5_flow_id_release(priv->sh->flow_id_pool,
- flow->hairpin_flow_id);
- MLX5_ASSERT(flow);
+ flow_mreg_del_copy_action(dev, flow);
flow_drv_destroy(dev, flow);
- rte_free(flow);
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
rte_errno = ret; /* Restore rte_errno. */
- return NULL;
+error_before_flow:
+ ret = rte_errno;
+ if (hairpin_id)
+ mlx5_flow_id_release(priv->sh->flow_id_pool,
+ hairpin_id);
+ rte_errno = ret;
+ priv->flow_idx = priv->flow_nested_idx;
+ if (priv->flow_nested_idx)
+ priv->flow_nested_idx = 0;
+ return 0;
}
/**
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_error error;
- return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern,
- actions, false, &error);
+ return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
+ &attr, &pattern,
+ actions, false, &error);
+}
+
+/**
+ * Validate a flow supported by the NIC.
+ *
+ * @see rte_flow_validate()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_validate(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ int hairpin_flow;
+
+ hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
+ return flow_drv_validate(dev, attr, items, actions,
+ true, hairpin_flow, error);
}
/**
{
struct mlx5_priv *priv = dev->data->dev_private;
- return flow_list_create(dev, &priv->flows,
- attr, items, actions, true, error);
+ /*
+ * If the device is not started yet, it is not allowed to created a
+ * flow from application. PMD default flows and traffic control flows
+ * are not affected.
+ */
+ if (unlikely(!dev->data->dev_started)) {
+ DRV_LOG(DEBUG, "port %u is not started when "
+ "inserting a flow", dev->data->port_id);
+ rte_flow_error_set(error, ENODEV,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "port not started");
+ return NULL;
+ }
+ return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
+ attr, items, actions, true, error);
}
/**
* @param dev
* Pointer to Ethernet device.
* @param list
- * Pointer to a TAILQ flow list. If this parameter NULL,
- * there is no flow removal from the list.
- * @param[in] flow
- * Flow to destroy.
+ * Pointer to the Indexed flow list. If this parameter NULL,
+ * there is no flow removal from the list. Be noted that as
+ * flow is add to the indexed list, memory of the indexed
+ * list points to maybe changed as flow destroyed.
+ * @param[in] flow_idx
+ * Index of flow to destroy.
*/
static void
-flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
- struct rte_flow *flow)
+flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+ uint32_t flow_idx)
{
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_fdir_flow *priv_fdir_flow = NULL;
+ struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
+ [MLX5_IPOOL_RTE_FLOW], flow_idx);
+ if (!flow)
+ return;
/*
* Update RX queue flags only if port is started, otherwise it is
* already clean.
flow->hairpin_flow_id);
flow_drv_destroy(dev, flow);
if (list)
- TAILQ_REMOVE(list, flow, next);
+ ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
+ flow_idx, flow, next);
flow_mreg_del_copy_action(dev, flow);
- rte_free(flow->fdir);
- rte_free(flow);
+ if (flow->fdir) {
+ LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
+ if (priv_fdir_flow->rix_flow == flow_idx)
+ break;
+ }
+ if (priv_fdir_flow) {
+ LIST_REMOVE(priv_fdir_flow, next);
+ rte_free(priv_fdir_flow->fdir);
+ rte_free(priv_fdir_flow);
+ }
+ }
+ mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
}
/**
* @param dev
* Pointer to Ethernet device.
* @param list
- * Pointer to a TAILQ flow list.
+ * Pointer to the Indexed flow list.
* @param active
* If flushing is called avtively.
*/
void
-mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list,
- bool active)
+mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
{
uint32_t num_flushed = 0;
- while (!TAILQ_EMPTY(list)) {
- struct rte_flow *flow;
-
- flow = TAILQ_FIRST(list);
- flow_list_destroy(dev, list, flow);
+ while (*list) {
+ flow_list_destroy(dev, list, *list);
num_flushed++;
}
if (active) {
* @param dev
* Pointer to Ethernet device.
* @param list
- * Pointer to a TAILQ flow list.
+ * Pointer to the Indexed flow list.
*/
void
-mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list)
{
- struct rte_flow *flow;
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow *flow = NULL;
+ uint32_t idx;
- TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
+ ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
+ flow, next) {
flow_drv_remove(dev, flow);
flow_mreg_stop_copy_action(dev, flow);
}
* @param dev
* Pointer to Ethernet device.
* @param list
- * Pointer to a TAILQ flow list.
+ * Pointer to the Indexed flow list.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
int
-mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list)
{
- struct rte_flow *flow;
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow *flow = NULL;
struct rte_flow_error error;
+ uint32_t idx;
int ret = 0;
/* Make sure default copy action (reg_c[0] -> reg_b) is created. */
if (ret < 0)
return -rte_errno;
/* Apply Flows created by application. */
- TAILQ_FOREACH(flow, list, next) {
+ ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
+ flow, next) {
ret = flow_mreg_start_copy_action(dev, flow);
if (ret < 0)
goto error;
mlx5_flow_stop_default(struct rte_eth_dev *dev)
{
flow_mreg_del_default_copy_action(dev);
+ flow_rxq_flags_clear(dev);
}
/**
{
struct mlx5_priv *priv = dev->data->dev_private;
- if (!priv->inter_flows)
- priv->inter_flows = rte_calloc(__func__, MLX5_NUM_MAX_DEV_FLOWS,
- sizeof(struct mlx5_flow), 0);
+ if (!priv->inter_flows) {
+ priv->inter_flows = rte_calloc(__func__, 1,
+ MLX5_NUM_MAX_DEV_FLOWS *
+ sizeof(struct mlx5_flow) +
+ (sizeof(struct mlx5_flow_rss_desc) +
+ sizeof(uint16_t) * UINT16_MAX) * 2, 0);
+ if (!priv->inter_flows) {
+ DRV_LOG(ERR, "can't allocate intermediate memory.");
+ return;
+ }
+ }
+ priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows)
+ [MLX5_NUM_MAX_DEV_FLOWS];
+ /* Reset the index. */
+ priv->flow_idx = 0;
+ priv->flow_nested_idx = 0;
}
/**
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow *flow;
+ uint32_t idx;
int ret = 0;
- TAILQ_FOREACH(flow, &priv->flows, next) {
+ ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
+ flow, next) {
DRV_LOG(DEBUG, "port %u flow %p still referenced",
dev->data->port_id, (void *)flow);
++ret;
.group = MLX5_HAIRPIN_TX_TABLE,
};
struct rte_flow_action actions[2];
- struct rte_flow *flow;
+ uint32_t flow_idx;
struct rte_flow_error error;
actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
actions[0].conf = &jump;
actions[1].type = RTE_FLOW_ACTION_TYPE_END;
- flow = flow_list_create(dev, &priv->ctrl_flows,
+ flow_idx = flow_list_create(dev, &priv->ctrl_flows,
&attr, items, actions, false, &error);
- if (!flow) {
+ if (!flow_idx) {
DRV_LOG(DEBUG,
"Failed to create ctrl flow: rte_errno(%d),"
" type(%d), message(%s)",
.type = RTE_FLOW_ACTION_TYPE_END,
},
};
- struct rte_flow *flow;
+ uint32_t flow_idx;
struct rte_flow_error error;
unsigned int i;
}
for (i = 0; i != priv->reta_idx_n; ++i)
queue[i] = (*priv->reta_idx)[i];
- flow = flow_list_create(dev, &priv->ctrl_flows,
+ flow_idx = flow_list_create(dev, &priv->ctrl_flows,
&attr, items, actions, false, &error);
- if (!flow)
+ if (!flow_idx)
return -rte_errno;
return 0;
}
{
struct mlx5_priv *priv = dev->data->dev_private;
- flow_list_destroy(dev, &priv->flows, flow);
+ flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
return 0;
}
*/
static int
flow_drv_query(struct rte_eth_dev *dev,
- struct rte_flow *flow,
+ uint32_t flow_idx,
const struct rte_flow_action *actions,
void *data,
struct rte_flow_error *error)
{
+ struct mlx5_priv *priv = dev->data->dev_private;
const struct mlx5_flow_driver_ops *fops;
- enum mlx5_flow_drv_type ftype = flow->drv_type;
+ struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
+ [MLX5_IPOOL_RTE_FLOW],
+ flow_idx);
+ enum mlx5_flow_drv_type ftype;
+ if (!flow) {
+ return rte_flow_error_set(error, ENOENT,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "invalid flow handle");
+ }
+ ftype = flow->drv_type;
MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
fops = flow_get_drv_ops(ftype);
{
int ret;
- ret = flow_drv_query(dev, flow, actions, data, error);
+ ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
+ error);
if (ret < 0)
return ret;
return 0;
* FDIR flow to lookup.
*
* @return
- * Pointer of flow if found, NULL otherwise.
+ * Index of flow if found, 0 otherwise.
*/
-static struct rte_flow *
+static uint32_t
flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct rte_flow *flow = NULL;
+ uint32_t flow_idx = 0;
+ struct mlx5_fdir_flow *priv_fdir_flow = NULL;
MLX5_ASSERT(fdir_flow);
- TAILQ_FOREACH(flow, &priv->flows, next) {
- if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) {
- DRV_LOG(DEBUG, "port %u found FDIR flow %p",
- dev->data->port_id, (void *)flow);
+ LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
+ if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) {
+ DRV_LOG(DEBUG, "port %u found FDIR flow %u",
+ dev->data->port_id, flow_idx);
+ flow_idx = priv_fdir_flow->rix_flow;
break;
}
}
- return flow;
+ return flow_idx;
}
/**
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_fdir *fdir_flow;
struct rte_flow *flow;
+ struct mlx5_fdir_flow *priv_fdir_flow = NULL;
+ uint32_t flow_idx;
int ret;
fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0);
ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
if (ret)
goto error;
- flow = flow_fdir_filter_lookup(dev, fdir_flow);
- if (flow) {
+ flow_idx = flow_fdir_filter_lookup(dev, fdir_flow);
+ if (flow_idx) {
rte_errno = EEXIST;
goto error;
}
- flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
- fdir_flow->items, fdir_flow->actions, true,
- NULL);
+ priv_fdir_flow = rte_zmalloc(__func__, sizeof(struct mlx5_fdir_flow),
+ 0);
+ if (!priv_fdir_flow) {
+ rte_errno = ENOMEM;
+ goto error;
+ }
+ flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
+ fdir_flow->items, fdir_flow->actions, true,
+ NULL);
+ flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
if (!flow)
goto error;
- MLX5_ASSERT(!flow->fdir);
- flow->fdir = fdir_flow;
+ flow->fdir = 1;
+ priv_fdir_flow->fdir = fdir_flow;
+ priv_fdir_flow->rix_flow = flow_idx;
+ LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next);
DRV_LOG(DEBUG, "port %u created FDIR flow %p",
dev->data->port_id, (void *)flow);
return 0;
error:
+ rte_free(priv_fdir_flow);
rte_free(fdir_flow);
return -rte_errno;
}
const struct rte_eth_fdir_filter *fdir_filter)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct rte_flow *flow;
+ uint32_t flow_idx;
struct mlx5_fdir fdir_flow = {
.attr.group = 0,
};
+ struct mlx5_fdir_flow *priv_fdir_flow = NULL;
int ret;
ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
if (ret)
return -rte_errno;
- flow = flow_fdir_filter_lookup(dev, &fdir_flow);
- if (!flow) {
- rte_errno = ENOENT;
- return -rte_errno;
+ LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
+ /* Find the fdir in priv list */
+ if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow))
+ break;
}
- flow_list_destroy(dev, &priv->flows, flow);
- DRV_LOG(DEBUG, "port %u deleted FDIR flow %p",
- dev->data->port_id, (void *)flow);
+ if (!priv_fdir_flow)
+ return 0;
+ LIST_REMOVE(priv_fdir_flow, next);
+ flow_idx = priv_fdir_flow->rix_flow;
+ flow_list_destroy(dev, &priv->flows, flow_idx);
+ rte_free(priv_fdir_flow->fdir);
+ rte_free(priv_fdir_flow);
+ DRV_LOG(DEBUG, "port %u deleted FDIR flow %u",
+ dev->data->port_id, flow_idx);
return 0;
}
flow_fdir_filter_flush(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
-
- mlx5_flow_list_flush(dev, &priv->flows, false);
+ struct mlx5_fdir_flow *priv_fdir_flow = NULL;
+
+ while (!LIST_EMPTY(&priv->fdir_flows)) {
+ priv_fdir_flow = LIST_FIRST(&priv->fdir_flows);
+ LIST_REMOVE(priv_fdir_flow, next);
+ flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow);
+ rte_free(priv_fdir_flow->fdir);
+ rte_free(priv_fdir_flow);
+ }
}
/**
* Pointer to Ethernet device structure.
*
* @return
- * Pointer to allocated counter on success, NULL otherwise.
+ * Index to allocated counter on success, 0 otherwise.
*/
-struct mlx5_flow_counter *
+uint32_t
mlx5_counter_alloc(struct rte_eth_dev *dev)
{
const struct mlx5_flow_driver_ops *fops;
DRV_LOG(ERR,
"port %u counter allocate is not supported.",
dev->data->port_id);
- return NULL;
+ return 0;
}
/**
* @param[in] dev
* Pointer to Ethernet device structure.
* @param[in] cnt
- * Pointer to counter to be free.
+ * Index to counter to be free.
*/
void
-mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt)
+mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
{
const struct mlx5_flow_driver_ops *fops;
struct rte_flow_attr attr = { .transfer = 0 };
* @param[in] dev
* Pointer to Ethernet device structure.
* @param[in] cnt
- * Pointer to counter to query.
+ * Index to counter to query.
* @param[in] clear
* Set to clear counter statistics.
* @param[out] pkts
* 0 on success, a negative errno value otherwise.
*/
int
-mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt,
+mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
bool clear, uint64_t *pkts, uint64_t *bytes)
{
const struct mlx5_flow_driver_ops *fops;
dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read
(&pool->a64_dcs);
offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL;
+ /*
+ * Identify the counters released between query trigger and query
+ * handle more effiecntly. The counter released in this gap period
+ * should wait for a new round of query as the new arrived packets
+ * will not be taken into account.
+ */
+ rte_atomic64_add(&pool->start_query_gen, 1);
ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL -
offset, NULL, NULL,
pool->raw_hw->mem_mng->dm->id,
sh->devx_comp,
(uint64_t)(uintptr_t)pool);
if (ret) {
+ rte_atomic64_sub(&pool->start_query_gen, 1);
DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
" %d", pool->min_dcs->id);
pool->raw_hw = NULL;
struct mlx5_counter_stats_raw *raw_to_free;
if (unlikely(status)) {
+ rte_atomic64_sub(&pool->start_query_gen, 1);
raw_to_free = pool->raw_hw;
} else {
raw_to_free = pool->raw;
rte_spinlock_lock(&pool->sl);
pool->raw = pool->raw_hw;
rte_spinlock_unlock(&pool->sl);
- rte_atomic64_add(&pool->query_gen, 1);
+ MLX5_ASSERT(rte_atomic64_read(&pool->end_query_gen) + 1 ==
+ rte_atomic64_read(&pool->start_query_gen));
+ rte_atomic64_set(&pool->end_query_gen,
+ rte_atomic64_read(&pool->start_query_gen));
/* Be sure the new raw counters data is updated in memory. */
rte_cio_wmb();
}
.type = RTE_FLOW_ACTION_TYPE_END,
},
};
+ uint32_t flow_idx;
struct rte_flow *flow;
struct rte_flow_error error;
if (!config->dv_flow_en)
break;
/* Create internal flow, validation skips copy action. */
- flow = flow_list_create(dev, NULL, &attr, items,
- actions, false, &error);
+ flow_idx = flow_list_create(dev, NULL, &attr, items,
+ actions, false, &error);
+ flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+ flow_idx);
if (!flow)
continue;
if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL))
config->flow_mreg_c[n++] = idx;
- flow_list_destroy(dev, NULL, flow);
+ flow_list_destroy(dev, NULL, flow_idx);
}
for (; n < MLX5_MREG_C_NUM; ++n)
config->flow_mreg_c[n] = REG_NONE;
git.droids-corp.org / dpdk.git / blobdiff