net/mlx5: set dynamic flow metadata in Rx queues

Using a global mbuf dynamic field for metadata incurs some
performance penalty on a datapath. Store this information in
the Rx queue descriptor for a better cache locality.

Fixes: a18ac6113331 ("net/mlx5: add metadata support to Rx datapath")
Cc: stable@dpdk.org
Signed-off-by: Alexander Kozyrev <akozyrev@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 770425c..50349ab 100644 (file)
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -753,6 +753,7 @@ int mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
                       bool clear, uint64_t *pkts, uint64_t *bytes);
 int mlx5_flow_dev_dump(struct rte_eth_dev *dev, FILE *file,
                       struct rte_flow_error *error);
+void mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev);
 
 /* mlx5_mp.c */
 int mlx5_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer);

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 84aa069..cba1f23 100644 (file)
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -894,6 +894,35 @@ flow_rxq_flags_clear(struct rte_eth_dev *dev)
        }
 }
 
+/**
+ * Set the Rx queue dynamic metadata (mask and offset) for a flow
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ */
+void
+mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
+{
+       struct mlx5_priv *priv = dev->data->dev_private;
+       struct mlx5_rxq_data *data;
+       unsigned int i;
+
+       for (i = 0; i != priv->rxqs_n; ++i) {
+               if (!(*priv->rxqs)[i])
+                       continue;
+               data = (*priv->rxqs)[i];
+               if (!rte_flow_dynf_metadata_avail()) {
+                       data->dynf_meta = 0;
+                       data->flow_meta_mask = 0;
+                       data->flow_meta_offset = -1;
+               } else {
+                       data->dynf_meta = 1;
+                       data->flow_meta_mask = rte_flow_dynf_metadata_mask;
+                       data->flow_meta_offset = rte_flow_dynf_metadata_offs;
+               }
+       }
+}
+
 /*
  * return a pointer to the desired action in the list of actions.
  *

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 3e583d4..0010b42 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -1339,9 +1339,10 @@ rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
                        pkt->hash.fdir.hi = mlx5_flow_mark_get(mark);
                }
        }
-       if (rte_flow_dynf_metadata_avail() && cqe->flow_table_metadata) {
-               pkt->ol_flags |= PKT_RX_DYNF_METADATA;
-               *RTE_FLOW_DYNF_METADATA(pkt) = cqe->flow_table_metadata;
+       if (rxq->dynf_meta && cqe->flow_table_metadata) {
+               pkt->ol_flags |= rxq->flow_meta_mask;
+               *RTE_MBUF_DYNFIELD(pkt, rxq->flow_meta_offset, uint32_t *) =
+                       cqe->flow_table_metadata;
        }
        if (rxq->csum)
                pkt->ol_flags |= rxq_cq_to_ol_flags(cqe);

diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 8695218..48f2b79 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -121,7 +121,7 @@ struct mlx5_rxq_data {
        unsigned int err_state:2; /* enum mlx5_rxq_err_state. */
        unsigned int strd_scatter_en:1; /* Scattered packets from a stride. */
        unsigned int lro:1; /* Enable LRO. */
-       unsigned int :1; /* Remaining bits. */
+       unsigned int dynf_meta:1; /* Dynamic metadata is configured. */
        volatile uint32_t *rq_db;
        volatile uint32_t *cq_db;
        uint16_t port_id;
@@ -159,6 +159,8 @@ struct mlx5_rxq_data {
        /* CQ (UAR) access lock required for 32bit implementations */
 #endif
        uint32_t tunnel; /* Tunnel information. */
+       uint64_t flow_meta_mask;
+       int32_t flow_meta_offset;
 } __rte_cache_aligned;
 
 enum mlx5_rxq_obj_type {

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
index 9778b0b..45ff8e6 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
@@ -264,17 +264,22 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
                        elts[pos + 2]->hash.fdir.hi = flow_tag;
                        elts[pos + 3]->hash.fdir.hi = flow_tag;
                }
-               if (rte_flow_dynf_metadata_avail()) {
-                       const uint32_t meta = *RTE_FLOW_DYNF_METADATA(t_pkt);
+               if (!!rxq->flow_meta_mask) {
+                       int32_t offs = rxq->flow_meta_offset;
+                       const uint32_t meta =
+                               *RTE_MBUF_DYNFIELD(t_pkt, offs, uint32_t *);
 
                        /* Check if title packet has valid metadata. */
                        if (meta) {
-                               MLX5_ASSERT(t_pkt->ol_flags &
-                                           PKT_RX_DYNF_METADATA);
-                               *RTE_FLOW_DYNF_METADATA(elts[pos]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 1]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 2]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 3]) = meta;
+                               MLX5_ASSERT(t_pkt->ol_flags & offs);
+                               *RTE_MBUF_DYNFIELD(elts[pos], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 1], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 2], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 3], offs,
+                                                       uint32_t *) = meta;
                        }
                }
 
@@ -1023,9 +1028,9 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
                        pkts[pos + 3]->timestamp =
                                rte_be_to_cpu_64(cq[pos + p3].timestamp);
                }
-               if (rte_flow_dynf_metadata_avail()) {
-                       uint64_t flag = rte_flow_dynf_metadata_mask;
-                       int offs = rte_flow_dynf_metadata_offs;
+               if (!!rxq->flow_meta_mask) {
+                       uint64_t flag = rxq->flow_meta_mask;
+                       int32_t offs = rxq->flow_meta_offset;
                        uint32_t metadata;
 
                        /* This code is subject for futher optimization. */

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
index 7b6c5db..d39e726 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
@@ -205,17 +205,22 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
                        elts[pos + 2]->hash.fdir.hi = flow_tag;
                        elts[pos + 3]->hash.fdir.hi = flow_tag;
                }
-               if (rte_flow_dynf_metadata_avail()) {
-                       const uint32_t meta = *RTE_FLOW_DYNF_METADATA(t_pkt);
+               if (!!rxq->flow_meta_mask) {
+                       int32_t offs = rxq->flow_meta_offset;
+                       const uint32_t meta =
+                               *RTE_MBUF_DYNFIELD(t_pkt, offs, uint32_t *);
 
                        /* Check if title packet has valid metadata. */
                        if (meta) {
-                               MLX5_ASSERT(t_pkt->ol_flags &
-                                           PKT_RX_DYNF_METADATA);
-                               *RTE_FLOW_DYNF_METADATA(elts[pos]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 1]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 2]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 3]) = meta;
+                               MLX5_ASSERT(t_pkt->ol_flags & offs);
+                               *RTE_MBUF_DYNFIELD(elts[pos], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 1], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 2], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 3], offs,
+                                                       uint32_t *) = meta;
                        }
                }
                pos += MLX5_VPMD_DESCS_PER_LOOP;
@@ -701,28 +706,30 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
                                        container_of(p3, struct mlx5_cqe,
                                                     pkt_info)->timestamp);
                }
-               if (rte_flow_dynf_metadata_avail()) {
+               if (!!rxq->flow_meta_mask) {
                        /* This code is subject for futher optimization. */
-                       *RTE_FLOW_DYNF_METADATA(elts[pos]) =
+                       int32_t offs = rxq->flow_meta_offset;
+
+                       *RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
                                container_of(p0, struct mlx5_cqe,
                                             pkt_info)->flow_table_metadata;
-                       *RTE_FLOW_DYNF_METADATA(elts[pos + 1]) =
+                       *RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
                                container_of(p1, struct mlx5_cqe,
                                             pkt_info)->flow_table_metadata;
-                       *RTE_FLOW_DYNF_METADATA(elts[pos + 2]) =
+                       *RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
                                container_of(p2, struct mlx5_cqe,
                                             pkt_info)->flow_table_metadata;
-                       *RTE_FLOW_DYNF_METADATA(elts[pos + 3]) =
+                       *RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
                                container_of(p3, struct mlx5_cqe,
                                             pkt_info)->flow_table_metadata;
-                       if (*RTE_FLOW_DYNF_METADATA(elts[pos]))
-                               elts[pos]->ol_flags |= PKT_RX_DYNF_METADATA;
-                       if (*RTE_FLOW_DYNF_METADATA(elts[pos + 1]))
-                               elts[pos + 1]->ol_flags |= PKT_RX_DYNF_METADATA;
-                       if (*RTE_FLOW_DYNF_METADATA(elts[pos + 2]))
-                               elts[pos + 2]->ol_flags |= PKT_RX_DYNF_METADATA;
-                       if (*RTE_FLOW_DYNF_METADATA(elts[pos + 3]))
-                               elts[pos + 3]->ol_flags |= PKT_RX_DYNF_METADATA;
+                       if (*RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *))
+                               elts[pos]->ol_flags |= rxq->flow_meta_mask;
+                       if (*RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *))
+                               elts[pos + 1]->ol_flags |= rxq->flow_meta_mask;
+                       if (*RTE_MBUF_DYNFIELD(pkts[pos + 2], offs, uint32_t *))
+                               elts[pos + 2]->ol_flags |= rxq->flow_meta_mask;
+                       if (*RTE_MBUF_DYNFIELD(pkts[pos + 3], offs, uint32_t *))
+                               elts[pos + 3]->ol_flags |= rxq->flow_meta_mask;
                }
 #ifdef MLX5_PMD_SOFT_COUNTERS
                /* Add up received bytes count. */

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index 4b711f0..f110f73 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -192,17 +192,22 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
                        elts[pos + 2]->hash.fdir.hi = flow_tag;
                        elts[pos + 3]->hash.fdir.hi = flow_tag;
                }
-               if (rte_flow_dynf_metadata_avail()) {
-                       const uint32_t meta = *RTE_FLOW_DYNF_METADATA(t_pkt);
+               if (rxq->dynf_meta) {
+                       int32_t offs = rxq->flow_meta_offset;
+                       const uint32_t meta =
+                               *RTE_MBUF_DYNFIELD(t_pkt, offs, uint32_t *);
 
                        /* Check if title packet has valid metadata. */
                        if (meta) {
-                               MLX5_ASSERT(t_pkt->ol_flags &
-                                           PKT_RX_DYNF_METADATA);
-                               *RTE_FLOW_DYNF_METADATA(elts[pos]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 1]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 2]) = meta;
-                               *RTE_FLOW_DYNF_METADATA(elts[pos + 3]) = meta;
+                               MLX5_ASSERT(t_pkt->ol_flags & offs);
+                               *RTE_MBUF_DYNFIELD(elts[pos], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 1], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 2], offs,
+                                                       uint32_t *) = meta;
+                               *RTE_MBUF_DYNFIELD(elts[pos + 3], offs,
+                                                       uint32_t *) = meta;
                        }
                }
                pos += MLX5_VPMD_DESCS_PER_LOOP;
@@ -654,24 +659,26 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
                        pkts[pos + 3]->timestamp =
                                rte_be_to_cpu_64(cq[pos + p3].timestamp);
                }
-               if (rte_flow_dynf_metadata_avail()) {
+               if (rxq->dynf_meta) {
                        /* This code is subject for futher optimization. */
-                       *RTE_FLOW_DYNF_METADATA(pkts[pos]) =
+                       int32_t offs = rxq->flow_meta_offset;
+
+                       *RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
                                cq[pos].flow_table_metadata;
-                       *RTE_FLOW_DYNF_METADATA(pkts[pos + 1]) =
+                       *RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *) =
                                cq[pos + p1].flow_table_metadata;
-                       *RTE_FLOW_DYNF_METADATA(pkts[pos + 2]) =
+                       *RTE_MBUF_DYNFIELD(pkts[pos + 2], offs, uint32_t *) =
                                cq[pos + p2].flow_table_metadata;
-                       *RTE_FLOW_DYNF_METADATA(pkts[pos + 3]) =
+                       *RTE_MBUF_DYNFIELD(pkts[pos + 3], offs, uint32_t *) =
                                cq[pos + p3].flow_table_metadata;
-                       if (*RTE_FLOW_DYNF_METADATA(pkts[pos]))
-                               pkts[pos]->ol_flags |= PKT_RX_DYNF_METADATA;
-                       if (*RTE_FLOW_DYNF_METADATA(pkts[pos + 1]))
-                               pkts[pos + 1]->ol_flags |= PKT_RX_DYNF_METADATA;
-                       if (*RTE_FLOW_DYNF_METADATA(pkts[pos + 2]))
-                               pkts[pos + 2]->ol_flags |= PKT_RX_DYNF_METADATA;
-                       if (*RTE_FLOW_DYNF_METADATA(pkts[pos + 3]))
-                               pkts[pos + 3]->ol_flags |= PKT_RX_DYNF_METADATA;
+                       if (*RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *))
+                               pkts[pos]->ol_flags |= rxq->flow_meta_mask;
+                       if (*RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *))
+                               pkts[pos + 1]->ol_flags |= rxq->flow_meta_mask;
+                       if (*RTE_MBUF_DYNFIELD(pkts[pos + 2], offs, uint32_t *))
+                               pkts[pos + 2]->ol_flags |= rxq->flow_meta_mask;
+                       if (*RTE_MBUF_DYNFIELD(pkts[pos + 3], offs, uint32_t *))
+                               pkts[pos + 3]->ol_flags |= rxq->flow_meta_mask;
                }
 #ifdef MLX5_PMD_SOFT_COUNTERS
                /* Add up received bytes count. */

diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 7596704..feb9154 100644 (file)
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -323,6 +323,8 @@ mlx5_dev_start(struct rte_eth_dev *dev)
                        dev->data->port_id);
                goto error;
        }
+       /* Set a mask and offset of dynamic metadata flows into Rx queues*/
+       mlx5_flow_rxq_dynf_metadata_set(dev);
        /*
         * In non-cached mode, it only needs to start the default mreg copy
         * action and no flow created by application exists anymore.

diff --git a/lib/librte_ethdev/rte_flow.c b/lib/librte_ethdev/rte_flow.c
index 3699edc..1685be5 100644 (file)
--- a/lib/librte_ethdev/rte_flow.c
+++ b/lib/librte_ethdev/rte_flow.c
@@ -19,7 +19,7 @@
 #include "rte_flow.h"
 
 /* Mbuf dynamic field name for metadata. */
-int rte_flow_dynf_metadata_offs = -1;
+int32_t rte_flow_dynf_metadata_offs = -1;
 
 /* Mbuf dynamic field flag bit number for metadata. */
 uint64_t rte_flow_dynf_metadata_mask;

diff --git a/lib/librte_ethdev/rte_flow.h b/lib/librte_ethdev/rte_flow.h
index fab44f6..132b44e 100644 (file)
--- a/lib/librte_ethdev/rte_flow.h
+++ b/lib/librte_ethdev/rte_flow.h
@@ -2653,7 +2653,7 @@ struct rte_flow_action_set_dscp {
 };
 
 /* Mbuf dynamic field offset for metadata. */
-extern int rte_flow_dynf_metadata_offs;
+extern int32_t rte_flow_dynf_metadata_offs;
 
 /* Mbuf dynamic field flag mask for metadata. */
 extern uint64_t rte_flow_dynf_metadata_mask;