From 1ad9a3d09fc222a99af4c734788eaf0bdbafb6ae Mon Sep 17 00:00:00 2001 From: Bing Zhao Date: Tue, 24 Mar 2020 20:59:01 +0800 Subject: [PATCH] net/mlx5: introduce buffer size parameter for hairpin When creating a hairpin queue, the total data size and the maximal number of packets are interrelated. The differ is the stride size. Larger buffer size means big packet like jumbo could be supported, but in the meanwhile, it will introduce more cache misses and have a side effect on the performance. Now a new device parameter "hp_buf_log_sz" is introduced for applications to set the total data buffer size (the logarithm value). Then the maximal number of packets will also be calculated automatically by this value. Applications could also change this value to a larger one in order to support larger packets in hairpin case. A smaller value will be beneficial for memory consumption. If it is not set, the default value will be used. Signed-off-by: Bing Zhao Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.c | 10 ++++++++++ drivers/net/mlx5/mlx5.h | 1 + drivers/net/mlx5/mlx5_defs.h | 2 +- drivers/net/mlx5/mlx5_rxq.c | 17 ++++++++++++++--- drivers/net/mlx5/mlx5_txq.c | 17 ++++++++++++++--- 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 8dda0c313f..6a11b141da 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -150,6 +150,12 @@ /* Configure timeout of LRO session (in microseconds). */ #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" +/* + * Device parameter to configure the total data buffer size for a single + * hairpin queue (logarithm value). + */ +#define MLX5_HP_BUF_SIZE "hp_buf_log_sz" + #ifndef HAVE_IBV_MLX5_MOD_MPW #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2) #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3) @@ -1591,6 +1597,8 @@ mlx5_args_check(const char *key, const char *val, void *opaque) config->lro.timeout = tmp; } else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) { DRV_LOG(DEBUG, "class argument is %s.", val); + } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { + config->log_hp_size = tmp; } else { DRV_LOG(WARNING, "%s: unknown parameter", key); rte_errno = EINVAL; @@ -1643,6 +1651,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) MLX5_MAX_DUMP_FILES_NUM, MLX5_LRO_TIMEOUT_USEC, MLX5_CLASS_ARG_NAME, + MLX5_HP_BUF_SIZE, NULL, }; struct rte_kvargs *kvlist; @@ -3358,6 +3367,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, }, .dv_esw_en = 1, .dv_flow_en = 1, + .log_hp_size = MLX5_ARG_UNSET, }; /* Device specific configuration. */ switch (pci_dev->id.device_id) { diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 63d23cb1e6..ca6a802211 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -191,6 +191,7 @@ struct mlx5_dev_config { unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */ unsigned int ind_table_max_size; /* Maximum indirection table size. */ unsigned int max_dump_files_num; /* Maximum dump files per queue. */ + unsigned int log_hp_size; /* Single hairpin queue data size in total. */ int txqs_inline; /* Queue number threshold for inlining. */ int txq_inline_min; /* Minimal amount of data bytes to inline. */ int txq_inline_max; /* Max packet size for inlining with SEND. */ diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h index 83ca367dba..19e8253d57 100644 --- a/drivers/net/mlx5/mlx5_defs.h +++ b/drivers/net/mlx5/mlx5_defs.h @@ -175,7 +175,7 @@ /* Hairpin TX/RX queue configuration parameters. */ #define MLX5_HAIRPIN_QUEUE_STRIDE 6 -#define MLX5_HAIRPIN_JUMBO_LOG_SIZE (15 + 2) +#define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2) /* Definition of static_assert found in /usr/include/assert.h */ #ifndef HAVE_STATIC_ASSERT diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 97ce206275..0a95e3c753 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -1286,9 +1286,20 @@ mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) attr.hairpin = 1; max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; /* Jumbo frames > 9KB should be supported, and more packets. */ - attr.wq_attr.log_hairpin_data_sz = - (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? - max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; + if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { + if (priv->config.log_hp_size > max_wq_data) { + DRV_LOG(ERR, "total data size %u power of 2 is " + "too large for hairpin", + priv->config.log_hp_size); + rte_errno = ERANGE; + return NULL; + } + attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; + } else { + attr.wq_attr.log_hairpin_data_sz = + (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? + max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; + } /* Set the packets number to the maximum value for performance. */ attr.wq_attr.log_hairpin_num_packets = attr.wq_attr.log_hairpin_data_sz - diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 57bc116450..0653f4cf30 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -512,9 +512,20 @@ mlx5_txq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) attr.tis_lst_sz = 1; max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; /* Jumbo frames > 9KB should be supported, and more packets. */ - attr.wq_attr.log_hairpin_data_sz = - (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? - max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; + if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { + if (priv->config.log_hp_size > max_wq_data) { + DRV_LOG(ERR, "total data size %u power of 2 is " + "too large for hairpin", + priv->config.log_hp_size); + rte_errno = ERANGE; + return NULL; + } + attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; + } else { + attr.wq_attr.log_hairpin_data_sz = + (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? + max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; + } /* Set the packets number to the maximum value for performance. */ attr.wq_attr.log_hairpin_num_packets = attr.wq_attr.log_hairpin_data_sz - -- 2.20.1