From 5f41b66d12cda394c919777ae9b80257e97b5018 Mon Sep 17 00:00:00 2001 From: Yuval Avnery Date: Mon, 20 Jul 2020 06:26:14 +0000 Subject: [PATCH] regex/mlx5: setup fast path Allocated and register input/output buffers and metadata. Signed-off-by: Yuval Avnery Acked-by: Ori Kam --- drivers/common/mlx5/mlx5_prm.h | 36 +++++ drivers/regex/mlx5/Makefile | 1 + drivers/regex/mlx5/meson.build | 1 + drivers/regex/mlx5/mlx5_regex.h | 8 + drivers/regex/mlx5/mlx5_regex_control.c | 2 + drivers/regex/mlx5/mlx5_regex_fastpath.c | 190 +++++++++++++++++++++++ 6 files changed, 238 insertions(+) create mode 100644 drivers/regex/mlx5/mlx5_regex_fastpath.c diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h index f6f4ec0467..bfbc58b799 100644 --- a/drivers/common/mlx5/mlx5_prm.h +++ b/drivers/common/mlx5/mlx5_prm.h @@ -373,6 +373,42 @@ struct mlx5_cqe { uint8_t op_own; }; +/* MMO metadata segment */ + +#define MLX5_OPCODE_MMO 0x2f +#define MLX5_OPC_MOD_MMO_REGEX 0x4 + +struct mlx5_wqe_metadata_seg { + uint32_t mmo_control_31_0; /* mmo_control_63_32 is in ctrl_seg.imm */ + uint32_t lkey; + uint64_t addr; +}; + +struct mlx5_ifc_regexp_mmo_control_bits { + uint8_t reserved_at_31[0x2]; + uint8_t le[0x1]; + uint8_t reserved_at_28[0x1]; + uint8_t subset_id_0[0xc]; + uint8_t reserved_at_16[0x4]; + uint8_t subset_id_1[0xc]; + uint8_t ctrl[0x4]; + uint8_t subset_id_2[0xc]; + uint8_t reserved_at_16_1[0x4]; + uint8_t subset_id_3[0xc]; +}; + +struct mlx5_ifc_regexp_metadata_bits { + uint8_t rof_version[0x10]; + uint8_t latency_count[0x10]; + uint8_t instruction_count[0x10]; + uint8_t primary_thread_count[0x10]; + uint8_t match_count[0x8]; + uint8_t detected_match_count[0x8]; + uint8_t status[0x10]; + uint8_t job_id[0x20]; + uint8_t reserved[0x80]; +}; + /* Adding direct verbs to data-path. */ /* CQ sequence number mask. */ diff --git a/drivers/regex/mlx5/Makefile b/drivers/regex/mlx5/Makefile index 44bf458e2f..11e3325feb 100644 --- a/drivers/regex/mlx5/Makefile +++ b/drivers/regex/mlx5/Makefile @@ -11,6 +11,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_rxp.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_devx.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_control.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_fastpath.c # Basic CFLAGS. CFLAGS += -O3 diff --git a/drivers/regex/mlx5/meson.build b/drivers/regex/mlx5/meson.build index 6f01df7c5c..7f800f2e3d 100644 --- a/drivers/regex/mlx5/meson.build +++ b/drivers/regex/mlx5/meson.build @@ -14,6 +14,7 @@ sources = files( 'mlx5_rxp.c', 'mlx5_regex_devx.c', 'mlx5_regex_control.c', + 'mlx5_regex_fastpath.c', ) cflags_options = [ '-std=c11', diff --git a/drivers/regex/mlx5/mlx5_regex.h b/drivers/regex/mlx5/mlx5_regex.h index 1cbbb168ae..f1abafc587 100644 --- a/drivers/regex/mlx5/mlx5_regex.h +++ b/drivers/regex/mlx5/mlx5_regex.h @@ -40,6 +40,11 @@ struct mlx5_regex_qp { struct mlx5_regex_sq *sqs; /* Pointer to sq array. */ uint16_t nb_obj; /* Number of sq objects. */ struct mlx5_regex_cq cq; /* CQ struct. */ + uint32_t free_sqs; + struct mlx5_regex_job *jobs; + struct ibv_mr *metadata; + struct ibv_mr *inputs; + struct ibv_mr *outputs; }; struct mlx5_regex_db { @@ -92,4 +97,7 @@ int mlx5_devx_regex_database_program(void *ctx, uint8_t engine, int mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind, const struct rte_regexdev_qp_conf *cfg); +/* mlx5_regex_fastpath.c */ +int mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id); + #endif /* MLX5_REGEX_H */ diff --git a/drivers/regex/mlx5/mlx5_regex_control.c b/drivers/regex/mlx5/mlx5_regex_control.c index 83d390abcb..ec57e24e90 100644 --- a/drivers/regex/mlx5/mlx5_regex_control.c +++ b/drivers/regex/mlx5/mlx5_regex_control.c @@ -363,6 +363,8 @@ mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind, goto error; } } + + mlx5_regexdev_setup_fastpath(priv, qp_ind); return 0; error: diff --git a/drivers/regex/mlx5/mlx5_regex_fastpath.c b/drivers/regex/mlx5/mlx5_regex_fastpath.c new file mode 100644 index 0000000000..3d8f8ccdfa --- /dev/null +++ b/drivers/regex/mlx5/mlx5_regex_fastpath.c @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2020 Mellanox Technologies, Ltd + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "mlx5_regex_utils.h" +#include "mlx5_rxp.h" +#include "mlx5_regex.h" + +#define MLX5_REGEX_METADATA_SIZE 64 +#define MLX5_REGEX_MAX_INPUT (1 << 14) +#define MLX5_REGEX_MAX_OUTPUT (1 << 11) + +#define MLX5_REGEX_WQE_METADATA_OFFSET 16 +#define MLX5_REGEX_WQE_GATHER_OFFSET 32 +#define MLX5_REGEX_WQE_SCATTER_OFFSET 48 + +static inline uint32_t +sq_size_get(struct mlx5_regex_sq *sq) +{ + return (1U << sq->log_nb_desc); +} + +struct mlx5_regex_job { + uint64_t user_id; + uint8_t *input; + volatile uint8_t *output; + volatile uint8_t *metadata; +} __rte_cached_aligned; + +static inline void +set_data_seg(struct mlx5_wqe_data_seg *seg, + uint32_t length, uint32_t lkey, + uintptr_t address) +{ + seg->byte_count = rte_cpu_to_be_32(length); + seg->lkey = rte_cpu_to_be_32(lkey); + seg->addr = rte_cpu_to_be_64(address); +} + +static inline void +set_metadata_seg(struct mlx5_wqe_metadata_seg *seg, + uint32_t mmo_control_31_0, uint32_t lkey, + uintptr_t address) +{ + seg->mmo_control_31_0 = htobe32(mmo_control_31_0); + seg->lkey = rte_cpu_to_be_32(lkey); + seg->addr = rte_cpu_to_be_64(address); +} + +static void +setup_sqs(struct mlx5_regex_qp *queue) +{ + size_t sqid, entry; + uint32_t job_id; + for (sqid = 0; sqid < queue->nb_obj; sqid++) { + struct mlx5_regex_sq *sq = &queue->sqs[sqid]; + uint8_t *wqe = (uint8_t *)sq->wqe; + for (entry = 0 ; entry < sq_size_get(sq); entry++) { + job_id = sqid * sq_size_get(sq) + entry; + struct mlx5_regex_job *job = &queue->jobs[job_id]; + + set_metadata_seg((struct mlx5_wqe_metadata_seg *) + (wqe + MLX5_REGEX_WQE_METADATA_OFFSET), + 0, queue->metadata->lkey, + (uintptr_t)job->metadata); + set_data_seg((struct mlx5_wqe_data_seg *) + (wqe + MLX5_REGEX_WQE_GATHER_OFFSET), + 0, queue->inputs->lkey, + (uintptr_t)job->input); + set_data_seg((struct mlx5_wqe_data_seg *) + (wqe + MLX5_REGEX_WQE_SCATTER_OFFSET), + MLX5_REGEX_MAX_OUTPUT, + queue->outputs->lkey, + (uintptr_t)job->output); + wqe += 64; + } + queue->free_sqs |= 1 << sqid; + } +} + +static int +setup_buffers(struct mlx5_regex_qp *qp, struct ibv_pd *pd) +{ + uint32_t i; + int err; + + void *ptr = rte_calloc(__func__, qp->nb_desc, + MLX5_REGEX_METADATA_SIZE, + MLX5_REGEX_METADATA_SIZE); + if (!ptr) + return -ENOMEM; + + qp->metadata = mlx5_glue->reg_mr(pd, ptr, + MLX5_REGEX_METADATA_SIZE*qp->nb_desc, + IBV_ACCESS_LOCAL_WRITE); + if (!qp->metadata) { + rte_free(ptr); + return -EINVAL; + } + ptr = rte_calloc(__func__, qp->nb_desc, + MLX5_REGEX_MAX_INPUT, + MLX5_REGEX_MAX_INPUT); + + if (!ptr) { + err = -ENOMEM; + goto err_input; + } + qp->inputs = mlx5_glue->reg_mr(pd, ptr, + MLX5_REGEX_MAX_INPUT*qp->nb_desc, + IBV_ACCESS_LOCAL_WRITE); + if (!qp->inputs) { + rte_free(ptr); + err = -EINVAL; + goto err_input; + } + + ptr = rte_calloc(__func__, qp->nb_desc, + MLX5_REGEX_MAX_OUTPUT, + MLX5_REGEX_MAX_OUTPUT); + if (!ptr) { + err = -ENOMEM; + goto err_output; + } + qp->outputs = mlx5_glue->reg_mr(pd, ptr, + MLX5_REGEX_MAX_OUTPUT * qp->nb_desc, + IBV_ACCESS_LOCAL_WRITE); + if (!qp->outputs) { + rte_free(ptr); + err = -EINVAL; + goto err_output; + } + + /* distribute buffers to jobs */ + for (i = 0; i < qp->nb_desc; i++) { + qp->jobs[i].input = + (uint8_t *)qp->inputs->addr + + (i % qp->nb_desc) * MLX5_REGEX_MAX_INPUT; + qp->jobs[i].output = + (uint8_t *)qp->outputs->addr + + (i % qp->nb_desc) * MLX5_REGEX_MAX_OUTPUT; + qp->jobs[i].metadata = + (uint8_t *)qp->metadata->addr + + (i % qp->nb_desc) * MLX5_REGEX_METADATA_SIZE; + } + return 0; + +err_output: + ptr = qp->inputs->addr; + rte_free(ptr); + mlx5_glue->dereg_mr(qp->inputs); +err_input: + ptr = qp->metadata->addr; + rte_free(ptr); + mlx5_glue->dereg_mr(qp->metadata); + return err; +} + +int +mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id) +{ + struct mlx5_regex_qp *qp = &priv->qps[qp_id]; + int err; + + qp->jobs = rte_calloc(__func__, qp->nb_desc, sizeof(*qp->jobs), + sizeof(*qp->jobs)); + if (!qp->jobs) + return -ENOMEM; + err = setup_buffers(qp, priv->pd); + if (err) + return err; + setup_sqs(qp); + return 0; +} -- 2.20.1