/**
* Free MR resources.
*
+ * @param[in] sh
+ * Pointer to shared device context.
* @param[in] mr
* MR to free.
*/
static void
-mlx5_aso_devx_dereg_mr(struct mlx5_aso_devx_mr *mr)
+mlx5_aso_dereg_mr(struct mlx5_dev_ctx_shared *sh, struct mlx5_pmd_mr *mr)
{
- claim_zero(mlx5_devx_cmd_destroy(mr->mkey));
- if (!mr->is_indirect && mr->umem)
- claim_zero(mlx5_glue->devx_umem_dereg(mr->umem));
- mlx5_free(mr->buf);
+ void *addr = mr->addr;
+
+ sh->share_cache.dereg_mr_cb(mr);
+ mlx5_free(addr);
memset(mr, 0, sizeof(*mr));
}
/**
* Register Memory Region.
*
- * @param[in] ctx
- * Context returned from mlx5 open_device() glue function.
+ * @param[in] sh
+ * Pointer to shared device context.
* @param[in] length
* Size of MR buffer.
* @param[in/out] mr
* Pointer to MR to create.
* @param[in] socket
* Socket to use for allocation.
- * @param[in] pdn
- * Protection Domain number to use.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-mlx5_aso_devx_reg_mr(void *ctx, size_t length, struct mlx5_aso_devx_mr *mr,
- int socket, int pdn)
+mlx5_aso_reg_mr(struct mlx5_dev_ctx_shared *sh, size_t length,
+ struct mlx5_pmd_mr *mr, int socket)
{
- struct mlx5_devx_mkey_attr mkey_attr;
- mr->buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096,
- socket);
- if (!mr->buf) {
- DRV_LOG(ERR, "Failed to create ASO bits mem for MR by Devx.");
+ int ret;
+
+ mr->addr = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096,
+ socket);
+ if (!mr->addr) {
+ DRV_LOG(ERR, "Failed to create ASO bits mem for MR.");
return -1;
}
- mr->umem = mlx5_os_umem_reg(ctx, mr->buf, length,
- IBV_ACCESS_LOCAL_WRITE);
- if (!mr->umem) {
- DRV_LOG(ERR, "Failed to register Umem for MR by Devx.");
- goto error;
- }
- mkey_attr.addr = (uintptr_t)mr->buf;
- mkey_attr.size = length;
- mkey_attr.umem_id = mlx5_os_get_umem_id(mr->umem);
- mkey_attr.pd = pdn;
- mkey_attr.pg_access = 1;
- mkey_attr.klm_array = NULL;
- mkey_attr.klm_num = 0;
- mkey_attr.relaxed_ordering_read = 0;
- mkey_attr.relaxed_ordering_write = 0;
- mr->mkey = mlx5_devx_cmd_mkey_create(ctx, &mkey_attr);
- if (!mr->mkey) {
+ ret = sh->share_cache.reg_mr_cb(sh->pd, mr->addr, length, mr);
+ if (ret) {
DRV_LOG(ERR, "Failed to create direct Mkey.");
- goto error;
+ mlx5_free(mr->addr);
+ return -1;
}
- mr->length = length;
- mr->is_indirect = false;
return 0;
-error:
- if (mr->umem)
- claim_zero(mlx5_glue->devx_umem_dereg(mr->umem));
- mlx5_free(mr->buf);
- return -1;
}
/**
for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
(sizeof(*wqe) >> 4));
- wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.mkey->id);
- addr = (uint64_t)((uint64_t *)sq->mr.buf + i *
+ wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);
+ addr = (uint64_t)((uint64_t *)sq->mr.addr + i *
MLX5_ASO_AGE_ACTIONS_PER_POOL / 64);
wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
volatile struct mlx5_aso_wqe *restrict wqe;
int i;
int size = 1 << sq->log_desc_n;
- uint32_t idx;
/* All the next fields state should stay constant. */
for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
(BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
MLX5_COMP_MODE_OFFSET);
- for (idx = 0; idx < MLX5_ASO_METERS_PER_WQE;
- idx++)
- wqe->aso_dseg.mtrs[idx].v_bo_sc_bbog_mm =
- RTE_BE32((1 << ASO_DSEG_VALID_OFFSET) |
- (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET));
+ }
+}
+
+/*
+ * Initialize Send Queue used for ASO connection tracking.
+ *
+ * @param[in] sq
+ * ASO SQ to initialize.
+ */
+static void
+mlx5_aso_ct_init_sq(struct mlx5_aso_sq *sq)
+{
+ volatile struct mlx5_aso_wqe *restrict wqe;
+ int i;
+ int size = 1 << sq->log_desc_n;
+ uint64_t addr;
+
+ /* All the next fields state should stay constant. */
+ for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
+ wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
+ (sizeof(*wqe) >> 4));
+ /* One unique MR for the query data. */
+ wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);
+ /* Magic number 64 represents the length of a ASO CT obj. */
+ addr = (uint64_t)((uintptr_t)sq->mr.addr + i * 64);
+ wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
+ wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
+ /*
+ * The values of operand_masks are different for modify
+ * and query.
+ * And data_mask may be different for each modification. In
+ * query, it could be zero and ignored.
+ * CQE generation is always needed, in order to decide when
+ * it is available to create the flow or read the data.
+ */
+ wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
+ MLX5_COMP_MODE_OFFSET);
}
}
* Protection Domain number to use.
* @param[in] log_desc_n
* Log of number of descriptors in queue.
+ * @param[in] ts_format
+ * timestamp format supported by the queue.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket,
- void *uar, uint32_t pdn, uint16_t log_desc_n,
- uint32_t ts_format)
+mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, void *uar,
+ uint32_t pdn, uint16_t log_desc_n, uint32_t ts_format)
{
struct mlx5_devx_create_sq_attr attr = {
.user_index = 0xFFFF,
sq->tail = 0;
sq->sqn = sq->sq_obj.sq->id;
sq->uar_addr = mlx5_os_get_devx_uar_reg_addr(uar);
+ rte_spinlock_init(&sq->sqsl);
return 0;
error:
mlx5_aso_destroy_sq(sq);
*
* @param[in] sh
* Pointer to shared device context.
+ * @param[in] aso_opc_mod
+ * Mode of ASO feature.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
int
mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
- enum mlx5_access_aso_opc_mod aso_opc_mod)
+ enum mlx5_access_aso_opc_mod aso_opc_mod)
{
uint32_t sq_desc_n = 1 << MLX5_ASO_QUEUE_LOG_DESC;
switch (aso_opc_mod) {
case ASO_OPC_MOD_FLOW_HIT:
- if (mlx5_aso_devx_reg_mr(sh->ctx,
- (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *
- sq_desc_n, &sh->aso_age_mng->aso_sq.mr, 0, sh->pdn))
+ if (mlx5_aso_reg_mr(sh, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *
+ sq_desc_n, &sh->aso_age_mng->aso_sq.mr, 0))
return -1;
if (mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0,
sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC,
sh->sq_ts_format)) {
- mlx5_aso_devx_dereg_mr(&sh->aso_age_mng->aso_sq.mr);
+ mlx5_aso_dereg_mr(sh, &sh->aso_age_mng->aso_sq.mr);
return -1;
}
mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq);
break;
case ASO_OPC_MOD_POLICER:
- if (mlx5_aso_sq_create(sh->ctx, &sh->mtrmng->sq, 0,
+ if (mlx5_aso_sq_create(sh->ctx, &sh->mtrmng->pools_mng.sq, 0,
sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC,
sh->sq_ts_format))
return -1;
- mlx5_aso_mtr_init_sq(&sh->mtrmng->sq);
+ mlx5_aso_mtr_init_sq(&sh->mtrmng->pools_mng.sq);
+ break;
+ case ASO_OPC_MOD_CONNECTION_TRACKING:
+ /* 64B per object for query. */
+ if (mlx5_aso_reg_mr(sh, 64 * sq_desc_n,
+ &sh->ct_mng->aso_sq.mr, 0))
+ return -1;
+ if (mlx5_aso_sq_create(sh->ctx, &sh->ct_mng->aso_sq, 0,
+ sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC,
+ sh->sq_ts_format)) {
+ mlx5_aso_dereg_mr(sh, &sh->ct_mng->aso_sq.mr);
+ return -1;
+ }
+ mlx5_aso_ct_init_sq(&sh->ct_mng->aso_sq);
break;
default:
DRV_LOG(ERR, "Unknown ASO operation mode");
*
* @param[in] sh
* Pointer to shared device context.
+ * @param[in] aso_opc_mod
+ * Mode of ASO feature.
*/
void
mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh,
- enum mlx5_access_aso_opc_mod aso_opc_mod)
+ enum mlx5_access_aso_opc_mod aso_opc_mod)
{
struct mlx5_aso_sq *sq;
switch (aso_opc_mod) {
case ASO_OPC_MOD_FLOW_HIT:
- mlx5_aso_devx_dereg_mr(&sh->aso_age_mng->aso_sq.mr);
+ mlx5_aso_dereg_mr(sh, &sh->aso_age_mng->aso_sq.mr);
sq = &sh->aso_age_mng->aso_sq;
break;
case ASO_OPC_MOD_POLICER:
- sq = &sh->mtrmng->sq;
+ sq = &sh->mtrmng->pools_mng.sq;
break;
default:
DRV_LOG(ERR, "Unknown ASO operation mode");
uint16_t idx = (sq->tail + i) & mask;
struct mlx5_aso_age_pool *pool = sq->elts[idx].pool;
uint64_t diff = curr - pool->time_of_last_age_check;
- uint64_t *addr = sq->mr.buf;
+ uint64_t *addr = sq->mr.addr;
int j;
addr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64;
{
volatile struct mlx5_aso_wqe *wqe = NULL;
struct mlx5_flow_meter_info *fm = NULL;
+ struct mlx5_flow_meter_profile *fmp;
uint16_t size = 1 << sq->log_desc_n;
uint16_t mask = size - 1;
- uint16_t res = size - (uint16_t)(sq->head - sq->tail);
+ uint16_t res;
uint32_t dseg_idx = 0;
struct mlx5_aso_mtr_pool *pool = NULL;
+ rte_spinlock_lock(&sq->sqsl);
+ res = size - (uint16_t)(sq->head - sq->tail);
if (unlikely(!res)) {
DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
+ rte_spinlock_unlock(&sq->sqsl);
return 0;
}
wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL);
wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = 0;
}
+ fmp = fm->profile;
+ if (fmp->profile.packet_mode)
+ wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm =
+ RTE_BE32((1 << ASO_DSEG_VALID_OFFSET) |
+ (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET) |
+ (MLX5_METER_MODE_PKT << ASO_DSEG_MTR_MODE));
+ else
+ wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm =
+ RTE_BE32((1 << ASO_DSEG_VALID_OFFSET) |
+ (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET));
sq->head++;
sq->pi += 2;/* Each WQE contains 2 WQEBB's. */
rte_io_wmb();
rte_wmb();
*sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */
rte_wmb();
+ rte_spinlock_unlock(&sq->sqsl);
return 1;
}
const unsigned int mask = cq_size - 1;
uint32_t idx;
uint32_t next_idx = cq->cq_ci & mask;
- const uint16_t max = (uint16_t)(sq->head - sq->tail);
+ uint16_t max;
uint16_t n = 0;
int ret;
- if (unlikely(!max))
+ rte_spinlock_lock(&sq->sqsl);
+ max = (uint16_t)(sq->head - sq->tail);
+ if (unlikely(!max)) {
+ rte_spinlock_unlock(&sq->sqsl);
return;
+ }
do {
idx = next_idx;
next_idx = (cq->cq_ci + 1) & mask;
rte_io_wmb();
cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
}
+ rte_spinlock_unlock(&sq->sqsl);
}
/**
mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh,
struct mlx5_aso_mtr *mtr)
{
- struct mlx5_aso_sq *sq = &sh->mtrmng->sq;
+ struct mlx5_aso_sq *sq = &sh->mtrmng->pools_mng.sq;
uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES;
do {
/* Waiting for wqe resource. */
rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
} while (--poll_wqe_times);
- DRV_LOG(ERR, "Fail to send WQE for ASO meter %d",
- mtr->fm.meter_id);
+ DRV_LOG(ERR, "Fail to send WQE for ASO meter offset %d",
+ mtr->offset);
return -1;
}
mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh,
struct mlx5_aso_mtr *mtr)
{
- struct mlx5_aso_sq *sq = &sh->mtrmng->sq;
+ struct mlx5_aso_sq *sq = &sh->mtrmng->pools_mng.sq;
uint32_t poll_cqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES;
if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) ==
/* Waiting for CQE ready. */
rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
} while (--poll_cqe_times);
- DRV_LOG(ERR, "Fail to poll CQE ready for ASO meter %d",
- mtr->fm.meter_id);
+ DRV_LOG(ERR, "Fail to poll CQE ready for ASO meter offset %d",
+ mtr->offset);
return -1;
}
git.droids-corp.org / dpdk.git / blobdiff