X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_aso.c;h=eb7fc43da34d84b4f43cd4d55398c1408f1c8ffd;hb=80f872ee0222e3936856e43a693168425b1c78ae;hp=bcdad31330738de30c2c4ef8d0129a0b1ffa7c6a;hpb=7cf2d15a399255eca96746948a715f678736eebf;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c index bcdad31330..eb7fc43da3 100644 --- a/drivers/net/mlx5/mlx5_flow_aso.c +++ b/drivers/net/mlx5/mlx5_flow_aso.c @@ -13,50 +13,6 @@ #include "mlx5.h" #include "mlx5_flow.h" -/** - * Destroy Completion Queue used for ASO access. - * - * @param[in] cq - * ASO CQ to destroy. - */ -static void -mlx5_aso_cq_destroy(struct mlx5_aso_cq *cq) -{ - if (cq->cq_obj.cq) - mlx5_devx_cq_destroy(&cq->cq_obj); - memset(cq, 0, sizeof(*cq)); -} - -/** - * Create Completion Queue used for ASO access. - * - * @param[in] ctx - * Context returned from mlx5 open_device() glue function. - * @param[in/out] cq - * Pointer to CQ to create. - * @param[in] log_desc_n - * Log of number of descriptors in queue. - * @param[in] socket - * Socket to use for allocation. - * @param[in] uar_page_id - * UAR page ID to use. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -static int -mlx5_aso_cq_create(void *ctx, struct mlx5_aso_cq *cq, uint16_t log_desc_n, - int socket, int uar_page_id) -{ - struct mlx5_devx_cq_attr attr = { - .uar_page_id = uar_page_id, - }; - - cq->log_desc_n = log_desc_n; - cq->cq_ci = 0; - return mlx5_devx_cq_create(ctx, &cq->cq_obj, log_desc_n, &attr, socket); -} - /** * Free MR resources. * @@ -84,21 +40,18 @@ mlx5_aso_dereg_mr(struct mlx5_common_device *cdev, struct mlx5_pmd_mr *mr) * Size of MR buffer. * @param[in/out] mr * Pointer to MR to create. - * @param[in] socket - * Socket to use for allocation. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_aso_reg_mr(struct mlx5_common_device *cdev, size_t length, - struct mlx5_pmd_mr *mr, int socket) + struct mlx5_pmd_mr *mr) { - int ret; mr->addr = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096, - socket); + SOCKET_ID_ANY); if (!mr->addr) { DRV_LOG(ERR, "Failed to create ASO bits mem for MR."); return -1; @@ -122,7 +75,7 @@ static void mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq) { mlx5_devx_sq_destroy(&sq->sq_obj); - mlx5_aso_cq_destroy(&sq->cq); + mlx5_devx_cq_destroy(&sq->cq.cq_obj); memset(sq, 0, sizeof(*sq)); } @@ -226,35 +179,31 @@ mlx5_aso_ct_init_sq(struct mlx5_aso_sq *sq) /** * Create Send Queue used for ASO access. * - * @param[in] ctx - * Context returned from mlx5 open_device() glue function. + * @param[in] cdev + * Pointer to the mlx5 common device. * @param[in/out] sq * Pointer to SQ to create. - * @param[in] socket - * Socket to use for allocation. * @param[in] uar * User Access Region object. - * @param[in] pdn - * Protection Domain number to use. - * @param[in] log_desc_n - * Log of number of descriptors in queue. - * @param[in] ts_format - * timestamp format supported by the queue. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, void *uar, - uint32_t pdn, uint16_t log_desc_n, uint32_t ts_format) +mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq, + void *uar) { - struct mlx5_devx_create_sq_attr attr = { + struct mlx5_devx_cq_attr cq_attr = { + .uar_page_id = mlx5_os_get_devx_uar_page_id(uar), + }; + struct mlx5_devx_create_sq_attr sq_attr = { .user_index = 0xFFFF, .wq_attr = (struct mlx5_devx_wq_attr){ - .pd = pdn, + .pd = cdev->pdn, .uar_page = mlx5_os_get_devx_uar_page_id(uar), }, - .ts_format = mlx5_ts_format_conv(ts_format), + .ts_format = + mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format), }; struct mlx5_devx_modify_sq_attr modify_attr = { .state = MLX5_SQC_STATE_RDY, @@ -262,14 +211,18 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, void *uar, uint16_t log_wqbb_n; int ret; - if (mlx5_aso_cq_create(ctx, &sq->cq, log_desc_n, socket, - mlx5_os_get_devx_uar_page_id(uar))) + if (mlx5_devx_cq_create(cdev->ctx, &sq->cq.cq_obj, + MLX5_ASO_QUEUE_LOG_DESC, &cq_attr, + SOCKET_ID_ANY)) goto error; - sq->log_desc_n = log_desc_n; - attr.cqn = sq->cq.cq_obj.cq->id; + sq->cq.cq_ci = 0; + sq->cq.log_desc_n = MLX5_ASO_QUEUE_LOG_DESC; + sq->log_desc_n = MLX5_ASO_QUEUE_LOG_DESC; + sq_attr.cqn = sq->cq.cq_obj.cq->id; /* for mlx5_aso_wqe that is twice the size of mlx5_wqe */ - log_wqbb_n = log_desc_n + 1; - ret = mlx5_devx_sq_create(ctx, &sq->sq_obj, log_wqbb_n, &attr, socket); + log_wqbb_n = sq->log_desc_n + 1; + ret = mlx5_devx_sq_create(cdev->ctx, &sq->sq_obj, log_wqbb_n, &sq_attr, + SOCKET_ID_ANY); if (ret) { DRV_LOG(ERR, "Can't create SQ object."); rte_errno = ENOMEM; @@ -285,7 +238,6 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, void *uar, sq->head = 0; sq->tail = 0; sq->sqn = sq->sq_obj.sq->id; - sq->uar_addr = mlx5_os_get_devx_uar_reg_addr(uar); rte_spinlock_init(&sq->sqsl); return 0; error: @@ -314,34 +266,28 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh, switch (aso_opc_mod) { case ASO_OPC_MOD_FLOW_HIT: if (mlx5_aso_reg_mr(cdev, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * - sq_desc_n, &sh->aso_age_mng->aso_sq.mr, 0)) + sq_desc_n, &sh->aso_age_mng->aso_sq.mr)) return -1; - if (mlx5_aso_sq_create(cdev->ctx, &sh->aso_age_mng->aso_sq, 0, - sh->tx_uar, cdev->pdn, - MLX5_ASO_QUEUE_LOG_DESC, - cdev->config.hca_attr.sq_ts_format)) { + if (mlx5_aso_sq_create(cdev, &sh->aso_age_mng->aso_sq, + sh->tx_uar.obj)) { mlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr); return -1; } mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq); break; case ASO_OPC_MOD_POLICER: - if (mlx5_aso_sq_create(cdev->ctx, &sh->mtrmng->pools_mng.sq, 0, - sh->tx_uar, cdev->pdn, - MLX5_ASO_QUEUE_LOG_DESC, - cdev->config.hca_attr.sq_ts_format)) + if (mlx5_aso_sq_create(cdev, &sh->mtrmng->pools_mng.sq, + sh->tx_uar.obj)) return -1; mlx5_aso_mtr_init_sq(&sh->mtrmng->pools_mng.sq); break; case ASO_OPC_MOD_CONNECTION_TRACKING: /* 64B per object for query. */ if (mlx5_aso_reg_mr(cdev, 64 * sq_desc_n, - &sh->ct_mng->aso_sq.mr, 0)) + &sh->ct_mng->aso_sq.mr)) return -1; - if (mlx5_aso_sq_create(cdev->ctx, &sh->ct_mng->aso_sq, 0, - sh->tx_uar, cdev->pdn, - MLX5_ASO_QUEUE_LOG_DESC, - cdev->config.hca_attr.sq_ts_format)) { + if (mlx5_aso_sq_create(cdev, &sh->ct_mng->aso_sq, + sh->tx_uar.obj)) { mlx5_aso_dereg_mr(cdev, &sh->ct_mng->aso_sq.mr); return -1; } @@ -390,8 +336,8 @@ mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, /** * Write a burst of WQEs to ASO SQ. * - * @param[in] mng - * ASO management data, contains the SQ. + * @param[in] sh + * Pointer to shared device context. * @param[in] n * Index of the last valid pool. * @@ -399,8 +345,9 @@ mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, * Number of WQEs in burst. */ static uint16_t -mlx5_aso_sq_enqueue_burst(struct mlx5_aso_age_mng *mng, uint16_t n) +mlx5_aso_sq_enqueue_burst(struct mlx5_dev_ctx_shared *sh, uint16_t n) { + struct mlx5_aso_age_mng *mng = sh->aso_age_mng; volatile struct mlx5_aso_wqe *wqe; struct mlx5_aso_sq *sq = &mng->aso_sq; struct mlx5_aso_age_pool *pool; @@ -439,11 +386,9 @@ mlx5_aso_sq_enqueue_burst(struct mlx5_aso_age_mng *mng, uint16_t n) } while (max); wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << MLX5_COMP_MODE_OFFSET); - rte_io_wmb(); - sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi); - rte_wmb(); - *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH.*/ - rte_wmb(); + mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, + sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], + !sh->tx_uar.dbnc); return sq->elts[start_head & mask].burst_size; } @@ -644,7 +589,7 @@ mlx5_flow_aso_alarm(void *arg) us = US_PER_S; sq->next = 0; } - mlx5_aso_sq_enqueue_burst(sh->aso_age_mng, n); + mlx5_aso_sq_enqueue_burst(sh, n); if (rte_eal_alarm_set(us, mlx5_flow_aso_alarm, sh)) DRV_LOG(ERR, "Cannot reinitialize aso alarm."); } @@ -695,8 +640,9 @@ mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh) } static uint16_t -mlx5_aso_mtr_sq_enqueue_single(struct mlx5_aso_sq *sq, - struct mlx5_aso_mtr *aso_mtr) +mlx5_aso_mtr_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_sq *sq, + struct mlx5_aso_mtr *aso_mtr) { volatile struct mlx5_aso_wqe *wqe = NULL; struct mlx5_flow_meter_info *fm = NULL; @@ -774,11 +720,9 @@ mlx5_aso_mtr_sq_enqueue_single(struct mlx5_aso_sq *sq, */ sq->head++; sq->pi += 2;/* Each WQE contains 2 WQEBB's. */ - rte_io_wmb(); - sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi); - rte_wmb(); - *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */ - rte_wmb(); + mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, + sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], + !sh->tx_uar.dbnc); rte_spinlock_unlock(&sq->sqsl); return 1; } @@ -871,7 +815,7 @@ mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh, do { mlx5_aso_mtr_completion_handle(sq); - if (mlx5_aso_mtr_sq_enqueue_single(sq, mtr)) + if (mlx5_aso_mtr_sq_enqueue_single(sh, sq, mtr)) return 0; /* Waiting for wqe resource. */ rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); @@ -920,8 +864,8 @@ mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh, /* * Post a WQE to the ASO CT SQ to modify the context. * - * @param[in] mng - * Pointer to the CT pools management structure. + * @param[in] sh + * Pointer to shared device context. * @param[in] ct * Pointer to the generic CT structure related to the context. * @param[in] profile @@ -931,12 +875,12 @@ mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh, * 1 on success (WQE number), 0 on failure. */ static uint16_t -mlx5_aso_ct_sq_enqueue_single(struct mlx5_aso_ct_pools_mng *mng, +mlx5_aso_ct_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh, struct mlx5_aso_ct_action *ct, const struct rte_flow_action_conntrack *profile) { volatile struct mlx5_aso_wqe *wqe = NULL; - struct mlx5_aso_sq *sq = &mng->aso_sq; + struct mlx5_aso_sq *sq = &sh->ct_mng->aso_sq; uint16_t size = 1 << sq->log_desc_n; uint16_t mask = size - 1; uint16_t res; @@ -1039,11 +983,9 @@ mlx5_aso_ct_sq_enqueue_single(struct mlx5_aso_ct_pools_mng *mng, profile->reply_dir.max_ack); sq->head++; sq->pi += 2; /* Each WQE contains 2 WQEBB's. */ - rte_io_wmb(); - sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi); - rte_wmb(); - *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */ - rte_wmb(); + mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, + sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], + !sh->tx_uar.dbnc); rte_spinlock_unlock(&sq->sqsl); return 1; } @@ -1084,8 +1026,8 @@ mlx5_aso_ct_status_update(struct mlx5_aso_sq *sq, uint16_t num) /* * Post a WQE to the ASO CT SQ to query the current context. * - * @param[in] mng - * Pointer to the CT pools management structure. + * @param[in] sh + * Pointer to shared device context. * @param[in] ct * Pointer to the generic CT structure related to the context. * @param[in] data @@ -1095,11 +1037,11 @@ mlx5_aso_ct_status_update(struct mlx5_aso_sq *sq, uint16_t num) * 1 on success (WQE number), 0 on failure. */ static int -mlx5_aso_ct_sq_query_single(struct mlx5_aso_ct_pools_mng *mng, +mlx5_aso_ct_sq_query_single(struct mlx5_dev_ctx_shared *sh, struct mlx5_aso_ct_action *ct, char *data) { volatile struct mlx5_aso_wqe *wqe = NULL; - struct mlx5_aso_sq *sq = &mng->aso_sq; + struct mlx5_aso_sq *sq = &sh->ct_mng->aso_sq; uint16_t size = 1 << sq->log_desc_n; uint16_t mask = size - 1; uint16_t res; @@ -1154,11 +1096,9 @@ mlx5_aso_ct_sq_query_single(struct mlx5_aso_ct_pools_mng *mng, * data segment is not used in this case. */ sq->pi += 2; - rte_io_wmb(); - sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi); - rte_wmb(); - *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */ - rte_wmb(); + mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, + sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], + !sh->tx_uar.dbnc); rte_spinlock_unlock(&sq->sqsl); return 1; } @@ -1238,14 +1178,13 @@ mlx5_aso_ct_update_by_wqe(struct mlx5_dev_ctx_shared *sh, struct mlx5_aso_ct_action *ct, const struct rte_flow_action_conntrack *profile) { - struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES; struct mlx5_aso_ct_pool *pool; MLX5_ASSERT(ct); do { - mlx5_aso_ct_completion_handle(mng); - if (mlx5_aso_ct_sq_enqueue_single(mng, ct, profile)) + mlx5_aso_ct_completion_handle(sh->ct_mng); + if (mlx5_aso_ct_sq_enqueue_single(sh, ct, profile)) return 0; /* Waiting for wqe resource. */ rte_delay_us_sleep(10u); @@ -1385,7 +1324,6 @@ mlx5_aso_ct_query_by_wqe(struct mlx5_dev_ctx_shared *sh, struct mlx5_aso_ct_action *ct, struct rte_flow_action_conntrack *profile) { - struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES; struct mlx5_aso_ct_pool *pool; char out_data[64 * 2]; @@ -1393,8 +1331,8 @@ mlx5_aso_ct_query_by_wqe(struct mlx5_dev_ctx_shared *sh, MLX5_ASSERT(ct); do { - mlx5_aso_ct_completion_handle(mng); - ret = mlx5_aso_ct_sq_query_single(mng, ct, out_data); + mlx5_aso_ct_completion_handle(sh->ct_mng); + ret = mlx5_aso_ct_sq_query_single(sh, ct, out_data); if (ret < 0) return ret; else if (ret > 0)