X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_txpp.c;h=37355fafbd34711c147938271e4dc7c3405c833e;hb=872ba77bf7a20ee8e321f67463ab38ca4c429bb2;hp=cdb00792d04f7ab104615328c61d1399053979da;hpb=3b025c0ca425a634b7eead08420fa5a5cfaa1445;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_txpp.c b/drivers/net/mlx5/mlx5_txpp.c index cdb00792d0..37355fafbd 100644 --- a/drivers/net/mlx5/mlx5_txpp.c +++ b/drivers/net/mlx5/mlx5_txpp.c @@ -10,6 +10,9 @@ #include #include #include +#include + +#include #include "mlx5.h" #include "mlx5_rxtx.h" @@ -28,36 +31,24 @@ static const char * const mlx5_txpp_stat_names[] = { /* Destroy Event Queue Notification Channel. */ static void -mlx5_txpp_destroy_eqn(struct mlx5_dev_ctx_shared *sh) +mlx5_txpp_destroy_event_channel(struct mlx5_dev_ctx_shared *sh) { if (sh->txpp.echan) { mlx5_glue->devx_destroy_event_channel(sh->txpp.echan); sh->txpp.echan = NULL; } - sh->txpp.eqn = 0; } /* Create Event Queue Notification Channel. */ static int -mlx5_txpp_create_eqn(struct mlx5_dev_ctx_shared *sh) +mlx5_txpp_create_event_channel(struct mlx5_dev_ctx_shared *sh) { - uint32_t lcore; - MLX5_ASSERT(!sh->txpp.echan); - lcore = (uint32_t)rte_lcore_to_cpu_id(-1); - if (mlx5_glue->devx_query_eqn(sh->ctx, lcore, &sh->txpp.eqn)) { - rte_errno = errno; - DRV_LOG(ERR, "Failed to query EQ number %d.", rte_errno); - sh->txpp.eqn = 0; - return -rte_errno; - } sh->txpp.echan = mlx5_glue->devx_create_event_channel(sh->ctx, MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA); if (!sh->txpp.echan) { - sh->txpp.eqn = 0; rte_errno = errno; - DRV_LOG(ERR, "Failed to create event channel %d.", - rte_errno); + DRV_LOG(ERR, "Failed to create event channel %d.", rte_errno); return -rte_errno; } return 0; @@ -110,13 +101,13 @@ mlx5_txpp_alloc_pp_index(struct mlx5_dev_ctx_shared *sh) rte_errno = errno; return -errno; } - if (!sh->txpp.pp->index) { + if (!((struct mlx5dv_pp *)sh->txpp.pp)->index) { DRV_LOG(ERR, "Zero packet pacing index allocated."); mlx5_txpp_free_pp_index(sh); rte_errno = ENOTSUP; return -ENOTSUP; } - sh->txpp.pp_id = sh->txpp.pp->index; + sh->txpp.pp_id = ((struct mlx5dv_pp *)(sh->txpp.pp))->index; return 0; #else RTE_SET_USED(sh); @@ -134,13 +125,13 @@ mlx5_txpp_destroy_send_queue(struct mlx5_txpp_wq *wq) if (wq->sq_umem) claim_zero(mlx5_glue->devx_umem_dereg(wq->sq_umem)); if (wq->sq_buf) - rte_free((void *)(uintptr_t)wq->sq_buf); + mlx5_free((void *)(uintptr_t)wq->sq_buf); if (wq->cq) claim_zero(mlx5_devx_cmd_destroy(wq->cq)); if (wq->cq_umem) claim_zero(mlx5_glue->devx_umem_dereg(wq->cq_umem)); if (wq->cq_buf) - rte_free((void *)(uintptr_t)wq->cq_buf); + mlx5_free((void *)(uintptr_t)wq->cq_buf); memset(wq, 0, sizeof(*wq)); } @@ -159,7 +150,7 @@ mlx5_txpp_destroy_clock_queue(struct mlx5_dev_ctx_shared *sh) mlx5_txpp_destroy_send_queue(wq); if (sh->txpp.tsa) { - rte_free(sh->txpp.tsa); + mlx5_free(sh->txpp.tsa); sh->txpp.tsa = NULL; } } @@ -172,6 +163,7 @@ mlx5_txpp_doorbell_rearm_queue(struct mlx5_dev_ctx_shared *sh, uint16_t ci) uint32_t w32[2]; uint64_t w64; } cs; + void *reg_addr; wq->sq_ci = ci + 1; cs.w32[0] = rte_cpu_to_be_32(rte_be_to_cpu_32 @@ -183,7 +175,8 @@ mlx5_txpp_doorbell_rearm_queue(struct mlx5_dev_ctx_shared *sh, uint16_t ci) /* Make sure the doorbell record is updated. */ rte_wmb(); /* Write to doorbel register to start processing. */ - __mlx5_uar_write64_relaxed(cs.w64, sh->tx_uar->reg_addr, NULL); + reg_addr = mlx5_os_get_devx_uar_reg_addr(sh->tx_uar); + __mlx5_uar_write64_relaxed(cs.w64, reg_addr, NULL); rte_wmb(); } @@ -247,16 +240,21 @@ mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh) struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; struct mlx5_devx_cq_attr cq_attr = { 0 }; struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; - size_t page_size = sysconf(_SC_PAGESIZE); + size_t page_size; uint32_t umem_size, umem_dbrec; int ret; + page_size = rte_mem_page_size(); + if (page_size == (size_t)-1) { + DRV_LOG(ERR, "Failed to get mem page size"); + return -ENOMEM; + } /* Allocate memory buffer for CQEs and doorbell record. */ umem_size = sizeof(struct mlx5_cqe) * MLX5_TXPP_REARM_CQ_SIZE; umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); umem_size += MLX5_DBR_SIZE; - wq->cq_buf = rte_zmalloc_socket(__func__, umem_size, - page_size, sh->numa_node); + wq->cq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, + page_size, sh->numa_node); if (!wq->cq_buf) { DRV_LOG(ERR, "Failed to allocate memory for Rearm Queue."); return -ENOMEM; @@ -274,8 +272,8 @@ mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh) /* Create completion queue object for Rearm Queue. */ cq_attr.cqe_size = (sizeof(struct mlx5_cqe) == 128) ? MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B; - cq_attr.uar_page_id = sh->tx_uar->page_id; - cq_attr.eqn = sh->txpp.eqn; + cq_attr.uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar); + cq_attr.eqn = sh->eqn; cq_attr.q_umem_valid = 1; cq_attr.q_umem_offset = 0; cq_attr.q_umem_id = mlx5_os_get_umem_id(wq->cq_umem); @@ -304,8 +302,8 @@ mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh) umem_size = MLX5_WQE_SIZE * wq->sq_size; umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); umem_size += MLX5_DBR_SIZE; - wq->sq_buf = rte_zmalloc_socket(__func__, umem_size, - page_size, sh->numa_node); + wq->sq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, + page_size, sh->numa_node); if (!wq->sq_buf) { DRV_LOG(ERR, "Failed to allocate memory for Rearm Queue."); rte_errno = ENOMEM; @@ -327,7 +325,7 @@ mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh) sq_attr.tis_num = sh->tis->id; sq_attr.cqn = wq->cq->id; sq_attr.cd_master = 1; - sq_attr.wq_attr.uar_page = sh->tx_uar->page_id; + sq_attr.wq_attr.uar_page = mlx5_os_get_devx_uar_page_id(sh->tx_uar); sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; sq_attr.wq_attr.pd = sh->pdn; sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); @@ -470,14 +468,19 @@ mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; struct mlx5_devx_cq_attr cq_attr = { 0 }; struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; - size_t page_size = sysconf(_SC_PAGESIZE); + size_t page_size; uint32_t umem_size, umem_dbrec; int ret; - sh->txpp.tsa = rte_zmalloc_socket(__func__, - MLX5_TXPP_REARM_SQ_SIZE * - sizeof(struct mlx5_txpp_ts), - 0, sh->numa_node); + page_size = rte_mem_page_size(); + if (page_size == (size_t)-1) { + DRV_LOG(ERR, "Failed to get mem page size"); + return -ENOMEM; + } + sh->txpp.tsa = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, + MLX5_TXPP_REARM_SQ_SIZE * + sizeof(struct mlx5_txpp_ts), + 0, sh->numa_node); if (!sh->txpp.tsa) { DRV_LOG(ERR, "Failed to allocate memory for CQ stats."); return -ENOMEM; @@ -488,7 +491,7 @@ mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) umem_size = sizeof(struct mlx5_cqe) * MLX5_TXPP_CLKQ_SIZE; umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); umem_size += MLX5_DBR_SIZE; - wq->cq_buf = rte_zmalloc_socket(__func__, umem_size, + wq->cq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, page_size, sh->numa_node); if (!wq->cq_buf) { DRV_LOG(ERR, "Failed to allocate memory for Clock Queue."); @@ -509,14 +512,14 @@ mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B; cq_attr.use_first_only = 1; cq_attr.overrun_ignore = 1; - cq_attr.uar_page_id = sh->tx_uar->page_id; - cq_attr.eqn = sh->txpp.eqn; + cq_attr.uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar); + cq_attr.eqn = sh->eqn; cq_attr.q_umem_valid = 1; cq_attr.q_umem_offset = 0; - cq_attr.q_umem_id = wq->cq_umem->umem_id; + cq_attr.q_umem_id = mlx5_os_get_umem_id(wq->cq_umem); cq_attr.db_umem_valid = 1; cq_attr.db_umem_offset = umem_dbrec; - cq_attr.db_umem_id = wq->cq_umem->umem_id; + cq_attr.db_umem_id = mlx5_os_get_umem_id(wq->cq_umem); cq_attr.log_cq_size = rte_log2_u32(MLX5_TXPP_CLKQ_SIZE); cq_attr.log_page_size = rte_log2_u32(page_size); wq->cq = mlx5_devx_cmd_create_cq(sh->ctx, &cq_attr); @@ -543,8 +546,8 @@ mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) umem_size = MLX5_WQE_SIZE * wq->sq_size; umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); umem_size += MLX5_DBR_SIZE; - wq->sq_buf = rte_zmalloc_socket(__func__, umem_size, - page_size, sh->numa_node); + wq->sq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, + page_size, sh->numa_node); if (!wq->sq_buf) { DRV_LOG(ERR, "Failed to allocate memory for Clock Queue."); rte_errno = ENOMEM; @@ -574,16 +577,16 @@ mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) sq_attr.cqn = wq->cq->id; sq_attr.packet_pacing_rate_limit_index = sh->txpp.pp_id; sq_attr.wq_attr.cd_slave = 1; - sq_attr.wq_attr.uar_page = sh->tx_uar->page_id; + sq_attr.wq_attr.uar_page = mlx5_os_get_devx_uar_page_id(sh->tx_uar); sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; sq_attr.wq_attr.pd = sh->pdn; sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); sq_attr.wq_attr.log_wq_sz = rte_log2_u32(wq->sq_size); sq_attr.wq_attr.dbr_umem_valid = 1; sq_attr.wq_attr.dbr_addr = umem_dbrec; - sq_attr.wq_attr.dbr_umem_id = wq->sq_umem->umem_id; + sq_attr.wq_attr.dbr_umem_id = mlx5_os_get_umem_id(wq->sq_umem); sq_attr.wq_attr.wq_umem_valid = 1; - sq_attr.wq_attr.wq_umem_id = wq->sq_umem->umem_id; + sq_attr.wq_attr.wq_umem_id = mlx5_os_get_umem_id(wq->sq_umem); /* umem_offset must be zero for static_sq_wq queue. */ sq_attr.wq_attr.wq_umem_offset = 0; wq->sq = mlx5_devx_cmd_create_sq(sh->ctx, &sq_attr); @@ -617,11 +620,14 @@ error: static inline void mlx5_txpp_cq_arm(struct mlx5_dev_ctx_shared *sh) { + void *base_addr; + struct mlx5_txpp_wq *aq = &sh->txpp.rearm_queue; uint32_t arm_sn = aq->arm_sn << MLX5_CQ_SQN_OFFSET; uint32_t db_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | aq->cq_ci; uint64_t db_be = rte_cpu_to_be_64(((uint64_t)db_hi << 32) | aq->cq->id); - uint32_t *addr = RTE_PTR_ADD(sh->tx_uar->base_addr, MLX5_CQ_DOORBELL); + base_addr = mlx5_os_get_devx_uar_base_addr(sh->tx_uar); + uint32_t *addr = RTE_PTR_ADD(base_addr, MLX5_CQ_DOORBELL); rte_compiler_barrier(); aq->cq_dbrec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(db_hi); @@ -868,8 +874,8 @@ static int mlx5_txpp_start_service(struct mlx5_dev_ctx_shared *sh) { uint16_t event_nums[1] = {0}; - int flags; int ret; + int fd; rte_atomic32_set(&sh->txpp.err_miss_int, 0); rte_atomic32_set(&sh->txpp.err_rearm_queue, 0); @@ -877,15 +883,16 @@ mlx5_txpp_start_service(struct mlx5_dev_ctx_shared *sh) rte_atomic32_set(&sh->txpp.err_ts_past, 0); rte_atomic32_set(&sh->txpp.err_ts_future, 0); /* Attach interrupt handler to process Rearm Queue completions. */ - flags = fcntl(sh->txpp.echan->fd, F_GETFL); - ret = fcntl(sh->txpp.echan->fd, F_SETFL, flags | O_NONBLOCK); + fd = mlx5_os_get_devx_channel_fd(sh->txpp.echan); + ret = mlx5_os_set_nonblock_channel_fd(fd); if (ret) { DRV_LOG(ERR, "Failed to change event channel FD."); rte_errno = errno; return -rte_errno; } memset(&sh->txpp.intr_handle, 0, sizeof(sh->txpp.intr_handle)); - sh->txpp.intr_handle.fd = sh->txpp.echan->fd; + fd = mlx5_os_get_devx_channel_fd(sh->txpp.echan); + sh->txpp.intr_handle.fd = fd; sh->txpp.intr_handle.type = RTE_INTR_HANDLE_EXT; if (rte_intr_callback_register(&sh->txpp.intr_handle, mlx5_txpp_interrupt_handler, sh)) { @@ -932,7 +939,7 @@ mlx5_txpp_create(struct mlx5_dev_ctx_shared *sh, struct mlx5_priv *priv) sh->txpp.test = !!(tx_pp < 0); sh->txpp.skew = priv->config.tx_skew; sh->txpp.freq = priv->config.hca_attr.dev_freq_khz; - ret = mlx5_txpp_create_eqn(sh); + ret = mlx5_txpp_create_event_channel(sh); if (ret) goto exit; ret = mlx5_txpp_alloc_pp_index(sh); @@ -953,7 +960,7 @@ exit: mlx5_txpp_destroy_rearm_queue(sh); mlx5_txpp_destroy_clock_queue(sh); mlx5_txpp_free_pp_index(sh); - mlx5_txpp_destroy_eqn(sh); + mlx5_txpp_destroy_event_channel(sh); sh->txpp.tick = 0; sh->txpp.test = 0; sh->txpp.skew = 0; @@ -975,7 +982,7 @@ mlx5_txpp_destroy(struct mlx5_dev_ctx_shared *sh) mlx5_txpp_destroy_rearm_queue(sh); mlx5_txpp_destroy_clock_queue(sh); mlx5_txpp_free_pp_index(sh); - mlx5_txpp_destroy_eqn(sh); + mlx5_txpp_destroy_event_channel(sh); sh->txpp.tick = 0; sh->txpp.test = 0; sh->txpp.skew = 0;