drivers/net/mlx5/mlx5_flow_age.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4 #include <mlx5_prm.h>
   5 #include <rte_malloc.h>
   6 #include <rte_cycles.h>
   7 #include <rte_eal_paging.h>
   8
   9 #include <mlx5_malloc.h>
  10 #include <mlx5_common_os.h>
  11
  12 #include "mlx5.h"
  13 #include "mlx5_flow.h"
  14
  15 /**
  16  * Destroy Completion Queue used for ASO access.
  17  *
  18  * @param[in] cq
  19  *   ASO CQ to destroy.
  20  */
  21 static void
  22 mlx5_aso_cq_destroy(struct mlx5_aso_cq *cq)
  23 {
  24         if (cq->cq)
  25                 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
  26         if (cq->umem_obj)
  27                 claim_zero(mlx5_glue->devx_umem_dereg(cq->umem_obj));
  28         if (cq->umem_buf)
  29                 mlx5_free((void *)(uintptr_t)cq->umem_buf);
  30         memset(cq, 0, sizeof(*cq));
  31 }
  32
  33 /**
  34  * Create Completion Queue used for ASO access.
  35  *
  36  * @param[in] ctx
  37  *   Context returned from mlx5 open_device() glue function.
  38  * @param[in/out] cq
  39  *   Pointer to CQ to create.
  40  * @param[in] log_desc_n
  41  *   Log of number of descriptors in queue.
  42  * @param[in] socket
  43  *   Socket to use for allocation.
  44  * @param[in] uar_page_id
  45  *   UAR page ID to use.
  46  * @param[in] eqn
  47  *   EQ number.
  48  *
  49  * @return
  50  *   0 on success, a negative errno value otherwise and rte_errno is set.
  51  */
  52 static int
  53 mlx5_aso_cq_create(void *ctx, struct mlx5_aso_cq *cq, uint16_t log_desc_n,
  54                    int socket, int uar_page_id, uint32_t eqn)
  55 {
  56         struct mlx5_devx_cq_attr attr = { 0 };
  57         size_t pgsize = rte_mem_page_size();
  58         uint32_t umem_size;
  59         uint16_t cq_size = 1 << log_desc_n;
  60
  61         cq->log_desc_n = log_desc_n;
  62         umem_size = sizeof(struct mlx5_cqe) * cq_size + sizeof(*cq->db_rec) * 2;
  63         cq->umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
  64                                    4096, socket);
  65         if (!cq->umem_buf) {
  66                 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
  67                 rte_errno = ENOMEM;
  68                 return -ENOMEM;
  69         }
  70         cq->umem_obj = mlx5_os_umem_reg(ctx,
  71                                                 (void *)(uintptr_t)cq->umem_buf,
  72                                                 umem_size,
  73                                                 IBV_ACCESS_LOCAL_WRITE);
  74         if (!cq->umem_obj) {
  75                 DRV_LOG(ERR, "Failed to register umem for aso CQ.");
  76                 goto error;
  77         }
  78         attr.q_umem_valid = 1;
  79         attr.db_umem_valid = 1;
  80         attr.use_first_only = 0;
  81         attr.overrun_ignore = 0;
  82         attr.uar_page_id = uar_page_id;
  83         attr.q_umem_id = mlx5_os_get_umem_id(cq->umem_obj);
  84         attr.q_umem_offset = 0;
  85         attr.db_umem_id = attr.q_umem_id;
  86         attr.db_umem_offset = sizeof(struct mlx5_cqe) * cq_size;
  87         attr.eqn = eqn;
  88         attr.log_cq_size = log_desc_n;
  89         attr.log_page_size = rte_log2_u32(pgsize);
  90         cq->cq = mlx5_devx_cmd_create_cq(ctx, &attr);
  91         if (!cq->cq)
  92                 goto error;
  93         cq->db_rec = RTE_PTR_ADD(cq->umem_buf, (uintptr_t)attr.db_umem_offset);
  94         cq->cq_ci = 0;
  95         memset((void *)(uintptr_t)cq->umem_buf, 0xFF, attr.db_umem_offset);
  96         return 0;
  97 error:
  98         mlx5_aso_cq_destroy(cq);
  99         return -1;
 100 }
 101
 102 /**
 103  * Free MR resources.
 104  *
 105  * @param[in] mr
 106  *   MR to free.
 107  */
 108 static void
 109 mlx5_aso_devx_dereg_mr(struct mlx5_aso_devx_mr *mr)
 110 {
 111         claim_zero(mlx5_devx_cmd_destroy(mr->mkey));
 112         if (!mr->is_indirect && mr->umem)
 113                 claim_zero(mlx5_glue->devx_umem_dereg(mr->umem));
 114         mlx5_free(mr->buf);
 115         memset(mr, 0, sizeof(*mr));
 116 }
 117
 118 /**
 119  * Register Memory Region.
 120  *
 121  * @param[in] ctx
 122  *   Context returned from mlx5 open_device() glue function.
 123  * @param[in] length
 124  *   Size of MR buffer.
 125  * @param[in/out] mr
 126  *   Pointer to MR to create.
 127  * @param[in] socket
 128  *   Socket to use for allocation.
 129  * @param[in] pdn
 130  *   Protection Domain number to use.
 131  *
 132  * @return
 133  *   0 on success, a negative errno value otherwise and rte_errno is set.
 134  */
 135 static int
 136 mlx5_aso_devx_reg_mr(void *ctx, size_t length, struct mlx5_aso_devx_mr *mr,
 137                      int socket, int pdn)
 138 {
 139         struct mlx5_devx_mkey_attr mkey_attr;
 140
 141         mr->buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096,
 142                               socket);
 143         if (!mr->buf) {
 144                 DRV_LOG(ERR, "Failed to create ASO bits mem for MR by Devx.");
 145                 return -1;
 146         }
 147         mr->umem = mlx5_os_umem_reg(ctx, mr->buf, length,
 148                                                  IBV_ACCESS_LOCAL_WRITE);
 149         if (!mr->umem) {
 150                 DRV_LOG(ERR, "Failed to register Umem for MR by Devx.");
 151                 goto error;
 152         }
 153         mkey_attr.addr = (uintptr_t)mr->buf;
 154         mkey_attr.size = length;
 155         mkey_attr.umem_id = mlx5_os_get_umem_id(mr->umem);
 156         mkey_attr.pd = pdn;
 157         mkey_attr.pg_access = 1;
 158         mkey_attr.klm_array = NULL;
 159         mkey_attr.klm_num = 0;
 160         mkey_attr.relaxed_ordering_read = 0;
 161         mkey_attr.relaxed_ordering_write = 0;
 162         mr->mkey = mlx5_devx_cmd_mkey_create(ctx, &mkey_attr);
 163         if (!mr->mkey) {
 164                 DRV_LOG(ERR, "Failed to create direct Mkey.");
 165                 goto error;
 166         }
 167         mr->length = length;
 168         mr->is_indirect = false;
 169         return 0;
 170 error:
 171         if (mr->umem)
 172                 claim_zero(mlx5_glue->devx_umem_dereg(mr->umem));
 173         mlx5_free(mr->buf);
 174         return -1;
 175 }
 176
 177 /**
 178  * Destroy Send Queue used for ASO access.
 179  *
 180  * @param[in] sq
 181  *   ASO SQ to destroy.
 182  */
 183 static void
 184 mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq)
 185 {
 186         if (sq->wqe_umem) {
 187                 mlx5_glue->devx_umem_dereg(sq->wqe_umem);
 188                 sq->wqe_umem = NULL;
 189         }
 190         if (sq->umem_buf) {
 191                 mlx5_free((void *)(uintptr_t)sq->umem_buf);
 192                 sq->umem_buf = NULL;
 193         }
 194         if (sq->sq) {
 195                 mlx5_devx_cmd_destroy(sq->sq);
 196                 sq->sq = NULL;
 197         }
 198         if (sq->cq.cq)
 199                 mlx5_aso_cq_destroy(&sq->cq);
 200         mlx5_aso_devx_dereg_mr(&sq->mr);
 201         memset(sq, 0, sizeof(*sq));
 202 }
 203
 204 /**
 205  * Initialize Send Queue used for ASO access.
 206  *
 207  * @param[in] sq
 208  *   ASO SQ to initialize.
 209  */
 210 static void
 211 mlx5_aso_init_sq(struct mlx5_aso_sq *sq)
 212 {
 213         volatile struct mlx5_aso_wqe *restrict wqe;
 214         int i;
 215         int size = 1 << sq->log_desc_n;
 216         uint64_t addr;
 217
 218         /* All the next fields state should stay constant. */
 219         for (i = 0, wqe = &sq->wqes[0]; i < size; ++i, ++wqe) {
 220                 wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
 221                                                           (sizeof(*wqe) >> 4));
 222                 wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.mkey->id);
 223                 addr = (uint64_t)((uint64_t *)sq->mr.buf + i *
 224                                             MLX5_ASO_AGE_ACTIONS_PER_POOL / 64);
 225                 wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
 226                 wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
 227                 wqe->aso_cseg.operand_masks = rte_cpu_to_be_32
 228                         (0u |
 229                          (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
 230                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
 231                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
 232                          (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
 233                 wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX);
 234         }
 235 }
 236
 237 /**
 238  * Create Send Queue used for ASO access.
 239  *
 240  * @param[in] ctx
 241  *   Context returned from mlx5 open_device() glue function.
 242  * @param[in/out] sq
 243  *   Pointer to SQ to create.
 244  * @param[in] socket
 245  *   Socket to use for allocation.
 246  * @param[in] uar
 247  *   User Access Region object.
 248  * @param[in] pdn
 249  *   Protection Domain number to use.
 250  * @param[in] eqn
 251  *   EQ number.
 252  * @param[in] log_desc_n
 253  *   Log of number of descriptors in queue.
 254  *
 255  * @return
 256  *   0 on success, a negative errno value otherwise and rte_errno is set.
 257  */
 258 static int
 259 mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket,
 260                    void *uar, uint32_t pdn,
 261                    uint32_t eqn,  uint16_t log_desc_n)
 262 {
 263         struct mlx5_devx_create_sq_attr attr = { 0 };
 264         struct mlx5_devx_modify_sq_attr modify_attr = { 0 };
 265         size_t pgsize = rte_mem_page_size();
 266         struct mlx5_devx_wq_attr *wq_attr = &attr.wq_attr;
 267         uint32_t sq_desc_n = 1 << log_desc_n;
 268         uint32_t wq_size = sizeof(struct mlx5_aso_wqe) * sq_desc_n;
 269         int ret;
 270
 271         if (mlx5_aso_devx_reg_mr(ctx, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *
 272                                  sq_desc_n, &sq->mr, socket, pdn))
 273                 return -1;
 274         if (mlx5_aso_cq_create(ctx, &sq->cq, log_desc_n, socket,
 275                                 mlx5_os_get_devx_uar_page_id(uar), eqn))
 276                 goto error;
 277         sq->log_desc_n = log_desc_n;
 278         sq->umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size +
 279                                    sizeof(*sq->db_rec) * 2, 4096, socket);
 280         if (!sq->umem_buf) {
 281                 DRV_LOG(ERR, "Can't allocate wqe buffer.");
 282                 return -ENOMEM;
 283         }
 284         sq->wqe_umem = mlx5_os_umem_reg(ctx,
 285                                                 (void *)(uintptr_t)sq->umem_buf,
 286                                                 wq_size +
 287                                                 sizeof(*sq->db_rec) * 2,
 288                                                 IBV_ACCESS_LOCAL_WRITE);
 289         if (!sq->wqe_umem) {
 290                 DRV_LOG(ERR, "Failed to register umem for SQ.");
 291                 rte_errno = ENOMEM;
 292                 goto error;
 293         }
 294         attr.state = MLX5_SQC_STATE_RST;
 295         attr.tis_lst_sz = 0;
 296         attr.tis_num = 0;
 297         attr.user_index = 0xFFFF;
 298         attr.cqn = sq->cq.cq->id;
 299         wq_attr->uar_page = mlx5_os_get_devx_uar_page_id(uar);
 300         wq_attr->pd = pdn;
 301         wq_attr->wq_type = MLX5_WQ_TYPE_CYCLIC;
 302         wq_attr->log_wq_pg_sz = rte_log2_u32(pgsize);
 303         wq_attr->wq_umem_id = mlx5_os_get_umem_id(sq->wqe_umem);
 304         wq_attr->wq_umem_offset = 0;
 305         wq_attr->wq_umem_valid = 1;
 306         wq_attr->log_wq_stride = 6;
 307         wq_attr->log_wq_sz = rte_log2_u32(wq_size) - 6;
 308         wq_attr->dbr_umem_id = wq_attr->wq_umem_id;
 309         wq_attr->dbr_addr = wq_size;
 310         wq_attr->dbr_umem_valid = 1;
 311         sq->sq = mlx5_devx_cmd_create_sq(ctx, &attr);
 312         if (!sq->sq) {
 313                 DRV_LOG(ERR, "Can't create sq object.");
 314                 rte_errno  = ENOMEM;
 315                 goto error;
 316         }
 317         modify_attr.state = MLX5_SQC_STATE_RDY;
 318         ret = mlx5_devx_cmd_modify_sq(sq->sq, &modify_attr);
 319         if (ret) {
 320                 DRV_LOG(ERR, "Can't change sq state to ready.");
 321                 rte_errno  = ENOMEM;
 322                 goto error;
 323         }
 324         sq->pi = 0;
 325         sq->head = 0;
 326         sq->tail = 0;
 327         sq->sqn = sq->sq->id;
 328         sq->db_rec = RTE_PTR_ADD(sq->umem_buf, (uintptr_t)(wq_attr->dbr_addr));
 329         sq->uar_addr = mlx5_os_get_devx_uar_reg_addr(uar);
 330         mlx5_aso_init_sq(sq);
 331         return 0;
 332 error:
 333         mlx5_aso_destroy_sq(sq);
 334         return -1;
 335 }
 336
 337 /**
 338  * API to create and initialize Send Queue used for ASO access.
 339  *
 340  * @param[in] sh
 341  *   Pointer to shared device context.
 342  *
 343  * @return
 344  *   0 on success, a negative errno value otherwise and rte_errno is set.
 345  */
 346 int
 347 mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh)
 348 {
 349         return mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0,
 350                                   sh->tx_uar, sh->pdn, sh->eqn,
 351                                   MLX5_ASO_QUEUE_LOG_DESC);
 352 }
 353
 354 /**
 355  * API to destroy Send Queue used for ASO access.
 356  *
 357  * @param[in] sh
 358  *   Pointer to shared device context.
 359  */
 360 void
 361 mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh)
 362 {
 363         mlx5_aso_destroy_sq(&sh->aso_age_mng->aso_sq);
 364 }
 365
 366 /**
 367  * Write a burst of WQEs to ASO SQ.
 368  *
 369  * @param[in] mng
 370  *   ASO management data, contains the SQ.
 371  * @param[in] n
 372  *   Index of the last valid pool.
 373  *
 374  * @return
 375  *   Number of WQEs in burst.
 376  */
 377 static uint16_t
 378 mlx5_aso_sq_enqueue_burst(struct mlx5_aso_age_mng *mng, uint16_t n)
 379 {
 380         volatile struct mlx5_aso_wqe *wqe;
 381         struct mlx5_aso_sq *sq = &mng->aso_sq;
 382         struct mlx5_aso_age_pool *pool;
 383         uint16_t size = 1 << sq->log_desc_n;
 384         uint16_t mask = size - 1;
 385         uint16_t max;
 386         uint16_t start_head = sq->head;
 387
 388         max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), n - sq->next);
 389         if (unlikely(!max))
 390                 return 0;
 391         sq->elts[start_head & mask].burst_size = max;
 392         do {
 393                 wqe = &sq->wqes[sq->head & mask];
 394                 rte_prefetch0(&sq->wqes[(sq->head + 1) & mask]);
 395                 /* Fill next WQE. */
 396                 rte_spinlock_lock(&mng->resize_sl);
 397                 pool = mng->pools[sq->next];
 398                 rte_spinlock_unlock(&mng->resize_sl);
 399                 sq->elts[sq->head & mask].pool = pool;
 400                 wqe->general_cseg.misc =
 401                                 rte_cpu_to_be_32(((struct mlx5_devx_obj *)
 402                                                  (pool->flow_hit_aso_obj))->id);
 403                 wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 404                                                          MLX5_COMP_MODE_OFFSET);
 405                 wqe->general_cseg.opcode = rte_cpu_to_be_32
 406                                                 (MLX5_OPCODE_ACCESS_ASO |
 407                                                  (ASO_OPC_MOD_FLOW_HIT <<
 408                                                   WQE_CSEG_OPC_MOD_OFFSET) |
 409                                                  (sq->pi <<
 410                                                   WQE_CSEG_WQE_INDEX_OFFSET));
 411                 sq->pi += 2; /* Each WQE contains 2 WQEBB's. */
 412                 sq->head++;
 413                 sq->next++;
 414                 max--;
 415         } while (max);
 416         wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
 417                                                          MLX5_COMP_MODE_OFFSET);
 418         rte_io_wmb();
 419         sq->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi);
 420         rte_wmb();
 421         *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH.*/
 422         rte_wmb();
 423         return sq->elts[start_head & mask].burst_size;
 424 }
 425
 426 /**
 427  * Debug utility function. Dump contents of error CQE and WQE.
 428  *
 429  * @param[in] cqe
 430  *   Error CQE to dump.
 431  * @param[in] wqe
 432  *   Error WQE to dump.
 433  */
 434 static void
 435 mlx5_aso_dump_err_objs(volatile uint32_t *cqe, volatile uint32_t *wqe)
 436 {
 437         int i;
 438
 439         DRV_LOG(ERR, "Error cqe:");
 440         for (i = 0; i < 16; i += 4)
 441                 DRV_LOG(ERR, "%08X %08X %08X %08X", cqe[i], cqe[i + 1],
 442                         cqe[i + 2], cqe[i + 3]);
 443         DRV_LOG(ERR, "\nError wqe:");
 444         for (i = 0; i < (int)sizeof(struct mlx5_aso_wqe) / 4; i += 4)
 445                 DRV_LOG(ERR, "%08X %08X %08X %08X", wqe[i], wqe[i + 1],
 446                         wqe[i + 2], wqe[i + 3]);
 447 }
 448
 449 /**
 450  * Handle case of error CQE.
 451  *
 452  * @param[in] sq
 453  *   ASO SQ to use.
 454  */
 455 static void
 456 mlx5_aso_cqe_err_handle(struct mlx5_aso_sq *sq)
 457 {
 458         struct mlx5_aso_cq *cq = &sq->cq;
 459         uint32_t idx = cq->cq_ci & ((1 << cq->log_desc_n) - 1);
 460         volatile struct mlx5_err_cqe *cqe =
 461                                 (volatile struct mlx5_err_cqe *)&cq->cqes[idx];
 462
 463         cq->errors++;
 464         idx = rte_be_to_cpu_16(cqe->wqe_counter) & (1u << sq->log_desc_n);
 465         mlx5_aso_dump_err_objs((volatile uint32_t *)cqe,
 466                                  (volatile uint32_t *)&sq->wqes[idx]);
 467 }
 468
 469 /**
 470  * Update ASO objects upon completion.
 471  *
 472  * @param[in] sh
 473  *   Shared device context.
 474  * @param[in] n
 475  *   Number of completed ASO objects.
 476  */
 477 static void
 478 mlx5_aso_age_action_update(struct mlx5_dev_ctx_shared *sh, uint16_t n)
 479 {
 480         struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
 481         struct mlx5_aso_sq *sq = &mng->aso_sq;
 482         struct mlx5_age_info *age_info;
 483         const uint16_t size = 1 << sq->log_desc_n;
 484         const uint16_t mask = size - 1;
 485         const uint64_t curr = MLX5_CURR_TIME_SEC;
 486         uint16_t expected = AGE_CANDIDATE;
 487         uint16_t i;
 488
 489         for (i = 0; i < n; ++i) {
 490                 uint16_t idx = (sq->tail + i) & mask;
 491                 struct mlx5_aso_age_pool *pool = sq->elts[idx].pool;
 492                 uint64_t diff = curr - pool->time_of_last_age_check;
 493                 uint64_t *addr = sq->mr.buf;
 494                 int j;
 495
 496                 addr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64;
 497                 pool->time_of_last_age_check = curr;
 498                 for (j = 0; j < MLX5_ASO_AGE_ACTIONS_PER_POOL; j++) {
 499                         struct mlx5_aso_age_action *act = &pool->actions[j];
 500                         struct mlx5_age_param *ap = &act->age_params;
 501                         uint8_t byte;
 502                         uint8_t offset;
 503                         uint8_t *u8addr;
 504                         uint8_t hit;
 505
 506                         if (__atomic_load_n(&ap->state, __ATOMIC_RELAXED) !=
 507                                             AGE_CANDIDATE)
 508                                 continue;
 509                         byte = 63 - (j / 8);
 510                         offset = j % 8;
 511                         u8addr = (uint8_t *)addr;
 512                         hit = (u8addr[byte] >> offset) & 0x1;
 513                         if (hit) {
 514                                 __atomic_store_n(&ap->sec_since_last_hit, 0,
 515                                                  __ATOMIC_RELAXED);
 516                         } else {
 517                                 struct mlx5_priv *priv;
 518
 519                                 __atomic_fetch_add(&ap->sec_since_last_hit,
 520                                                    diff, __ATOMIC_RELAXED);
 521                                 /* If timeout passed add to aged-out list. */
 522                                 if (ap->sec_since_last_hit <= ap->timeout)
 523                                         continue;
 524                                 priv =
 525                                 rte_eth_devices[ap->port_id].data->dev_private;
 526                                 age_info = GET_PORT_AGE_INFO(priv);
 527                                 rte_spinlock_lock(&age_info->aged_sl);
 528                                 if (__atomic_compare_exchange_n(&ap->state,
 529                                                                 &expected,
 530                                                                 AGE_TMOUT,
 531                                                                 false,
 532                                                                __ATOMIC_RELAXED,
 533                                                             __ATOMIC_RELAXED)) {
 534                                         LIST_INSERT_HEAD(&age_info->aged_aso,
 535                                                          act, next);
 536                                         MLX5_AGE_SET(age_info,
 537                                                      MLX5_AGE_EVENT_NEW);
 538                                 }
 539                                 rte_spinlock_unlock(&age_info->aged_sl);
 540                         }
 541                 }
 542         }
 543         mlx5_age_event_prepare(sh);
 544 }
 545
 546 /**
 547  * Handle completions from WQEs sent to ASO SQ.
 548  *
 549  * @param[in] sh
 550  *   Shared device context.
 551  *
 552  * @return
 553  *   Number of CQEs handled.
 554  */
 555 static uint16_t
 556 mlx5_aso_completion_handle(struct mlx5_dev_ctx_shared *sh)
 557 {
 558         struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
 559         struct mlx5_aso_sq *sq = &mng->aso_sq;
 560         struct mlx5_aso_cq *cq = &sq->cq;
 561         volatile struct mlx5_cqe *restrict cqe;
 562         const unsigned int cq_size = 1 << cq->log_desc_n;
 563         const unsigned int mask = cq_size - 1;
 564         uint32_t idx;
 565         uint32_t next_idx = cq->cq_ci & mask;
 566         const uint16_t max = (uint16_t)(sq->head - sq->tail);
 567         uint16_t i = 0;
 568         int ret;
 569         if (unlikely(!max))
 570                 return 0;
 571         do {
 572                 idx = next_idx;
 573                 next_idx = (cq->cq_ci + 1) & mask;
 574                 rte_prefetch0(&cq->cqes[next_idx]);
 575                 cqe = &cq->cqes[idx];
 576                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
 577                 /*
 578                  * Be sure owner read is done before any other cookie field or
 579                  * opaque field.
 580                  */
 581                 rte_io_rmb();
 582                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 583                         if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
 584                                 break;
 585                         mlx5_aso_cqe_err_handle(sq);
 586                 } else {
 587                         i += sq->elts[(sq->tail + i) & mask].burst_size;
 588                 }
 589                 cq->cq_ci++;
 590         } while (1);
 591         if (likely(i)) {
 592                 mlx5_aso_age_action_update(sh, i);
 593                 sq->tail += i;
 594                 rte_io_wmb();
 595                 cq->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
 596         }
 597         return i;
 598 }
 599
 600 /**
 601  * Periodically read CQEs and send WQEs to ASO SQ.
 602  *
 603  * @param[in] arg
 604  *   Shared device context containing the ASO SQ.
 605  */
 606 static void
 607 mlx5_flow_aso_alarm(void *arg)
 608 {
 609         struct mlx5_dev_ctx_shared *sh = arg;
 610         struct mlx5_aso_sq *sq = &sh->aso_age_mng->aso_sq;
 611         uint32_t us = 100u;
 612         uint16_t n;
 613
 614         rte_spinlock_lock(&sh->aso_age_mng->resize_sl);
 615         n = sh->aso_age_mng->next;
 616         rte_spinlock_unlock(&sh->aso_age_mng->resize_sl);
 617         mlx5_aso_completion_handle(sh);
 618         if (sq->next == n) {
 619                 /* End of loop: wait 1 second. */
 620                 us = US_PER_S;
 621                 sq->next = 0;
 622         }
 623         mlx5_aso_sq_enqueue_burst(sh->aso_age_mng, n);
 624         if (rte_eal_alarm_set(us, mlx5_flow_aso_alarm, sh))
 625                 DRV_LOG(ERR, "Cannot reinitialize aso alarm.");
 626 }
 627
 628 /**
 629  * API to start ASO access using ASO SQ.
 630  *
 631  * @param[in] sh
 632  *   Pointer to shared device context.
 633  *
 634  * @return
 635  *   0 on success, a negative errno value otherwise and rte_errno is set.
 636  */
 637 int
 638 mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh)
 639 {
 640         if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) {
 641                 DRV_LOG(ERR, "Cannot reinitialize ASO age alarm.");
 642                 return -rte_errno;
 643         }
 644         return 0;
 645 }
 646
 647 /**
 648  * API to stop ASO access using ASO SQ.
 649  *
 650  * @param[in] sh
 651  *   Pointer to shared device context.
 652  *
 653  * @return
 654  *   0 on success, a negative errno value otherwise and rte_errno is set.
 655  */
 656 int
 657 mlx5_aso_queue_stop(struct mlx5_dev_ctx_shared *sh)
 658 {
 659         int retries = 1024;
 660
 661         if (!sh->aso_age_mng->aso_sq.sq)
 662                 return -EINVAL;
 663         rte_errno = 0;
 664         while (--retries) {
 665                 rte_eal_alarm_cancel(mlx5_flow_aso_alarm, sh);
 666                 if (rte_errno != EINPROGRESS)
 667                         break;
 668                 rte_pause();
 669         }
 670         return -rte_errno;
 671 }