drivers/net/mlx5/mlx5_flow_aso.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4 #include <mlx5_prm.h>
   5 #include <rte_malloc.h>
   6 #include <rte_cycles.h>
   7 #include <rte_eal_paging.h>
   8
   9 #include <mlx5_malloc.h>
  10 #include <mlx5_common_os.h>
  11 #include <mlx5_common_devx.h>
  12
  13 #include "mlx5.h"
  14 #include "mlx5_flow.h"
  15
  16 /**
  17  * Destroy Completion Queue used for ASO access.
  18  *
  19  * @param[in] cq
  20  *   ASO CQ to destroy.
  21  */
  22 static void
  23 mlx5_aso_cq_destroy(struct mlx5_aso_cq *cq)
  24 {
  25         if (cq->cq_obj.cq)
  26                 mlx5_devx_cq_destroy(&cq->cq_obj);
  27         memset(cq, 0, sizeof(*cq));
  28 }
  29
  30 /**
  31  * Create Completion Queue used for ASO access.
  32  *
  33  * @param[in] ctx
  34  *   Context returned from mlx5 open_device() glue function.
  35  * @param[in/out] cq
  36  *   Pointer to CQ to create.
  37  * @param[in] log_desc_n
  38  *   Log of number of descriptors in queue.
  39  * @param[in] socket
  40  *   Socket to use for allocation.
  41  * @param[in] uar_page_id
  42  *   UAR page ID to use.
  43  *
  44  * @return
  45  *   0 on success, a negative errno value otherwise and rte_errno is set.
  46  */
  47 static int
  48 mlx5_aso_cq_create(void *ctx, struct mlx5_aso_cq *cq, uint16_t log_desc_n,
  49                    int socket, int uar_page_id)
  50 {
  51         struct mlx5_devx_cq_attr attr = {
  52                 .uar_page_id = uar_page_id,
  53         };
  54
  55         cq->log_desc_n = log_desc_n;
  56         cq->cq_ci = 0;
  57         return mlx5_devx_cq_create(ctx, &cq->cq_obj, log_desc_n, &attr, socket);
  58 }
  59
  60 /**
  61  * Free MR resources.
  62  *
  63  * @param[in] sh
  64  *   Pointer to shared device context.
  65  * @param[in] mr
  66  *   MR to free.
  67  */
  68 static void
  69 mlx5_aso_dereg_mr(struct mlx5_dev_ctx_shared *sh, struct mlx5_pmd_mr *mr)
  70 {
  71         void *addr = mr->addr;
  72
  73         sh->share_cache.dereg_mr_cb(mr);
  74         mlx5_free(addr);
  75         memset(mr, 0, sizeof(*mr));
  76 }
  77
  78 /**
  79  * Register Memory Region.
  80  *
  81  * @param[in] sh
  82  *   Pointer to shared device context.
  83  * @param[in] length
  84  *   Size of MR buffer.
  85  * @param[in/out] mr
  86  *   Pointer to MR to create.
  87  * @param[in] socket
  88  *   Socket to use for allocation.
  89  *
  90  * @return
  91  *   0 on success, a negative errno value otherwise and rte_errno is set.
  92  */
  93 static int
  94 mlx5_aso_reg_mr(struct mlx5_dev_ctx_shared *sh, size_t length,
  95                 struct mlx5_pmd_mr *mr, int socket)
  96 {
  97
  98         int ret;
  99
 100         mr->addr = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096,
 101                                socket);
 102         if (!mr->addr) {
 103                 DRV_LOG(ERR, "Failed to create ASO bits mem for MR.");
 104                 return -1;
 105         }
 106         ret = sh->share_cache.reg_mr_cb(sh->pd, mr->addr, length, mr);
 107         if (ret) {
 108                 DRV_LOG(ERR, "Failed to create direct Mkey.");
 109                 mlx5_free(mr->addr);
 110                 return -1;
 111         }
 112         return 0;
 113 }
 114
 115 /**
 116  * Destroy Send Queue used for ASO access.
 117  *
 118  * @param[in] sq
 119  *   ASO SQ to destroy.
 120  */
 121 static void
 122 mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq)
 123 {
 124         mlx5_devx_sq_destroy(&sq->sq_obj);
 125         mlx5_aso_cq_destroy(&sq->cq);
 126         memset(sq, 0, sizeof(*sq));
 127 }
 128
 129 /**
 130  * Initialize Send Queue used for ASO access.
 131  *
 132  * @param[in] sq
 133  *   ASO SQ to initialize.
 134  */
 135 static void
 136 mlx5_aso_age_init_sq(struct mlx5_aso_sq *sq)
 137 {
 138         volatile struct mlx5_aso_wqe *restrict wqe;
 139         int i;
 140         int size = 1 << sq->log_desc_n;
 141         uint64_t addr;
 142
 143         /* All the next fields state should stay constant. */
 144         for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
 145                 wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
 146                                                           (sizeof(*wqe) >> 4));
 147                 wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);
 148                 addr = (uint64_t)((uint64_t *)sq->mr.addr + i *
 149                                             MLX5_ASO_AGE_ACTIONS_PER_POOL / 64);
 150                 wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
 151                 wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
 152                 wqe->aso_cseg.operand_masks = rte_cpu_to_be_32
 153                         (0u |
 154                          (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
 155                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
 156                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
 157                          (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
 158                 wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX);
 159         }
 160 }
 161
 162 /**
 163  * Initialize Send Queue used for ASO flow meter access.
 164  *
 165  * @param[in] sq
 166  *   ASO SQ to initialize.
 167  */
 168 static void
 169 mlx5_aso_mtr_init_sq(struct mlx5_aso_sq *sq)
 170 {
 171         volatile struct mlx5_aso_wqe *restrict wqe;
 172         int i;
 173         int size = 1 << sq->log_desc_n;
 174
 175         /* All the next fields state should stay constant. */
 176         for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
 177                 wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
 178                                                           (sizeof(*wqe) >> 4));
 179                 wqe->aso_cseg.operand_masks = RTE_BE32(0u |
 180                          (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
 181                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
 182                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
 183                          (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
 184                 wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
 185                                                          MLX5_COMP_MODE_OFFSET);
 186         }
 187 }
 188
 189 /*
 190  * Initialize Send Queue used for ASO connection tracking.
 191  *
 192  * @param[in] sq
 193  *   ASO SQ to initialize.
 194  */
 195 static void
 196 mlx5_aso_ct_init_sq(struct mlx5_aso_sq *sq)
 197 {
 198         volatile struct mlx5_aso_wqe *restrict wqe;
 199         int i;
 200         int size = 1 << sq->log_desc_n;
 201         uint64_t addr;
 202
 203         /* All the next fields state should stay constant. */
 204         for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
 205                 wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
 206                                                           (sizeof(*wqe) >> 4));
 207                 /* One unique MR for the query data. */
 208                 wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);
 209                 /* Magic number 64 represents the length of a ASO CT obj. */
 210                 addr = (uint64_t)((uintptr_t)sq->mr.addr + i * 64);
 211                 wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
 212                 wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
 213                 /*
 214                  * The values of operand_masks are different for modify
 215                  * and query.
 216                  * And data_mask may be different for each modification. In
 217                  * query, it could be zero and ignored.
 218                  * CQE generation is always needed, in order to decide when
 219                  * it is available to create the flow or read the data.
 220                  */
 221                 wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
 222                                                    MLX5_COMP_MODE_OFFSET);
 223         }
 224 }
 225
 226 /**
 227  * Create Send Queue used for ASO access.
 228  *
 229  * @param[in] ctx
 230  *   Context returned from mlx5 open_device() glue function.
 231  * @param[in/out] sq
 232  *   Pointer to SQ to create.
 233  * @param[in] socket
 234  *   Socket to use for allocation.
 235  * @param[in] uar
 236  *   User Access Region object.
 237  * @param[in] pdn
 238  *   Protection Domain number to use.
 239  * @param[in] log_desc_n
 240  *   Log of number of descriptors in queue.
 241  * @param[in] ts_format
 242  *   timestamp format supported by the queue.
 243  *
 244  * @return
 245  *   0 on success, a negative errno value otherwise and rte_errno is set.
 246  */
 247 static int
 248 mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, void *uar,
 249                    uint32_t pdn, uint16_t log_desc_n, uint32_t ts_format)
 250 {
 251         struct mlx5_devx_create_sq_attr attr = {
 252                 .user_index = 0xFFFF,
 253                 .wq_attr = (struct mlx5_devx_wq_attr){
 254                         .pd = pdn,
 255                         .uar_page = mlx5_os_get_devx_uar_page_id(uar),
 256                 },
 257                 .ts_format = mlx5_ts_format_conv(ts_format),
 258         };
 259         struct mlx5_devx_modify_sq_attr modify_attr = {
 260                 .state = MLX5_SQC_STATE_RDY,
 261         };
 262         uint16_t log_wqbb_n;
 263         int ret;
 264
 265         if (mlx5_aso_cq_create(ctx, &sq->cq, log_desc_n, socket,
 266                                mlx5_os_get_devx_uar_page_id(uar)))
 267                 goto error;
 268         sq->log_desc_n = log_desc_n;
 269         attr.cqn = sq->cq.cq_obj.cq->id;
 270         /* for mlx5_aso_wqe that is twice the size of mlx5_wqe */
 271         log_wqbb_n = log_desc_n + 1;
 272         ret = mlx5_devx_sq_create(ctx, &sq->sq_obj, log_wqbb_n, &attr, socket);
 273         if (ret) {
 274                 DRV_LOG(ERR, "Can't create SQ object.");
 275                 rte_errno = ENOMEM;
 276                 goto error;
 277         }
 278         ret = mlx5_devx_cmd_modify_sq(sq->sq_obj.sq, &modify_attr);
 279         if (ret) {
 280                 DRV_LOG(ERR, "Can't change SQ state to ready.");
 281                 rte_errno = ENOMEM;
 282                 goto error;
 283         }
 284         sq->pi = 0;
 285         sq->head = 0;
 286         sq->tail = 0;
 287         sq->sqn = sq->sq_obj.sq->id;
 288         sq->uar_addr = mlx5_os_get_devx_uar_reg_addr(uar);
 289         rte_spinlock_init(&sq->sqsl);
 290         return 0;
 291 error:
 292         mlx5_aso_destroy_sq(sq);
 293         return -1;
 294 }
 295
 296 /**
 297  * API to create and initialize Send Queue used for ASO access.
 298  *
 299  * @param[in] sh
 300  *   Pointer to shared device context.
 301  * @param[in] aso_opc_mod
 302  *   Mode of ASO feature.
 303  *
 304  * @return
 305  *   0 on success, a negative errno value otherwise and rte_errno is set.
 306  */
 307 int
 308 mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
 309                     enum mlx5_access_aso_opc_mod aso_opc_mod)
 310 {
 311         uint32_t sq_desc_n = 1 << MLX5_ASO_QUEUE_LOG_DESC;
 312
 313         switch (aso_opc_mod) {
 314         case ASO_OPC_MOD_FLOW_HIT:
 315                 if (mlx5_aso_reg_mr(sh, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *
 316                                     sq_desc_n, &sh->aso_age_mng->aso_sq.mr, 0))
 317                         return -1;
 318                 if (mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0,
 319                                   sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC,
 320                                   sh->sq_ts_format)) {
 321                         mlx5_aso_dereg_mr(sh, &sh->aso_age_mng->aso_sq.mr);
 322                         return -1;
 323                 }
 324                 mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq);
 325                 break;
 326         case ASO_OPC_MOD_POLICER:
 327                 if (mlx5_aso_sq_create(sh->ctx, &sh->mtrmng->pools_mng.sq, 0,
 328                                   sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC,
 329                                   sh->sq_ts_format))
 330                         return -1;
 331                 mlx5_aso_mtr_init_sq(&sh->mtrmng->pools_mng.sq);
 332                 break;
 333         case ASO_OPC_MOD_CONNECTION_TRACKING:
 334                 /* 64B per object for query. */
 335                 if (mlx5_aso_reg_mr(sh, 64 * sq_desc_n,
 336                                     &sh->ct_mng->aso_sq.mr, 0))
 337                         return -1;
 338                 if (mlx5_aso_sq_create(sh->ctx, &sh->ct_mng->aso_sq, 0,
 339                                 sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC,
 340                                 sh->sq_ts_format)) {
 341                         mlx5_aso_dereg_mr(sh, &sh->ct_mng->aso_sq.mr);
 342                         return -1;
 343                 }
 344                 mlx5_aso_ct_init_sq(&sh->ct_mng->aso_sq);
 345                 break;
 346         default:
 347                 DRV_LOG(ERR, "Unknown ASO operation mode");
 348                 return -1;
 349         }
 350         return 0;
 351 }
 352
 353 /**
 354  * API to destroy Send Queue used for ASO access.
 355  *
 356  * @param[in] sh
 357  *   Pointer to shared device context.
 358  * @param[in] aso_opc_mod
 359  *   Mode of ASO feature.
 360  */
 361 void
 362 mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh,
 363                       enum mlx5_access_aso_opc_mod aso_opc_mod)
 364 {
 365         struct mlx5_aso_sq *sq;
 366
 367         switch (aso_opc_mod) {
 368         case ASO_OPC_MOD_FLOW_HIT:
 369                 mlx5_aso_dereg_mr(sh, &sh->aso_age_mng->aso_sq.mr);
 370                 sq = &sh->aso_age_mng->aso_sq;
 371                 break;
 372         case ASO_OPC_MOD_POLICER:
 373                 sq = &sh->mtrmng->pools_mng.sq;
 374                 break;
 375         default:
 376                 DRV_LOG(ERR, "Unknown ASO operation mode");
 377                 return;
 378         }
 379         mlx5_aso_destroy_sq(sq);
 380 }
 381
 382 /**
 383  * Write a burst of WQEs to ASO SQ.
 384  *
 385  * @param[in] mng
 386  *   ASO management data, contains the SQ.
 387  * @param[in] n
 388  *   Index of the last valid pool.
 389  *
 390  * @return
 391  *   Number of WQEs in burst.
 392  */
 393 static uint16_t
 394 mlx5_aso_sq_enqueue_burst(struct mlx5_aso_age_mng *mng, uint16_t n)
 395 {
 396         volatile struct mlx5_aso_wqe *wqe;
 397         struct mlx5_aso_sq *sq = &mng->aso_sq;
 398         struct mlx5_aso_age_pool *pool;
 399         uint16_t size = 1 << sq->log_desc_n;
 400         uint16_t mask = size - 1;
 401         uint16_t max;
 402         uint16_t start_head = sq->head;
 403
 404         max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), n - sq->next);
 405         if (unlikely(!max))
 406                 return 0;
 407         sq->elts[start_head & mask].burst_size = max;
 408         do {
 409                 wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
 410                 rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
 411                 /* Fill next WQE. */
 412                 rte_spinlock_lock(&mng->resize_sl);
 413                 pool = mng->pools[sq->next];
 414                 rte_spinlock_unlock(&mng->resize_sl);
 415                 sq->elts[sq->head & mask].pool = pool;
 416                 wqe->general_cseg.misc =
 417                                 rte_cpu_to_be_32(((struct mlx5_devx_obj *)
 418                                                  (pool->flow_hit_aso_obj))->id);
 419                 wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 420                                                          MLX5_COMP_MODE_OFFSET);
 421                 wqe->general_cseg.opcode = rte_cpu_to_be_32
 422                                                 (MLX5_OPCODE_ACCESS_ASO |
 423                                                  (ASO_OPC_MOD_FLOW_HIT <<
 424                                                   WQE_CSEG_OPC_MOD_OFFSET) |
 425                                                  (sq->pi <<
 426                                                   WQE_CSEG_WQE_INDEX_OFFSET));
 427                 sq->pi += 2; /* Each WQE contains 2 WQEBB's. */
 428                 sq->head++;
 429                 sq->next++;
 430                 max--;
 431         } while (max);
 432         wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
 433                                                          MLX5_COMP_MODE_OFFSET);
 434         rte_io_wmb();
 435         sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi);
 436         rte_wmb();
 437         *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH.*/
 438         rte_wmb();
 439         return sq->elts[start_head & mask].burst_size;
 440 }
 441
 442 /**
 443  * Debug utility function. Dump contents of error CQE and WQE.
 444  *
 445  * @param[in] cqe
 446  *   Error CQE to dump.
 447  * @param[in] wqe
 448  *   Error WQE to dump.
 449  */
 450 static void
 451 mlx5_aso_dump_err_objs(volatile uint32_t *cqe, volatile uint32_t *wqe)
 452 {
 453         int i;
 454
 455         DRV_LOG(ERR, "Error cqe:");
 456         for (i = 0; i < 16; i += 4)
 457                 DRV_LOG(ERR, "%08X %08X %08X %08X", cqe[i], cqe[i + 1],
 458                         cqe[i + 2], cqe[i + 3]);
 459         DRV_LOG(ERR, "\nError wqe:");
 460         for (i = 0; i < (int)sizeof(struct mlx5_aso_wqe) / 4; i += 4)
 461                 DRV_LOG(ERR, "%08X %08X %08X %08X", wqe[i], wqe[i + 1],
 462                         wqe[i + 2], wqe[i + 3]);
 463 }
 464
 465 /**
 466  * Handle case of error CQE.
 467  *
 468  * @param[in] sq
 469  *   ASO SQ to use.
 470  */
 471 static void
 472 mlx5_aso_cqe_err_handle(struct mlx5_aso_sq *sq)
 473 {
 474         struct mlx5_aso_cq *cq = &sq->cq;
 475         uint32_t idx = cq->cq_ci & ((1 << cq->log_desc_n) - 1);
 476         volatile struct mlx5_err_cqe *cqe =
 477                         (volatile struct mlx5_err_cqe *)&cq->cq_obj.cqes[idx];
 478
 479         cq->errors++;
 480         idx = rte_be_to_cpu_16(cqe->wqe_counter) & (1u << sq->log_desc_n);
 481         mlx5_aso_dump_err_objs((volatile uint32_t *)cqe,
 482                                (volatile uint32_t *)&sq->sq_obj.aso_wqes[idx]);
 483 }
 484
 485 /**
 486  * Update ASO objects upon completion.
 487  *
 488  * @param[in] sh
 489  *   Shared device context.
 490  * @param[in] n
 491  *   Number of completed ASO objects.
 492  */
 493 static void
 494 mlx5_aso_age_action_update(struct mlx5_dev_ctx_shared *sh, uint16_t n)
 495 {
 496         struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
 497         struct mlx5_aso_sq *sq = &mng->aso_sq;
 498         struct mlx5_age_info *age_info;
 499         const uint16_t size = 1 << sq->log_desc_n;
 500         const uint16_t mask = size - 1;
 501         const uint64_t curr = MLX5_CURR_TIME_SEC;
 502         uint16_t expected = AGE_CANDIDATE;
 503         uint16_t i;
 504
 505         for (i = 0; i < n; ++i) {
 506                 uint16_t idx = (sq->tail + i) & mask;
 507                 struct mlx5_aso_age_pool *pool = sq->elts[idx].pool;
 508                 uint64_t diff = curr - pool->time_of_last_age_check;
 509                 uint64_t *addr = sq->mr.addr;
 510                 int j;
 511
 512                 addr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64;
 513                 pool->time_of_last_age_check = curr;
 514                 for (j = 0; j < MLX5_ASO_AGE_ACTIONS_PER_POOL; j++) {
 515                         struct mlx5_aso_age_action *act = &pool->actions[j];
 516                         struct mlx5_age_param *ap = &act->age_params;
 517                         uint8_t byte;
 518                         uint8_t offset;
 519                         uint8_t *u8addr;
 520                         uint8_t hit;
 521
 522                         if (__atomic_load_n(&ap->state, __ATOMIC_RELAXED) !=
 523                                             AGE_CANDIDATE)
 524                                 continue;
 525                         byte = 63 - (j / 8);
 526                         offset = j % 8;
 527                         u8addr = (uint8_t *)addr;
 528                         hit = (u8addr[byte] >> offset) & 0x1;
 529                         if (hit) {
 530                                 __atomic_store_n(&ap->sec_since_last_hit, 0,
 531                                                  __ATOMIC_RELAXED);
 532                         } else {
 533                                 struct mlx5_priv *priv;
 534
 535                                 __atomic_fetch_add(&ap->sec_since_last_hit,
 536                                                    diff, __ATOMIC_RELAXED);
 537                                 /* If timeout passed add to aged-out list. */
 538                                 if (ap->sec_since_last_hit <= ap->timeout)
 539                                         continue;
 540                                 priv =
 541                                 rte_eth_devices[ap->port_id].data->dev_private;
 542                                 age_info = GET_PORT_AGE_INFO(priv);
 543                                 rte_spinlock_lock(&age_info->aged_sl);
 544                                 if (__atomic_compare_exchange_n(&ap->state,
 545                                                                 &expected,
 546                                                                 AGE_TMOUT,
 547                                                                 false,
 548                                                                __ATOMIC_RELAXED,
 549                                                             __ATOMIC_RELAXED)) {
 550                                         LIST_INSERT_HEAD(&age_info->aged_aso,
 551                                                          act, next);
 552                                         MLX5_AGE_SET(age_info,
 553                                                      MLX5_AGE_EVENT_NEW);
 554                                 }
 555                                 rte_spinlock_unlock(&age_info->aged_sl);
 556                         }
 557                 }
 558         }
 559         mlx5_age_event_prepare(sh);
 560 }
 561
 562 /**
 563  * Handle completions from WQEs sent to ASO SQ.
 564  *
 565  * @param[in] sh
 566  *   Shared device context.
 567  *
 568  * @return
 569  *   Number of CQEs handled.
 570  */
 571 static uint16_t
 572 mlx5_aso_completion_handle(struct mlx5_dev_ctx_shared *sh)
 573 {
 574         struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
 575         struct mlx5_aso_sq *sq = &mng->aso_sq;
 576         struct mlx5_aso_cq *cq = &sq->cq;
 577         volatile struct mlx5_cqe *restrict cqe;
 578         const unsigned int cq_size = 1 << cq->log_desc_n;
 579         const unsigned int mask = cq_size - 1;
 580         uint32_t idx;
 581         uint32_t next_idx = cq->cq_ci & mask;
 582         const uint16_t max = (uint16_t)(sq->head - sq->tail);
 583         uint16_t i = 0;
 584         int ret;
 585         if (unlikely(!max))
 586                 return 0;
 587         do {
 588                 idx = next_idx;
 589                 next_idx = (cq->cq_ci + 1) & mask;
 590                 rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
 591                 cqe = &cq->cq_obj.cqes[idx];
 592                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
 593                 /*
 594                  * Be sure owner read is done before any other cookie field or
 595                  * opaque field.
 596                  */
 597                 rte_io_rmb();
 598                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 599                         if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
 600                                 break;
 601                         mlx5_aso_cqe_err_handle(sq);
 602                 } else {
 603                         i += sq->elts[(sq->tail + i) & mask].burst_size;
 604                 }
 605                 cq->cq_ci++;
 606         } while (1);
 607         if (likely(i)) {
 608                 mlx5_aso_age_action_update(sh, i);
 609                 sq->tail += i;
 610                 rte_io_wmb();
 611                 cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
 612         }
 613         return i;
 614 }
 615
 616 /**
 617  * Periodically read CQEs and send WQEs to ASO SQ.
 618  *
 619  * @param[in] arg
 620  *   Shared device context containing the ASO SQ.
 621  */
 622 static void
 623 mlx5_flow_aso_alarm(void *arg)
 624 {
 625         struct mlx5_dev_ctx_shared *sh = arg;
 626         struct mlx5_aso_sq *sq = &sh->aso_age_mng->aso_sq;
 627         uint32_t us = 100u;
 628         uint16_t n;
 629
 630         rte_spinlock_lock(&sh->aso_age_mng->resize_sl);
 631         n = sh->aso_age_mng->next;
 632         rte_spinlock_unlock(&sh->aso_age_mng->resize_sl);
 633         mlx5_aso_completion_handle(sh);
 634         if (sq->next == n) {
 635                 /* End of loop: wait 1 second. */
 636                 us = US_PER_S;
 637                 sq->next = 0;
 638         }
 639         mlx5_aso_sq_enqueue_burst(sh->aso_age_mng, n);
 640         if (rte_eal_alarm_set(us, mlx5_flow_aso_alarm, sh))
 641                 DRV_LOG(ERR, "Cannot reinitialize aso alarm.");
 642 }
 643
 644 /**
 645  * API to start ASO access using ASO SQ.
 646  *
 647  * @param[in] sh
 648  *   Pointer to shared device context.
 649  *
 650  * @return
 651  *   0 on success, a negative errno value otherwise and rte_errno is set.
 652  */
 653 int
 654 mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh)
 655 {
 656         if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) {
 657                 DRV_LOG(ERR, "Cannot reinitialize ASO age alarm.");
 658                 return -rte_errno;
 659         }
 660         return 0;
 661 }
 662
 663 /**
 664  * API to stop ASO access using ASO SQ.
 665  *
 666  * @param[in] sh
 667  *   Pointer to shared device context.
 668  *
 669  * @return
 670  *   0 on success, a negative errno value otherwise and rte_errno is set.
 671  */
 672 int
 673 mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh)
 674 {
 675         int retries = 1024;
 676
 677         if (!sh->aso_age_mng->aso_sq.sq_obj.sq)
 678                 return -EINVAL;
 679         rte_errno = 0;
 680         while (--retries) {
 681                 rte_eal_alarm_cancel(mlx5_flow_aso_alarm, sh);
 682                 if (rte_errno != EINPROGRESS)
 683                         break;
 684                 rte_pause();
 685         }
 686         return -rte_errno;
 687 }
 688
 689 static uint16_t
 690 mlx5_aso_mtr_sq_enqueue_single(struct mlx5_aso_sq *sq,
 691                 struct mlx5_aso_mtr *aso_mtr)
 692 {
 693         volatile struct mlx5_aso_wqe *wqe = NULL;
 694         struct mlx5_flow_meter_info *fm = NULL;
 695         struct mlx5_flow_meter_profile *fmp;
 696         uint16_t size = 1 << sq->log_desc_n;
 697         uint16_t mask = size - 1;
 698         uint16_t res;
 699         uint32_t dseg_idx = 0;
 700         struct mlx5_aso_mtr_pool *pool = NULL;
 701
 702         rte_spinlock_lock(&sq->sqsl);
 703         res = size - (uint16_t)(sq->head - sq->tail);
 704         if (unlikely(!res)) {
 705                 DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
 706                 rte_spinlock_unlock(&sq->sqsl);
 707                 return 0;
 708         }
 709         wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
 710         rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
 711         /* Fill next WQE. */
 712         fm = &aso_mtr->fm;
 713         sq->elts[sq->head & mask].mtr = aso_mtr;
 714         pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool,
 715                         mtrs[aso_mtr->offset]);
 716         wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id +
 717                         (aso_mtr->offset >> 1));
 718         wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO |
 719                         (ASO_OPC_MOD_POLICER <<
 720                         WQE_CSEG_OPC_MOD_OFFSET) |
 721                         sq->pi << WQE_CSEG_WQE_INDEX_OFFSET);
 722         /* There are 2 meters in one ASO cache line. */
 723         dseg_idx = aso_mtr->offset & 0x1;
 724         wqe->aso_cseg.data_mask =
 725                 RTE_BE64(MLX5_IFC_FLOW_METER_PARAM_MASK << (32 * !dseg_idx));
 726         if (fm->is_enable) {
 727                 wqe->aso_dseg.mtrs[dseg_idx].cbs_cir =
 728                         fm->profile->srtcm_prm.cbs_cir;
 729                 wqe->aso_dseg.mtrs[dseg_idx].ebs_eir =
 730                         fm->profile->srtcm_prm.ebs_eir;
 731         } else {
 732                 wqe->aso_dseg.mtrs[dseg_idx].cbs_cir =
 733                         RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL);
 734                 wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = 0;
 735         }
 736         fmp = fm->profile;
 737         if (fmp->profile.packet_mode)
 738                 wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm =
 739                                 RTE_BE32((1 << ASO_DSEG_VALID_OFFSET) |
 740                                 (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET) |
 741                                 (MLX5_METER_MODE_PKT << ASO_DSEG_MTR_MODE));
 742         else
 743                 wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm =
 744                                 RTE_BE32((1 << ASO_DSEG_VALID_OFFSET) |
 745                                 (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET));
 746         sq->head++;
 747         sq->pi += 2;/* Each WQE contains 2 WQEBB's. */
 748         rte_io_wmb();
 749         sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi);
 750         rte_wmb();
 751         *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */
 752         rte_wmb();
 753         rte_spinlock_unlock(&sq->sqsl);
 754         return 1;
 755 }
 756
 757 static void
 758 mlx5_aso_mtrs_status_update(struct mlx5_aso_sq *sq, uint16_t aso_mtrs_nums)
 759 {
 760         uint16_t size = 1 << sq->log_desc_n;
 761         uint16_t mask = size - 1;
 762         uint16_t i;
 763         struct mlx5_aso_mtr *aso_mtr = NULL;
 764         uint8_t exp_state = ASO_METER_WAIT;
 765
 766         for (i = 0; i < aso_mtrs_nums; ++i) {
 767                 aso_mtr = sq->elts[(sq->tail + i) & mask].mtr;
 768                 MLX5_ASSERT(aso_mtr);
 769                 (void)__atomic_compare_exchange_n(&aso_mtr->state,
 770                                 &exp_state, ASO_METER_READY,
 771                                 false, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
 772         }
 773 }
 774
 775 static void
 776 mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq)
 777 {
 778         struct mlx5_aso_cq *cq = &sq->cq;
 779         volatile struct mlx5_cqe *restrict cqe;
 780         const unsigned int cq_size = 1 << cq->log_desc_n;
 781         const unsigned int mask = cq_size - 1;
 782         uint32_t idx;
 783         uint32_t next_idx = cq->cq_ci & mask;
 784         uint16_t max;
 785         uint16_t n = 0;
 786         int ret;
 787
 788         rte_spinlock_lock(&sq->sqsl);
 789         max = (uint16_t)(sq->head - sq->tail);
 790         if (unlikely(!max)) {
 791                 rte_spinlock_unlock(&sq->sqsl);
 792                 return;
 793         }
 794         do {
 795                 idx = next_idx;
 796                 next_idx = (cq->cq_ci + 1) & mask;
 797                 rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
 798                 cqe = &cq->cq_obj.cqes[idx];
 799                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
 800                 /*
 801                  * Be sure owner read is done before any other cookie field or
 802                  * opaque field.
 803                  */
 804                 rte_io_rmb();
 805                 if (ret != MLX5_CQE_STATUS_SW_OWN) {
 806                         if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
 807                                 break;
 808                         mlx5_aso_cqe_err_handle(sq);
 809                 } else {
 810                         n++;
 811                 }
 812                 cq->cq_ci++;
 813         } while (1);
 814         if (likely(n)) {
 815                 mlx5_aso_mtrs_status_update(sq, n);
 816                 sq->tail += n;
 817                 rte_io_wmb();
 818                 cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
 819         }
 820         rte_spinlock_unlock(&sq->sqsl);
 821 }
 822
 823 /**
 824  * Update meter parameter by send WQE.
 825  *
 826  * @param[in] dev
 827  *   Pointer to Ethernet device.
 828  * @param[in] priv
 829  *   Pointer to mlx5 private data structure.
 830  * @param[in] fm
 831  *   Pointer to flow meter to be modified.
 832  *
 833  * @return
 834  *   0 on success, a negative errno value otherwise and rte_errno is set.
 835  */
 836 int
 837 mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh,
 838                         struct mlx5_aso_mtr *mtr)
 839 {
 840         struct mlx5_aso_sq *sq = &sh->mtrmng->pools_mng.sq;
 841         uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES;
 842
 843         do {
 844                 mlx5_aso_mtr_completion_handle(sq);
 845                 if (mlx5_aso_mtr_sq_enqueue_single(sq, mtr))
 846                         return 0;
 847                 /* Waiting for wqe resource. */
 848                 rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
 849         } while (--poll_wqe_times);
 850         DRV_LOG(ERR, "Fail to send WQE for ASO meter offset %d",
 851                         mtr->offset);
 852         return -1;
 853 }
 854
 855 /**
 856  * Wait for meter to be ready.
 857  *
 858  * @param[in] dev
 859  *   Pointer to Ethernet device.
 860  * @param[in] priv
 861  *   Pointer to mlx5 private data structure.
 862  * @param[in] fm
 863  *   Pointer to flow meter to be modified.
 864  *
 865  * @return
 866  *   0 on success, a negative errno value otherwise and rte_errno is set.
 867  */
 868 int
 869 mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh,
 870                         struct mlx5_aso_mtr *mtr)
 871 {
 872         struct mlx5_aso_sq *sq = &sh->mtrmng->pools_mng.sq;
 873         uint32_t poll_cqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES;
 874
 875         if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) ==
 876                                             ASO_METER_READY)
 877                 return 0;
 878         do {
 879                 mlx5_aso_mtr_completion_handle(sq);
 880                 if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) ==
 881                                             ASO_METER_READY)
 882                         return 0;
 883                 /* Waiting for CQE ready. */
 884                 rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
 885         } while (--poll_cqe_times);
 886         DRV_LOG(ERR, "Fail to poll CQE ready for ASO meter offset %d",
 887                         mtr->offset);
 888         return -1;
 889 }
 890
 891 /*
 892  * Post a WQE to the ASO CT SQ to modify the context.
 893  *
 894  * @param[in] mng
 895  *   Pointer to the CT pools management structure.
 896  * @param[in] ct
 897  *   Pointer to the generic CT structure related to the context.
 898  * @param[in] profile
 899  *   Pointer to configuration profile.
 900  *
 901  * @return
 902  *   1 on success (WQE number), 0 on failure.
 903  */
 904 static uint16_t
 905 mlx5_aso_ct_sq_enqueue_single(struct mlx5_aso_ct_pools_mng *mng,
 906                               struct mlx5_aso_ct_action *ct,
 907                               const struct rte_flow_action_conntrack *profile)
 908 {
 909         volatile struct mlx5_aso_wqe *wqe = NULL;
 910         struct mlx5_aso_sq *sq = &mng->aso_sq;
 911         uint16_t size = 1 << sq->log_desc_n;
 912         uint16_t mask = size - 1;
 913         uint16_t res;
 914         struct mlx5_aso_ct_pool *pool;
 915         void *desg;
 916         void *orig_dir;
 917         void *reply_dir;
 918
 919         rte_spinlock_lock(&sq->sqsl);
 920         /* Prevent other threads to update the index. */
 921         res = size - (uint16_t)(sq->head - sq->tail);
 922         if (unlikely(!res)) {
 923                 rte_spinlock_unlock(&sq->sqsl);
 924                 DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
 925                 return 0;
 926         }
 927         wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
 928         rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
 929         /* Fill next WQE. */
 930         MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_WAIT);
 931         sq->elts[sq->head & mask].ct = ct;
 932         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
 933         /* Each WQE will have a single CT object. */
 934         wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id +
 935                                                   ct->offset);
 936         wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO |
 937                         (ASO_OPC_MOD_CONNECTION_TRACKING <<
 938                          WQE_CSEG_OPC_MOD_OFFSET) |
 939                         sq->pi << WQE_CSEG_WQE_INDEX_OFFSET);
 940         wqe->aso_cseg.operand_masks = rte_cpu_to_be_32
 941                         (0u |
 942                          (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
 943                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
 944                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
 945                          (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
 946         wqe->aso_cseg.data_mask = UINT64_MAX;
 947         /* To make compiler happy. */
 948         desg = (void *)(uintptr_t)wqe->aso_dseg.data;
 949         MLX5_SET(conn_track_aso, desg, valid, 1);
 950         MLX5_SET(conn_track_aso, desg, state, profile->state);
 951         MLX5_SET(conn_track_aso, desg, freeze_track, !profile->enable);
 952         MLX5_SET(conn_track_aso, desg, connection_assured,
 953                  profile->live_connection);
 954         MLX5_SET(conn_track_aso, desg, sack_permitted, profile->selective_ack);
 955         MLX5_SET(conn_track_aso, desg, challenged_acked,
 956                  profile->challenge_ack_passed);
 957         /* Heartbeat, retransmission_counter, retranmission_limit_exceeded: 0 */
 958         MLX5_SET(conn_track_aso, desg, heartbeat, 0);
 959         MLX5_SET(conn_track_aso, desg, max_ack_window,
 960                  profile->max_ack_window);
 961         MLX5_SET(conn_track_aso, desg, retransmission_counter, 0);
 962         MLX5_SET(conn_track_aso, desg, retranmission_limit_exceeded, 0);
 963         MLX5_SET(conn_track_aso, desg, retranmission_limit,
 964                  profile->retransmission_limit);
 965         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_scale,
 966                  profile->reply_dir.scale);
 967         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_close_initiated,
 968                  profile->reply_dir.close_initiated);
 969         /* Both directions will use the same liberal mode. */
 970         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_liberal_enabled,
 971                  profile->liberal_mode);
 972         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_data_unacked,
 973                  profile->reply_dir.data_unacked);
 974         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_max_ack,
 975                  profile->reply_dir.last_ack_seen);
 976         MLX5_SET(conn_track_aso, desg, original_direction_tcp_scale,
 977                  profile->original_dir.scale);
 978         MLX5_SET(conn_track_aso, desg, original_direction_tcp_close_initiated,
 979                  profile->original_dir.close_initiated);
 980         MLX5_SET(conn_track_aso, desg, original_direction_tcp_liberal_enabled,
 981                  profile->liberal_mode);
 982         MLX5_SET(conn_track_aso, desg, original_direction_tcp_data_unacked,
 983                  profile->original_dir.data_unacked);
 984         MLX5_SET(conn_track_aso, desg, original_direction_tcp_max_ack,
 985                  profile->original_dir.last_ack_seen);
 986         MLX5_SET(conn_track_aso, desg, last_win, profile->last_window);
 987         MLX5_SET(conn_track_aso, desg, last_dir, profile->last_direction);
 988         MLX5_SET(conn_track_aso, desg, last_index, profile->last_index);
 989         MLX5_SET(conn_track_aso, desg, last_seq, profile->last_seq);
 990         MLX5_SET(conn_track_aso, desg, last_ack, profile->last_ack);
 991         MLX5_SET(conn_track_aso, desg, last_end, profile->last_end);
 992         orig_dir = MLX5_ADDR_OF(conn_track_aso, desg, original_dir);
 993         MLX5_SET(tcp_window_params, orig_dir, sent_end,
 994                  profile->original_dir.sent_end);
 995         MLX5_SET(tcp_window_params, orig_dir, reply_end,
 996                  profile->original_dir.reply_end);
 997         MLX5_SET(tcp_window_params, orig_dir, max_win,
 998                  profile->original_dir.max_win);
 999         MLX5_SET(tcp_window_params, orig_dir, max_ack,
1000                  profile->original_dir.max_ack);
1001         reply_dir = MLX5_ADDR_OF(conn_track_aso, desg, reply_dir);
1002         MLX5_SET(tcp_window_params, reply_dir, sent_end,
1003                  profile->reply_dir.sent_end);
1004         MLX5_SET(tcp_window_params, reply_dir, reply_end,
1005                  profile->reply_dir.reply_end);
1006         MLX5_SET(tcp_window_params, reply_dir, max_win,
1007                  profile->reply_dir.max_win);
1008         MLX5_SET(tcp_window_params, reply_dir, max_ack,
1009                  profile->reply_dir.max_ack);
1010         sq->head++;
1011         sq->pi += 2; /* Each WQE contains 2 WQEBB's. */
1012         rte_io_wmb();
1013         sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi);
1014         rte_wmb();
1015         *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */
1016         rte_wmb();
1017         rte_spinlock_unlock(&sq->sqsl);
1018         return 1;
1019 }
1020
1021 /*
1022  * Update the status field of CTs to indicate ready to be used by flows.
1023  * A continuous number of CTs since last update.
1024  *
1025  * @param[in] sq
1026  *   Pointer to ASO CT SQ.
1027  * @param[in] num
1028  *   Number of CT structures to be updated.
1029  *
1030  * @return
1031  *   0 on success, a negative value.
1032  */
1033 static void
1034 mlx5_aso_ct_status_update(struct mlx5_aso_sq *sq, uint16_t num)
1035 {
1036         uint16_t size = 1 << sq->log_desc_n;
1037         uint16_t mask = size - 1;
1038         uint16_t i;
1039         struct mlx5_aso_ct_action *ct = NULL;
1040         uint16_t idx;
1041
1042         for (i = 0; i < num; i++) {
1043                 idx = (uint16_t)((sq->tail + i) & mask);
1044                 ct = sq->elts[idx].ct;
1045                 MLX5_ASSERT(ct);
1046                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_READY);
1047         }
1048 }
1049
1050 /*
1051  * Handle completions from WQEs sent to ASO CT.
1052  *
1053  * @param[in] mng
1054  *   Pointer to the CT pools management structure.
1055  */
1056 static void
1057 mlx5_aso_ct_completion_handle(struct mlx5_aso_ct_pools_mng *mng)
1058 {
1059         struct mlx5_aso_sq *sq = &mng->aso_sq;
1060         struct mlx5_aso_cq *cq = &sq->cq;
1061         volatile struct mlx5_cqe *restrict cqe;
1062         const uint32_t cq_size = 1 << cq->log_desc_n;
1063         const uint32_t mask = cq_size - 1;
1064         uint32_t idx;
1065         uint32_t next_idx;
1066         uint16_t max;
1067         uint16_t n = 0;
1068         int ret;
1069
1070         rte_spinlock_lock(&sq->sqsl);
1071         max = (uint16_t)(sq->head - sq->tail);
1072         if (unlikely(!max)) {
1073                 rte_spinlock_unlock(&sq->sqsl);
1074                 return;
1075         }
1076         next_idx = cq->cq_ci & mask;
1077         do {
1078                 idx = next_idx;
1079                 next_idx = (cq->cq_ci + 1) & mask;
1080                 /* Need to confirm the position of the prefetch. */
1081                 rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
1082                 cqe = &cq->cq_obj.cqes[idx];
1083                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
1084                 /*
1085                  * Be sure owner read is done before any other cookie field or
1086                  * opaque field.
1087                  */
1088                 rte_io_rmb();
1089                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
1090                         if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
1091                                 break;
1092                         mlx5_aso_cqe_err_handle(sq);
1093                 } else {
1094                         n++;
1095                 }
1096                 cq->cq_ci++;
1097         } while (1);
1098         if (likely(n)) {
1099                 mlx5_aso_ct_status_update(sq, n);
1100                 sq->tail += n;
1101                 rte_io_wmb();
1102                 cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
1103         }
1104         rte_spinlock_unlock(&sq->sqsl);
1105 }
1106
1107 /*
1108  * Update connection tracking ASO context by sending WQE.
1109  *
1110  * @param[in] sh
1111  *   Pointer to mlx5_dev_ctx_shared object.
1112  * @param[in] ct
1113  *   Pointer to connection tracking offload object.
1114  * @param[in] profile
1115  *   Pointer to connection tracking TCP parameter.
1116  *
1117  * @return
1118  *   0 on success, -1 on failure.
1119  */
1120 int
1121 mlx5_aso_ct_update_by_wqe(struct mlx5_dev_ctx_shared *sh,
1122                           struct mlx5_aso_ct_action *ct,
1123                           const struct rte_flow_action_conntrack *profile)
1124 {
1125         struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng;
1126         uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
1127         struct mlx5_aso_ct_pool *pool;
1128
1129         MLX5_ASSERT(ct);
1130         do {
1131                 mlx5_aso_ct_completion_handle(mng);
1132                 if (mlx5_aso_ct_sq_enqueue_single(mng, ct, profile))
1133                         return 0;
1134                 /* Waiting for wqe resource. */
1135                 rte_delay_us_sleep(10u);
1136         } while (--poll_wqe_times);
1137         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
1138         DRV_LOG(ERR, "Fail to send WQE for ASO CT %d in pool %d",
1139                 ct->offset, pool->index);
1140         return -1;
1141 }