drivers/net/mlx5/mlx5_flow_aso.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4 #include <mlx5_prm.h>
   5 #include <rte_malloc.h>
   6 #include <rte_cycles.h>
   7 #include <rte_eal_paging.h>
   8
   9 #include <mlx5_malloc.h>
  10 #include <mlx5_common_os.h>
  11 #include <mlx5_common_devx.h>
  12
  13 #include "mlx5.h"
  14 #include "mlx5_flow.h"
  15
  16 /**
  17  * Free MR resources.
  18  *
  19  * @param[in] cdev
  20  *   Pointer to the mlx5 common device.
  21  * @param[in] mr
  22  *   MR to free.
  23  */
  24 static void
  25 mlx5_aso_dereg_mr(struct mlx5_common_device *cdev, struct mlx5_pmd_mr *mr)
  26 {
  27         void *addr = mr->addr;
  28
  29         cdev->mr_scache.dereg_mr_cb(mr);
  30         mlx5_free(addr);
  31         memset(mr, 0, sizeof(*mr));
  32 }
  33
  34 /**
  35  * Register Memory Region.
  36  *
  37  * @param[in] cdev
  38  *   Pointer to the mlx5 common device.
  39  * @param[in] length
  40  *   Size of MR buffer.
  41  * @param[in/out] mr
  42  *   Pointer to MR to create.
  43  *
  44  * @return
  45  *   0 on success, a negative errno value otherwise and rte_errno is set.
  46  */
  47 static int
  48 mlx5_aso_reg_mr(struct mlx5_common_device *cdev, size_t length,
  49                 struct mlx5_pmd_mr *mr)
  50 {
  51         int ret;
  52
  53         mr->addr = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096,
  54                                SOCKET_ID_ANY);
  55         if (!mr->addr) {
  56                 DRV_LOG(ERR, "Failed to create ASO bits mem for MR.");
  57                 return -1;
  58         }
  59         ret = cdev->mr_scache.reg_mr_cb(cdev->pd, mr->addr, length, mr);
  60         if (ret) {
  61                 DRV_LOG(ERR, "Failed to create direct Mkey.");
  62                 mlx5_free(mr->addr);
  63                 return -1;
  64         }
  65         return 0;
  66 }
  67
  68 /**
  69  * Destroy Send Queue used for ASO access.
  70  *
  71  * @param[in] sq
  72  *   ASO SQ to destroy.
  73  */
  74 static void
  75 mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq)
  76 {
  77         mlx5_devx_sq_destroy(&sq->sq_obj);
  78         mlx5_devx_cq_destroy(&sq->cq.cq_obj);
  79         memset(sq, 0, sizeof(*sq));
  80 }
  81
  82 /**
  83  * Initialize Send Queue used for ASO access.
  84  *
  85  * @param[in] sq
  86  *   ASO SQ to initialize.
  87  */
  88 static void
  89 mlx5_aso_age_init_sq(struct mlx5_aso_sq *sq)
  90 {
  91         volatile struct mlx5_aso_wqe *restrict wqe;
  92         int i;
  93         int size = 1 << sq->log_desc_n;
  94         uint64_t addr;
  95
  96         /* All the next fields state should stay constant. */
  97         for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
  98                 wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
  99                                                           (sizeof(*wqe) >> 4));
 100                 wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);
 101                 addr = (uint64_t)((uint64_t *)sq->mr.addr + i *
 102                                             MLX5_ASO_AGE_ACTIONS_PER_POOL / 64);
 103                 wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
 104                 wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
 105                 wqe->aso_cseg.operand_masks = rte_cpu_to_be_32
 106                         (0u |
 107                          (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
 108                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
 109                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
 110                          (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
 111                 wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX);
 112         }
 113 }
 114
 115 /**
 116  * Initialize Send Queue used for ASO flow meter access.
 117  *
 118  * @param[in] sq
 119  *   ASO SQ to initialize.
 120  */
 121 static void
 122 mlx5_aso_mtr_init_sq(struct mlx5_aso_sq *sq)
 123 {
 124         volatile struct mlx5_aso_wqe *restrict wqe;
 125         int i;
 126         int size = 1 << sq->log_desc_n;
 127
 128         /* All the next fields state should stay constant. */
 129         for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
 130                 wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
 131                                                           (sizeof(*wqe) >> 4));
 132                 wqe->aso_cseg.operand_masks = RTE_BE32(0u |
 133                          (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
 134                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
 135                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
 136                          (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
 137                 wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
 138                                                          MLX5_COMP_MODE_OFFSET);
 139         }
 140 }
 141
 142 /*
 143  * Initialize Send Queue used for ASO connection tracking.
 144  *
 145  * @param[in] sq
 146  *   ASO SQ to initialize.
 147  */
 148 static void
 149 mlx5_aso_ct_init_sq(struct mlx5_aso_sq *sq)
 150 {
 151         volatile struct mlx5_aso_wqe *restrict wqe;
 152         int i;
 153         int size = 1 << sq->log_desc_n;
 154         uint64_t addr;
 155
 156         /* All the next fields state should stay constant. */
 157         for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
 158                 wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
 159                                                           (sizeof(*wqe) >> 4));
 160                 /* One unique MR for the query data. */
 161                 wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);
 162                 /* Magic number 64 represents the length of a ASO CT obj. */
 163                 addr = (uint64_t)((uintptr_t)sq->mr.addr + i * 64);
 164                 wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
 165                 wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
 166                 /*
 167                  * The values of operand_masks are different for modify
 168                  * and query.
 169                  * And data_mask may be different for each modification. In
 170                  * query, it could be zero and ignored.
 171                  * CQE generation is always needed, in order to decide when
 172                  * it is available to create the flow or read the data.
 173                  */
 174                 wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
 175                                                    MLX5_COMP_MODE_OFFSET);
 176         }
 177 }
 178
 179 /**
 180  * Create Send Queue used for ASO access.
 181  *
 182  * @param[in] cdev
 183  *   Pointer to the mlx5 common device.
 184  * @param[in/out] sq
 185  *   Pointer to SQ to create.
 186  * @param[in] uar
 187  *   User Access Region object.
 188  *
 189  * @return
 190  *   0 on success, a negative errno value otherwise and rte_errno is set.
 191  */
 192 static int
 193 mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq,
 194                    void *uar)
 195 {
 196         struct mlx5_devx_cq_attr cq_attr = {
 197                 .uar_page_id = mlx5_os_get_devx_uar_page_id(uar),
 198         };
 199         struct mlx5_devx_create_sq_attr sq_attr = {
 200                 .user_index = 0xFFFF,
 201                 .wq_attr = (struct mlx5_devx_wq_attr){
 202                         .pd = cdev->pdn,
 203                         .uar_page = mlx5_os_get_devx_uar_page_id(uar),
 204                 },
 205                 .ts_format =
 206                         mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format),
 207         };
 208         struct mlx5_devx_modify_sq_attr modify_attr = {
 209                 .state = MLX5_SQC_STATE_RDY,
 210         };
 211         uint16_t log_wqbb_n;
 212         int ret;
 213
 214         if (mlx5_devx_cq_create(cdev->ctx, &sq->cq.cq_obj,
 215                                 MLX5_ASO_QUEUE_LOG_DESC, &cq_attr,
 216                                 SOCKET_ID_ANY))
 217                 goto error;
 218         sq->cq.cq_ci = 0;
 219         sq->cq.log_desc_n = MLX5_ASO_QUEUE_LOG_DESC;
 220         sq->log_desc_n = MLX5_ASO_QUEUE_LOG_DESC;
 221         sq_attr.cqn = sq->cq.cq_obj.cq->id;
 222         /* for mlx5_aso_wqe that is twice the size of mlx5_wqe */
 223         log_wqbb_n = sq->log_desc_n + 1;
 224         ret = mlx5_devx_sq_create(cdev->ctx, &sq->sq_obj, log_wqbb_n, &sq_attr,
 225                                   SOCKET_ID_ANY);
 226         if (ret) {
 227                 DRV_LOG(ERR, "Can't create SQ object.");
 228                 rte_errno = ENOMEM;
 229                 goto error;
 230         }
 231         ret = mlx5_devx_cmd_modify_sq(sq->sq_obj.sq, &modify_attr);
 232         if (ret) {
 233                 DRV_LOG(ERR, "Can't change SQ state to ready.");
 234                 rte_errno = ENOMEM;
 235                 goto error;
 236         }
 237         sq->pi = 0;
 238         sq->head = 0;
 239         sq->tail = 0;
 240         sq->sqn = sq->sq_obj.sq->id;
 241         rte_spinlock_init(&sq->sqsl);
 242         return 0;
 243 error:
 244         mlx5_aso_destroy_sq(sq);
 245         return -1;
 246 }
 247
 248 /**
 249  * API to create and initialize Send Queue used for ASO access.
 250  *
 251  * @param[in] sh
 252  *   Pointer to shared device context.
 253  * @param[in] aso_opc_mod
 254  *   Mode of ASO feature.
 255  *
 256  * @return
 257  *   0 on success, a negative errno value otherwise and rte_errno is set.
 258  */
 259 int
 260 mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
 261                     enum mlx5_access_aso_opc_mod aso_opc_mod)
 262 {
 263         uint32_t sq_desc_n = 1 << MLX5_ASO_QUEUE_LOG_DESC;
 264         struct mlx5_common_device *cdev = sh->cdev;
 265
 266         switch (aso_opc_mod) {
 267         case ASO_OPC_MOD_FLOW_HIT:
 268                 if (mlx5_aso_reg_mr(cdev, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *
 269                                     sq_desc_n, &sh->aso_age_mng->aso_sq.mr))
 270                         return -1;
 271                 if (mlx5_aso_sq_create(cdev, &sh->aso_age_mng->aso_sq,
 272                                        sh->tx_uar.obj)) {
 273                         mlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr);
 274                         return -1;
 275                 }
 276                 mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq);
 277                 break;
 278         case ASO_OPC_MOD_POLICER:
 279                 if (mlx5_aso_sq_create(cdev, &sh->mtrmng->pools_mng.sq,
 280                                        sh->tx_uar.obj))
 281                         return -1;
 282                 mlx5_aso_mtr_init_sq(&sh->mtrmng->pools_mng.sq);
 283                 break;
 284         case ASO_OPC_MOD_CONNECTION_TRACKING:
 285                 /* 64B per object for query. */
 286                 if (mlx5_aso_reg_mr(cdev, 64 * sq_desc_n,
 287                                     &sh->ct_mng->aso_sq.mr))
 288                         return -1;
 289                 if (mlx5_aso_sq_create(cdev, &sh->ct_mng->aso_sq,
 290                                        sh->tx_uar.obj)) {
 291                         mlx5_aso_dereg_mr(cdev, &sh->ct_mng->aso_sq.mr);
 292                         return -1;
 293                 }
 294                 mlx5_aso_ct_init_sq(&sh->ct_mng->aso_sq);
 295                 break;
 296         default:
 297                 DRV_LOG(ERR, "Unknown ASO operation mode");
 298                 return -1;
 299         }
 300         return 0;
 301 }
 302
 303 /**
 304  * API to destroy Send Queue used for ASO access.
 305  *
 306  * @param[in] sh
 307  *   Pointer to shared device context.
 308  * @param[in] aso_opc_mod
 309  *   Mode of ASO feature.
 310  */
 311 void
 312 mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh,
 313                       enum mlx5_access_aso_opc_mod aso_opc_mod)
 314 {
 315         struct mlx5_aso_sq *sq;
 316
 317         switch (aso_opc_mod) {
 318         case ASO_OPC_MOD_FLOW_HIT:
 319                 mlx5_aso_dereg_mr(sh->cdev, &sh->aso_age_mng->aso_sq.mr);
 320                 sq = &sh->aso_age_mng->aso_sq;
 321                 break;
 322         case ASO_OPC_MOD_POLICER:
 323                 sq = &sh->mtrmng->pools_mng.sq;
 324                 break;
 325         case ASO_OPC_MOD_CONNECTION_TRACKING:
 326                 mlx5_aso_dereg_mr(sh->cdev, &sh->ct_mng->aso_sq.mr);
 327                 sq = &sh->ct_mng->aso_sq;
 328                 break;
 329         default:
 330                 DRV_LOG(ERR, "Unknown ASO operation mode");
 331                 return;
 332         }
 333         mlx5_aso_destroy_sq(sq);
 334 }
 335
 336 /**
 337  * Write a burst of WQEs to ASO SQ.
 338  *
 339  * @param[in] sh
 340  *   Pointer to shared device context.
 341  * @param[in] n
 342  *   Index of the last valid pool.
 343  *
 344  * @return
 345  *   Number of WQEs in burst.
 346  */
 347 static uint16_t
 348 mlx5_aso_sq_enqueue_burst(struct mlx5_dev_ctx_shared *sh, uint16_t n)
 349 {
 350         struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
 351         volatile struct mlx5_aso_wqe *wqe;
 352         struct mlx5_aso_sq *sq = &mng->aso_sq;
 353         struct mlx5_aso_age_pool *pool;
 354         uint16_t size = 1 << sq->log_desc_n;
 355         uint16_t mask = size - 1;
 356         uint16_t max;
 357         uint16_t start_head = sq->head;
 358
 359         max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), n - sq->next);
 360         if (unlikely(!max))
 361                 return 0;
 362         sq->elts[start_head & mask].burst_size = max;
 363         do {
 364                 wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
 365                 rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
 366                 /* Fill next WQE. */
 367                 rte_rwlock_read_lock(&mng->resize_rwl);
 368                 pool = mng->pools[sq->next];
 369                 rte_rwlock_read_unlock(&mng->resize_rwl);
 370                 sq->elts[sq->head & mask].pool = pool;
 371                 wqe->general_cseg.misc =
 372                                 rte_cpu_to_be_32(((struct mlx5_devx_obj *)
 373                                                  (pool->flow_hit_aso_obj))->id);
 374                 wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 375                                                          MLX5_COMP_MODE_OFFSET);
 376                 wqe->general_cseg.opcode = rte_cpu_to_be_32
 377                                                 (MLX5_OPCODE_ACCESS_ASO |
 378                                                  (ASO_OPC_MOD_FLOW_HIT <<
 379                                                   WQE_CSEG_OPC_MOD_OFFSET) |
 380                                                  (sq->pi <<
 381                                                   WQE_CSEG_WQE_INDEX_OFFSET));
 382                 sq->pi += 2; /* Each WQE contains 2 WQEBB's. */
 383                 sq->head++;
 384                 sq->next++;
 385                 max--;
 386         } while (max);
 387         wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
 388                                                          MLX5_COMP_MODE_OFFSET);
 389         mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
 390                            sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
 391                            !sh->tx_uar.dbnc);
 392         return sq->elts[start_head & mask].burst_size;
 393 }
 394
 395 /**
 396  * Debug utility function. Dump contents of error CQE and WQE.
 397  *
 398  * @param[in] cqe
 399  *   Error CQE to dump.
 400  * @param[in] wqe
 401  *   Error WQE to dump.
 402  */
 403 static void
 404 mlx5_aso_dump_err_objs(volatile uint32_t *cqe, volatile uint32_t *wqe)
 405 {
 406         int i;
 407
 408         DRV_LOG(ERR, "Error cqe:");
 409         for (i = 0; i < 16; i += 4)
 410                 DRV_LOG(ERR, "%08X %08X %08X %08X", cqe[i], cqe[i + 1],
 411                         cqe[i + 2], cqe[i + 3]);
 412         DRV_LOG(ERR, "\nError wqe:");
 413         for (i = 0; i < (int)sizeof(struct mlx5_aso_wqe) / 4; i += 4)
 414                 DRV_LOG(ERR, "%08X %08X %08X %08X", wqe[i], wqe[i + 1],
 415                         wqe[i + 2], wqe[i + 3]);
 416 }
 417
 418 /**
 419  * Handle case of error CQE.
 420  *
 421  * @param[in] sq
 422  *   ASO SQ to use.
 423  */
 424 static void
 425 mlx5_aso_cqe_err_handle(struct mlx5_aso_sq *sq)
 426 {
 427         struct mlx5_aso_cq *cq = &sq->cq;
 428         uint32_t idx = cq->cq_ci & ((1 << cq->log_desc_n) - 1);
 429         volatile struct mlx5_err_cqe *cqe =
 430                         (volatile struct mlx5_err_cqe *)&cq->cq_obj.cqes[idx];
 431
 432         cq->errors++;
 433         idx = rte_be_to_cpu_16(cqe->wqe_counter) & (1u << sq->log_desc_n);
 434         mlx5_aso_dump_err_objs((volatile uint32_t *)cqe,
 435                                (volatile uint32_t *)&sq->sq_obj.aso_wqes[idx]);
 436 }
 437
 438 /**
 439  * Update ASO objects upon completion.
 440  *
 441  * @param[in] sh
 442  *   Shared device context.
 443  * @param[in] n
 444  *   Number of completed ASO objects.
 445  */
 446 static void
 447 mlx5_aso_age_action_update(struct mlx5_dev_ctx_shared *sh, uint16_t n)
 448 {
 449         struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
 450         struct mlx5_aso_sq *sq = &mng->aso_sq;
 451         struct mlx5_age_info *age_info;
 452         const uint16_t size = 1 << sq->log_desc_n;
 453         const uint16_t mask = size - 1;
 454         const uint64_t curr = MLX5_CURR_TIME_SEC;
 455         uint16_t expected = AGE_CANDIDATE;
 456         uint16_t i;
 457
 458         for (i = 0; i < n; ++i) {
 459                 uint16_t idx = (sq->tail + i) & mask;
 460                 struct mlx5_aso_age_pool *pool = sq->elts[idx].pool;
 461                 uint64_t diff = curr - pool->time_of_last_age_check;
 462                 uint64_t *addr = sq->mr.addr;
 463                 int j;
 464
 465                 addr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64;
 466                 pool->time_of_last_age_check = curr;
 467                 for (j = 0; j < MLX5_ASO_AGE_ACTIONS_PER_POOL; j++) {
 468                         struct mlx5_aso_age_action *act = &pool->actions[j];
 469                         struct mlx5_age_param *ap = &act->age_params;
 470                         uint8_t byte;
 471                         uint8_t offset;
 472                         uint8_t *u8addr;
 473                         uint8_t hit;
 474
 475                         if (__atomic_load_n(&ap->state, __ATOMIC_RELAXED) !=
 476                                             AGE_CANDIDATE)
 477                                 continue;
 478                         byte = 63 - (j / 8);
 479                         offset = j % 8;
 480                         u8addr = (uint8_t *)addr;
 481                         hit = (u8addr[byte] >> offset) & 0x1;
 482                         if (hit) {
 483                                 __atomic_store_n(&ap->sec_since_last_hit, 0,
 484                                                  __ATOMIC_RELAXED);
 485                         } else {
 486                                 struct mlx5_priv *priv;
 487
 488                                 __atomic_fetch_add(&ap->sec_since_last_hit,
 489                                                    diff, __ATOMIC_RELAXED);
 490                                 /* If timeout passed add to aged-out list. */
 491                                 if (ap->sec_since_last_hit <= ap->timeout)
 492                                         continue;
 493                                 priv =
 494                                 rte_eth_devices[ap->port_id].data->dev_private;
 495                                 age_info = GET_PORT_AGE_INFO(priv);
 496                                 rte_spinlock_lock(&age_info->aged_sl);
 497                                 if (__atomic_compare_exchange_n(&ap->state,
 498                                                                 &expected,
 499                                                                 AGE_TMOUT,
 500                                                                 false,
 501                                                                __ATOMIC_RELAXED,
 502                                                             __ATOMIC_RELAXED)) {
 503                                         LIST_INSERT_HEAD(&age_info->aged_aso,
 504                                                          act, next);
 505                                         MLX5_AGE_SET(age_info,
 506                                                      MLX5_AGE_EVENT_NEW);
 507                                 }
 508                                 rte_spinlock_unlock(&age_info->aged_sl);
 509                         }
 510                 }
 511         }
 512         mlx5_age_event_prepare(sh);
 513 }
 514
 515 /**
 516  * Handle completions from WQEs sent to ASO SQ.
 517  *
 518  * @param[in] sh
 519  *   Shared device context.
 520  *
 521  * @return
 522  *   Number of CQEs handled.
 523  */
 524 static uint16_t
 525 mlx5_aso_completion_handle(struct mlx5_dev_ctx_shared *sh)
 526 {
 527         struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
 528         struct mlx5_aso_sq *sq = &mng->aso_sq;
 529         struct mlx5_aso_cq *cq = &sq->cq;
 530         volatile struct mlx5_cqe *restrict cqe;
 531         const unsigned int cq_size = 1 << cq->log_desc_n;
 532         const unsigned int mask = cq_size - 1;
 533         uint32_t idx;
 534         uint32_t next_idx = cq->cq_ci & mask;
 535         const uint16_t max = (uint16_t)(sq->head - sq->tail);
 536         uint16_t i = 0;
 537         int ret;
 538         if (unlikely(!max))
 539                 return 0;
 540         do {
 541                 idx = next_idx;
 542                 next_idx = (cq->cq_ci + 1) & mask;
 543                 rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
 544                 cqe = &cq->cq_obj.cqes[idx];
 545                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
 546                 /*
 547                  * Be sure owner read is done before any other cookie field or
 548                  * opaque field.
 549                  */
 550                 rte_io_rmb();
 551                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 552                         if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
 553                                 break;
 554                         mlx5_aso_cqe_err_handle(sq);
 555                 } else {
 556                         i += sq->elts[(sq->tail + i) & mask].burst_size;
 557                 }
 558                 cq->cq_ci++;
 559         } while (1);
 560         if (likely(i)) {
 561                 mlx5_aso_age_action_update(sh, i);
 562                 sq->tail += i;
 563                 rte_io_wmb();
 564                 cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
 565         }
 566         return i;
 567 }
 568
 569 /**
 570  * Periodically read CQEs and send WQEs to ASO SQ.
 571  *
 572  * @param[in] arg
 573  *   Shared device context containing the ASO SQ.
 574  */
 575 static void
 576 mlx5_flow_aso_alarm(void *arg)
 577 {
 578         struct mlx5_dev_ctx_shared *sh = arg;
 579         struct mlx5_aso_sq *sq = &sh->aso_age_mng->aso_sq;
 580         uint32_t us = 100u;
 581         uint16_t n;
 582
 583         rte_rwlock_read_lock(&sh->aso_age_mng->resize_rwl);
 584         n = sh->aso_age_mng->next;
 585         rte_rwlock_read_unlock(&sh->aso_age_mng->resize_rwl);
 586         mlx5_aso_completion_handle(sh);
 587         if (sq->next == n) {
 588                 /* End of loop: wait 1 second. */
 589                 us = US_PER_S;
 590                 sq->next = 0;
 591         }
 592         mlx5_aso_sq_enqueue_burst(sh, n);
 593         if (rte_eal_alarm_set(us, mlx5_flow_aso_alarm, sh))
 594                 DRV_LOG(ERR, "Cannot reinitialize aso alarm.");
 595 }
 596
 597 /**
 598  * API to start ASO access using ASO SQ.
 599  *
 600  * @param[in] sh
 601  *   Pointer to shared device context.
 602  *
 603  * @return
 604  *   0 on success, a negative errno value otherwise and rte_errno is set.
 605  */
 606 int
 607 mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh)
 608 {
 609         if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) {
 610                 DRV_LOG(ERR, "Cannot reinitialize ASO age alarm.");
 611                 return -rte_errno;
 612         }
 613         return 0;
 614 }
 615
 616 /**
 617  * API to stop ASO access using ASO SQ.
 618  *
 619  * @param[in] sh
 620  *   Pointer to shared device context.
 621  *
 622  * @return
 623  *   0 on success, a negative errno value otherwise and rte_errno is set.
 624  */
 625 int
 626 mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh)
 627 {
 628         int retries = 1024;
 629
 630         if (!sh->aso_age_mng->aso_sq.sq_obj.sq)
 631                 return -EINVAL;
 632         rte_errno = 0;
 633         while (--retries) {
 634                 rte_eal_alarm_cancel(mlx5_flow_aso_alarm, sh);
 635                 if (rte_errno != EINPROGRESS)
 636                         break;
 637                 rte_pause();
 638         }
 639         return -rte_errno;
 640 }
 641
 642 static uint16_t
 643 mlx5_aso_mtr_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh,
 644                                struct mlx5_aso_sq *sq,
 645                                struct mlx5_aso_mtr *aso_mtr)
 646 {
 647         volatile struct mlx5_aso_wqe *wqe = NULL;
 648         struct mlx5_flow_meter_info *fm = NULL;
 649         struct mlx5_flow_meter_profile *fmp;
 650         uint16_t size = 1 << sq->log_desc_n;
 651         uint16_t mask = size - 1;
 652         uint16_t res;
 653         uint32_t dseg_idx = 0;
 654         struct mlx5_aso_mtr_pool *pool = NULL;
 655         uint32_t param_le;
 656
 657         rte_spinlock_lock(&sq->sqsl);
 658         res = size - (uint16_t)(sq->head - sq->tail);
 659         if (unlikely(!res)) {
 660                 DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
 661                 rte_spinlock_unlock(&sq->sqsl);
 662                 return 0;
 663         }
 664         wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
 665         rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
 666         /* Fill next WQE. */
 667         fm = &aso_mtr->fm;
 668         sq->elts[sq->head & mask].mtr = aso_mtr;
 669         pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool,
 670                         mtrs[aso_mtr->offset]);
 671         wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id +
 672                         (aso_mtr->offset >> 1));
 673         wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO |
 674                         (ASO_OPC_MOD_POLICER <<
 675                         WQE_CSEG_OPC_MOD_OFFSET) |
 676                         sq->pi << WQE_CSEG_WQE_INDEX_OFFSET);
 677         /* There are 2 meters in one ASO cache line. */
 678         dseg_idx = aso_mtr->offset & 0x1;
 679         wqe->aso_cseg.data_mask =
 680                 RTE_BE64(MLX5_IFC_FLOW_METER_PARAM_MASK << (32 * !dseg_idx));
 681         if (fm->is_enable) {
 682                 wqe->aso_dseg.mtrs[dseg_idx].cbs_cir =
 683                         fm->profile->srtcm_prm.cbs_cir;
 684                 wqe->aso_dseg.mtrs[dseg_idx].ebs_eir =
 685                         fm->profile->srtcm_prm.ebs_eir;
 686         } else {
 687                 wqe->aso_dseg.mtrs[dseg_idx].cbs_cir =
 688                         RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL);
 689                 wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = 0;
 690         }
 691         fmp = fm->profile;
 692         param_le = (1 << ASO_DSEG_VALID_OFFSET);
 693         if (fm->color_aware)
 694                 param_le |= (MLX5_FLOW_COLOR_UNDEFINED << ASO_DSEG_SC_OFFSET);
 695         else
 696                 param_le |= (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET);
 697         if (fmp->profile.packet_mode)
 698                 param_le |= (MLX5_METER_MODE_PKT << ASO_DSEG_MTR_MODE);
 699         wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm = RTE_BE32(param_le);
 700         switch (fmp->profile.alg) {
 701         case RTE_MTR_SRTCM_RFC2697:
 702                 /* Only needed for RFC2697. */
 703                 if (fm->profile->srtcm_prm.ebs_eir)
 704                         wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |=
 705                                         RTE_BE32(1 << ASO_DSEG_BO_OFFSET);
 706                 break;
 707         case RTE_MTR_TRTCM_RFC2698:
 708                 wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |=
 709                                 RTE_BE32(1 << ASO_DSEG_BBOG_OFFSET);
 710                 break;
 711         case RTE_MTR_TRTCM_RFC4115:
 712                 wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |=
 713                                 RTE_BE32(1 << ASO_DSEG_BO_OFFSET);
 714                 break;
 715         default:
 716                 break;
 717         }
 718         /*
 719          * Note:
 720          * Due to software performance reason, the token fields will not be
 721          * set when posting the WQE to ASO SQ. It will be filled by the HW
 722          * automatically.
 723          */
 724         sq->head++;
 725         sq->pi += 2;/* Each WQE contains 2 WQEBB's. */
 726         mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
 727                            sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
 728                            !sh->tx_uar.dbnc);
 729         rte_spinlock_unlock(&sq->sqsl);
 730         return 1;
 731 }
 732
 733 static void
 734 mlx5_aso_mtrs_status_update(struct mlx5_aso_sq *sq, uint16_t aso_mtrs_nums)
 735 {
 736         uint16_t size = 1 << sq->log_desc_n;
 737         uint16_t mask = size - 1;
 738         uint16_t i;
 739         struct mlx5_aso_mtr *aso_mtr = NULL;
 740         uint8_t exp_state = ASO_METER_WAIT;
 741
 742         for (i = 0; i < aso_mtrs_nums; ++i) {
 743                 aso_mtr = sq->elts[(sq->tail + i) & mask].mtr;
 744                 MLX5_ASSERT(aso_mtr);
 745                 (void)__atomic_compare_exchange_n(&aso_mtr->state,
 746                                 &exp_state, ASO_METER_READY,
 747                                 false, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
 748         }
 749 }
 750
 751 static void
 752 mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq)
 753 {
 754         struct mlx5_aso_cq *cq = &sq->cq;
 755         volatile struct mlx5_cqe *restrict cqe;
 756         const unsigned int cq_size = 1 << cq->log_desc_n;
 757         const unsigned int mask = cq_size - 1;
 758         uint32_t idx;
 759         uint32_t next_idx = cq->cq_ci & mask;
 760         uint16_t max;
 761         uint16_t n = 0;
 762         int ret;
 763
 764         rte_spinlock_lock(&sq->sqsl);
 765         max = (uint16_t)(sq->head - sq->tail);
 766         if (unlikely(!max)) {
 767                 rte_spinlock_unlock(&sq->sqsl);
 768                 return;
 769         }
 770         do {
 771                 idx = next_idx;
 772                 next_idx = (cq->cq_ci + 1) & mask;
 773                 rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
 774                 cqe = &cq->cq_obj.cqes[idx];
 775                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
 776                 /*
 777                  * Be sure owner read is done before any other cookie field or
 778                  * opaque field.
 779                  */
 780                 rte_io_rmb();
 781                 if (ret != MLX5_CQE_STATUS_SW_OWN) {
 782                         if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
 783                                 break;
 784                         mlx5_aso_cqe_err_handle(sq);
 785                 } else {
 786                         n++;
 787                 }
 788                 cq->cq_ci++;
 789         } while (1);
 790         if (likely(n)) {
 791                 mlx5_aso_mtrs_status_update(sq, n);
 792                 sq->tail += n;
 793                 rte_io_wmb();
 794                 cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
 795         }
 796         rte_spinlock_unlock(&sq->sqsl);
 797 }
 798
 799 /**
 800  * Update meter parameter by send WQE.
 801  *
 802  * @param[in] dev
 803  *   Pointer to Ethernet device.
 804  * @param[in] priv
 805  *   Pointer to mlx5 private data structure.
 806  * @param[in] fm
 807  *   Pointer to flow meter to be modified.
 808  *
 809  * @return
 810  *   0 on success, a negative errno value otherwise and rte_errno is set.
 811  */
 812 int
 813 mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh,
 814                         struct mlx5_aso_mtr *mtr)
 815 {
 816         struct mlx5_aso_sq *sq = &sh->mtrmng->pools_mng.sq;
 817         uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES;
 818
 819         do {
 820                 mlx5_aso_mtr_completion_handle(sq);
 821                 if (mlx5_aso_mtr_sq_enqueue_single(sh, sq, mtr))
 822                         return 0;
 823                 /* Waiting for wqe resource. */
 824                 rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
 825         } while (--poll_wqe_times);
 826         DRV_LOG(ERR, "Fail to send WQE for ASO meter offset %d",
 827                         mtr->offset);
 828         return -1;
 829 }
 830
 831 /**
 832  * Wait for meter to be ready.
 833  *
 834  * @param[in] dev
 835  *   Pointer to Ethernet device.
 836  * @param[in] priv
 837  *   Pointer to mlx5 private data structure.
 838  * @param[in] fm
 839  *   Pointer to flow meter to be modified.
 840  *
 841  * @return
 842  *   0 on success, a negative errno value otherwise and rte_errno is set.
 843  */
 844 int
 845 mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh,
 846                         struct mlx5_aso_mtr *mtr)
 847 {
 848         struct mlx5_aso_sq *sq = &sh->mtrmng->pools_mng.sq;
 849         uint32_t poll_cqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES;
 850
 851         if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) ==
 852                                             ASO_METER_READY)
 853                 return 0;
 854         do {
 855                 mlx5_aso_mtr_completion_handle(sq);
 856                 if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) ==
 857                                             ASO_METER_READY)
 858                         return 0;
 859                 /* Waiting for CQE ready. */
 860                 rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
 861         } while (--poll_cqe_times);
 862         DRV_LOG(ERR, "Fail to poll CQE ready for ASO meter offset %d",
 863                         mtr->offset);
 864         return -1;
 865 }
 866
 867 /*
 868  * Post a WQE to the ASO CT SQ to modify the context.
 869  *
 870  * @param[in] sh
 871  *   Pointer to shared device context.
 872  * @param[in] ct
 873  *   Pointer to the generic CT structure related to the context.
 874  * @param[in] profile
 875  *   Pointer to configuration profile.
 876  *
 877  * @return
 878  *   1 on success (WQE number), 0 on failure.
 879  */
 880 static uint16_t
 881 mlx5_aso_ct_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh,
 882                               struct mlx5_aso_ct_action *ct,
 883                               const struct rte_flow_action_conntrack *profile)
 884 {
 885         volatile struct mlx5_aso_wqe *wqe = NULL;
 886         struct mlx5_aso_sq *sq = &sh->ct_mng->aso_sq;
 887         uint16_t size = 1 << sq->log_desc_n;
 888         uint16_t mask = size - 1;
 889         uint16_t res;
 890         struct mlx5_aso_ct_pool *pool;
 891         void *desg;
 892         void *orig_dir;
 893         void *reply_dir;
 894
 895         rte_spinlock_lock(&sq->sqsl);
 896         /* Prevent other threads to update the index. */
 897         res = size - (uint16_t)(sq->head - sq->tail);
 898         if (unlikely(!res)) {
 899                 rte_spinlock_unlock(&sq->sqsl);
 900                 DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
 901                 return 0;
 902         }
 903         wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
 904         rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
 905         /* Fill next WQE. */
 906         MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_WAIT);
 907         sq->elts[sq->head & mask].ct = ct;
 908         sq->elts[sq->head & mask].query_data = NULL;
 909         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
 910         /* Each WQE will have a single CT object. */
 911         wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id +
 912                                                   ct->offset);
 913         wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO |
 914                         (ASO_OPC_MOD_CONNECTION_TRACKING <<
 915                          WQE_CSEG_OPC_MOD_OFFSET) |
 916                         sq->pi << WQE_CSEG_WQE_INDEX_OFFSET);
 917         wqe->aso_cseg.operand_masks = rte_cpu_to_be_32
 918                         (0u |
 919                          (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
 920                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
 921                          (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
 922                          (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
 923         wqe->aso_cseg.data_mask = UINT64_MAX;
 924         /* To make compiler happy. */
 925         desg = (void *)(uintptr_t)wqe->aso_dseg.data;
 926         MLX5_SET(conn_track_aso, desg, valid, 1);
 927         MLX5_SET(conn_track_aso, desg, state, profile->state);
 928         MLX5_SET(conn_track_aso, desg, freeze_track, !profile->enable);
 929         MLX5_SET(conn_track_aso, desg, connection_assured,
 930                  profile->live_connection);
 931         MLX5_SET(conn_track_aso, desg, sack_permitted, profile->selective_ack);
 932         MLX5_SET(conn_track_aso, desg, challenged_acked,
 933                  profile->challenge_ack_passed);
 934         /* Heartbeat, retransmission_counter, retranmission_limit_exceeded: 0 */
 935         MLX5_SET(conn_track_aso, desg, heartbeat, 0);
 936         MLX5_SET(conn_track_aso, desg, max_ack_window,
 937                  profile->max_ack_window);
 938         MLX5_SET(conn_track_aso, desg, retransmission_counter, 0);
 939         MLX5_SET(conn_track_aso, desg, retranmission_limit_exceeded, 0);
 940         MLX5_SET(conn_track_aso, desg, retranmission_limit,
 941                  profile->retransmission_limit);
 942         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_scale,
 943                  profile->reply_dir.scale);
 944         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_close_initiated,
 945                  profile->reply_dir.close_initiated);
 946         /* Both directions will use the same liberal mode. */
 947         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_liberal_enabled,
 948                  profile->liberal_mode);
 949         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_data_unacked,
 950                  profile->reply_dir.data_unacked);
 951         MLX5_SET(conn_track_aso, desg, reply_direction_tcp_max_ack,
 952                  profile->reply_dir.last_ack_seen);
 953         MLX5_SET(conn_track_aso, desg, original_direction_tcp_scale,
 954                  profile->original_dir.scale);
 955         MLX5_SET(conn_track_aso, desg, original_direction_tcp_close_initiated,
 956                  profile->original_dir.close_initiated);
 957         MLX5_SET(conn_track_aso, desg, original_direction_tcp_liberal_enabled,
 958                  profile->liberal_mode);
 959         MLX5_SET(conn_track_aso, desg, original_direction_tcp_data_unacked,
 960                  profile->original_dir.data_unacked);
 961         MLX5_SET(conn_track_aso, desg, original_direction_tcp_max_ack,
 962                  profile->original_dir.last_ack_seen);
 963         MLX5_SET(conn_track_aso, desg, last_win, profile->last_window);
 964         MLX5_SET(conn_track_aso, desg, last_dir, profile->last_direction);
 965         MLX5_SET(conn_track_aso, desg, last_index, profile->last_index);
 966         MLX5_SET(conn_track_aso, desg, last_seq, profile->last_seq);
 967         MLX5_SET(conn_track_aso, desg, last_ack, profile->last_ack);
 968         MLX5_SET(conn_track_aso, desg, last_end, profile->last_end);
 969         orig_dir = MLX5_ADDR_OF(conn_track_aso, desg, original_dir);
 970         MLX5_SET(tcp_window_params, orig_dir, sent_end,
 971                  profile->original_dir.sent_end);
 972         MLX5_SET(tcp_window_params, orig_dir, reply_end,
 973                  profile->original_dir.reply_end);
 974         MLX5_SET(tcp_window_params, orig_dir, max_win,
 975                  profile->original_dir.max_win);
 976         MLX5_SET(tcp_window_params, orig_dir, max_ack,
 977                  profile->original_dir.max_ack);
 978         reply_dir = MLX5_ADDR_OF(conn_track_aso, desg, reply_dir);
 979         MLX5_SET(tcp_window_params, reply_dir, sent_end,
 980                  profile->reply_dir.sent_end);
 981         MLX5_SET(tcp_window_params, reply_dir, reply_end,
 982                  profile->reply_dir.reply_end);
 983         MLX5_SET(tcp_window_params, reply_dir, max_win,
 984                  profile->reply_dir.max_win);
 985         MLX5_SET(tcp_window_params, reply_dir, max_ack,
 986                  profile->reply_dir.max_ack);
 987         sq->head++;
 988         sq->pi += 2; /* Each WQE contains 2 WQEBB's. */
 989         mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
 990                            sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
 991                            !sh->tx_uar.dbnc);
 992         rte_spinlock_unlock(&sq->sqsl);
 993         return 1;
 994 }
 995
 996 /*
 997  * Update the status field of CTs to indicate ready to be used by flows.
 998  * A continuous number of CTs since last update.
 999  *
1000  * @param[in] sq
1001  *   Pointer to ASO CT SQ.
1002  * @param[in] num
1003  *   Number of CT structures to be updated.
1004  *
1005  * @return
1006  *   0 on success, a negative value.
1007  */
1008 static void
1009 mlx5_aso_ct_status_update(struct mlx5_aso_sq *sq, uint16_t num)
1010 {
1011         uint16_t size = 1 << sq->log_desc_n;
1012         uint16_t mask = size - 1;
1013         uint16_t i;
1014         struct mlx5_aso_ct_action *ct = NULL;
1015         uint16_t idx;
1016
1017         for (i = 0; i < num; i++) {
1018                 idx = (uint16_t)((sq->tail + i) & mask);
1019                 ct = sq->elts[idx].ct;
1020                 MLX5_ASSERT(ct);
1021                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_READY);
1022                 if (sq->elts[idx].query_data)
1023                         rte_memcpy(sq->elts[idx].query_data,
1024                                    (char *)((uintptr_t)sq->mr.addr + idx * 64),
1025                                    64);
1026         }
1027 }
1028
1029 /*
1030  * Post a WQE to the ASO CT SQ to query the current context.
1031  *
1032  * @param[in] sh
1033  *   Pointer to shared device context.
1034  * @param[in] ct
1035  *   Pointer to the generic CT structure related to the context.
1036  * @param[in] data
1037  *   Pointer to data area to be filled.
1038  *
1039  * @return
1040  *   1 on success (WQE number), 0 on failure.
1041  */
1042 static int
1043 mlx5_aso_ct_sq_query_single(struct mlx5_dev_ctx_shared *sh,
1044                             struct mlx5_aso_ct_action *ct, char *data)
1045 {
1046         volatile struct mlx5_aso_wqe *wqe = NULL;
1047         struct mlx5_aso_sq *sq = &sh->ct_mng->aso_sq;
1048         uint16_t size = 1 << sq->log_desc_n;
1049         uint16_t mask = size - 1;
1050         uint16_t res;
1051         uint16_t wqe_idx;
1052         struct mlx5_aso_ct_pool *pool;
1053         enum mlx5_aso_ct_state state =
1054                                 __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
1055
1056         if (state == ASO_CONNTRACK_FREE) {
1057                 DRV_LOG(ERR, "Fail: No context to query");
1058                 return -1;
1059         } else if (state == ASO_CONNTRACK_WAIT) {
1060                 return 0;
1061         }
1062         rte_spinlock_lock(&sq->sqsl);
1063         res = size - (uint16_t)(sq->head - sq->tail);
1064         if (unlikely(!res)) {
1065                 rte_spinlock_unlock(&sq->sqsl);
1066                 DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
1067                 return 0;
1068         }
1069         MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_QUERY);
1070         wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
1071         /* Confirm the location and address of the prefetch instruction. */
1072         rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
1073         /* Fill next WQE. */
1074         wqe_idx = sq->head & mask;
1075         sq->elts[wqe_idx].ct = ct;
1076         sq->elts[wqe_idx].query_data = data;
1077         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
1078         /* Each WQE will have a single CT object. */
1079         wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id +
1080                                                   ct->offset);
1081         wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO |
1082                         (ASO_OPC_MOD_CONNECTION_TRACKING <<
1083                          WQE_CSEG_OPC_MOD_OFFSET) |
1084                         sq->pi << WQE_CSEG_WQE_INDEX_OFFSET);
1085         /*
1086          * There is no write request is required.
1087          * ASO_OPER_LOGICAL_AND and ASO_OP_ALWAYS_FALSE are both 0.
1088          * "BYTEWISE_64BYTE" is needed for a whole context.
1089          * Set to 0 directly to reduce an endian swap. (Modify should rewrite.)
1090          * "data_mask" is ignored.
1091          * Buffer address was already filled during initialization.
1092          */
1093         wqe->aso_cseg.operand_masks = rte_cpu_to_be_32(BYTEWISE_64BYTE <<
1094                                         ASO_CSEG_DATA_MASK_MODE_OFFSET);
1095         wqe->aso_cseg.data_mask = 0;
1096         sq->head++;
1097         /*
1098          * Each WQE contains 2 WQEBB's, even though
1099          * data segment is not used in this case.
1100          */
1101         sq->pi += 2;
1102         mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
1103                            sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
1104                            !sh->tx_uar.dbnc);
1105         rte_spinlock_unlock(&sq->sqsl);
1106         return 1;
1107 }
1108
1109 /*
1110  * Handle completions from WQEs sent to ASO CT.
1111  *
1112  * @param[in] mng
1113  *   Pointer to the CT pools management structure.
1114  */
1115 static void
1116 mlx5_aso_ct_completion_handle(struct mlx5_aso_ct_pools_mng *mng)
1117 {
1118         struct mlx5_aso_sq *sq = &mng->aso_sq;
1119         struct mlx5_aso_cq *cq = &sq->cq;
1120         volatile struct mlx5_cqe *restrict cqe;
1121         const uint32_t cq_size = 1 << cq->log_desc_n;
1122         const uint32_t mask = cq_size - 1;
1123         uint32_t idx;
1124         uint32_t next_idx;
1125         uint16_t max;
1126         uint16_t n = 0;
1127         int ret;
1128
1129         rte_spinlock_lock(&sq->sqsl);
1130         max = (uint16_t)(sq->head - sq->tail);
1131         if (unlikely(!max)) {
1132                 rte_spinlock_unlock(&sq->sqsl);
1133                 return;
1134         }
1135         next_idx = cq->cq_ci & mask;
1136         do {
1137                 idx = next_idx;
1138                 next_idx = (cq->cq_ci + 1) & mask;
1139                 /* Need to confirm the position of the prefetch. */
1140                 rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
1141                 cqe = &cq->cq_obj.cqes[idx];
1142                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
1143                 /*
1144                  * Be sure owner read is done before any other cookie field or
1145                  * opaque field.
1146                  */
1147                 rte_io_rmb();
1148                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
1149                         if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
1150                                 break;
1151                         mlx5_aso_cqe_err_handle(sq);
1152                 } else {
1153                         n++;
1154                 }
1155                 cq->cq_ci++;
1156         } while (1);
1157         if (likely(n)) {
1158                 mlx5_aso_ct_status_update(sq, n);
1159                 sq->tail += n;
1160                 rte_io_wmb();
1161                 cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
1162         }
1163         rte_spinlock_unlock(&sq->sqsl);
1164 }
1165
1166 /*
1167  * Update connection tracking ASO context by sending WQE.
1168  *
1169  * @param[in] sh
1170  *   Pointer to mlx5_dev_ctx_shared object.
1171  * @param[in] ct
1172  *   Pointer to connection tracking offload object.
1173  * @param[in] profile
1174  *   Pointer to connection tracking TCP parameter.
1175  *
1176  * @return
1177  *   0 on success, -1 on failure.
1178  */
1179 int
1180 mlx5_aso_ct_update_by_wqe(struct mlx5_dev_ctx_shared *sh,
1181                           struct mlx5_aso_ct_action *ct,
1182                           const struct rte_flow_action_conntrack *profile)
1183 {
1184         uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
1185         struct mlx5_aso_ct_pool *pool;
1186
1187         MLX5_ASSERT(ct);
1188         do {
1189                 mlx5_aso_ct_completion_handle(sh->ct_mng);
1190                 if (mlx5_aso_ct_sq_enqueue_single(sh, ct, profile))
1191                         return 0;
1192                 /* Waiting for wqe resource. */
1193                 rte_delay_us_sleep(10u);
1194         } while (--poll_wqe_times);
1195         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
1196         DRV_LOG(ERR, "Fail to send WQE for ASO CT %d in pool %d",
1197                 ct->offset, pool->index);
1198         return -1;
1199 }
1200
1201 /*
1202  * The routine is used to wait for WQE completion to continue with queried data.
1203  *
1204  * @param[in] sh
1205  *   Pointer to mlx5_dev_ctx_shared object.
1206  * @param[in] ct
1207  *   Pointer to connection tracking offload object.
1208  *
1209  * @return
1210  *   0 on success, -1 on failure.
1211  */
1212 int
1213 mlx5_aso_ct_wait_ready(struct mlx5_dev_ctx_shared *sh,
1214                        struct mlx5_aso_ct_action *ct)
1215 {
1216         struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng;
1217         uint32_t poll_cqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
1218         struct mlx5_aso_ct_pool *pool;
1219
1220         if (__atomic_load_n(&ct->state, __ATOMIC_RELAXED) ==
1221             ASO_CONNTRACK_READY)
1222                 return 0;
1223         do {
1224                 mlx5_aso_ct_completion_handle(mng);
1225                 if (__atomic_load_n(&ct->state, __ATOMIC_RELAXED) ==
1226                     ASO_CONNTRACK_READY)
1227                         return 0;
1228                 /* Waiting for CQE ready, consider should block or sleep. */
1229                 rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
1230         } while (--poll_cqe_times);
1231         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
1232         DRV_LOG(ERR, "Fail to poll CQE for ASO CT %d in pool %d",
1233                 ct->offset, pool->index);
1234         return -1;
1235 }
1236
1237 /*
1238  * Convert the hardware conntrack data format into the profile.
1239  *
1240  * @param[in] profile
1241  *   Pointer to conntrack profile to be filled after query.
1242  * @param[in] wdata
1243  *   Pointer to data fetched from hardware.
1244  */
1245 static inline void
1246 mlx5_aso_ct_obj_analyze(struct rte_flow_action_conntrack *profile,
1247                         char *wdata)
1248 {
1249         void *o_dir = MLX5_ADDR_OF(conn_track_aso, wdata, original_dir);
1250         void *r_dir = MLX5_ADDR_OF(conn_track_aso, wdata, reply_dir);
1251
1252         /* MLX5_GET16 should be taken into consideration. */
1253         profile->state = (enum rte_flow_conntrack_state)
1254                          MLX5_GET(conn_track_aso, wdata, state);
1255         profile->enable = !MLX5_GET(conn_track_aso, wdata, freeze_track);
1256         profile->selective_ack = MLX5_GET(conn_track_aso, wdata,
1257                                           sack_permitted);
1258         profile->live_connection = MLX5_GET(conn_track_aso, wdata,
1259                                             connection_assured);
1260         profile->challenge_ack_passed = MLX5_GET(conn_track_aso, wdata,
1261                                                  challenged_acked);
1262         profile->max_ack_window = MLX5_GET(conn_track_aso, wdata,
1263                                            max_ack_window);
1264         profile->retransmission_limit = MLX5_GET(conn_track_aso, wdata,
1265                                                  retranmission_limit);
1266         profile->last_window = MLX5_GET(conn_track_aso, wdata, last_win);
1267         profile->last_direction = MLX5_GET(conn_track_aso, wdata, last_dir);
1268         profile->last_index = (enum rte_flow_conntrack_tcp_last_index)
1269                               MLX5_GET(conn_track_aso, wdata, last_index);
1270         profile->last_seq = MLX5_GET(conn_track_aso, wdata, last_seq);
1271         profile->last_ack = MLX5_GET(conn_track_aso, wdata, last_ack);
1272         profile->last_end = MLX5_GET(conn_track_aso, wdata, last_end);
1273         profile->liberal_mode = MLX5_GET(conn_track_aso, wdata,
1274                                 reply_direction_tcp_liberal_enabled) |
1275                                 MLX5_GET(conn_track_aso, wdata,
1276                                 original_direction_tcp_liberal_enabled);
1277         /* No liberal in the RTE structure profile. */
1278         profile->reply_dir.scale = MLX5_GET(conn_track_aso, wdata,
1279                                             reply_direction_tcp_scale);
1280         profile->reply_dir.close_initiated = MLX5_GET(conn_track_aso, wdata,
1281                                         reply_direction_tcp_close_initiated);
1282         profile->reply_dir.data_unacked = MLX5_GET(conn_track_aso, wdata,
1283                                         reply_direction_tcp_data_unacked);
1284         profile->reply_dir.last_ack_seen = MLX5_GET(conn_track_aso, wdata,
1285                                         reply_direction_tcp_max_ack);
1286         profile->reply_dir.sent_end = MLX5_GET(tcp_window_params,
1287                                                r_dir, sent_end);
1288         profile->reply_dir.reply_end = MLX5_GET(tcp_window_params,
1289                                                 r_dir, reply_end);
1290         profile->reply_dir.max_win = MLX5_GET(tcp_window_params,
1291                                               r_dir, max_win);
1292         profile->reply_dir.max_ack = MLX5_GET(tcp_window_params,
1293                                               r_dir, max_ack);
1294         profile->original_dir.scale = MLX5_GET(conn_track_aso, wdata,
1295                                                original_direction_tcp_scale);
1296         profile->original_dir.close_initiated = MLX5_GET(conn_track_aso, wdata,
1297                                         original_direction_tcp_close_initiated);
1298         profile->original_dir.data_unacked = MLX5_GET(conn_track_aso, wdata,
1299                                         original_direction_tcp_data_unacked);
1300         profile->original_dir.last_ack_seen = MLX5_GET(conn_track_aso, wdata,
1301                                         original_direction_tcp_max_ack);
1302         profile->original_dir.sent_end = MLX5_GET(tcp_window_params,
1303                                                   o_dir, sent_end);
1304         profile->original_dir.reply_end = MLX5_GET(tcp_window_params,
1305                                                    o_dir, reply_end);
1306         profile->original_dir.max_win = MLX5_GET(tcp_window_params,
1307                                                  o_dir, max_win);
1308         profile->original_dir.max_ack = MLX5_GET(tcp_window_params,
1309                                                  o_dir, max_ack);
1310 }
1311
1312 /*
1313  * Query connection tracking information parameter by send WQE.
1314  *
1315  * @param[in] dev
1316  *   Pointer to Ethernet device.
1317  * @param[in] ct
1318  *   Pointer to connection tracking offload object.
1319  * @param[out] profile
1320  *   Pointer to connection tracking TCP information.
1321  *
1322  * @return
1323  *   0 on success, -1 on failure.
1324  */
1325 int
1326 mlx5_aso_ct_query_by_wqe(struct mlx5_dev_ctx_shared *sh,
1327                          struct mlx5_aso_ct_action *ct,
1328                          struct rte_flow_action_conntrack *profile)
1329 {
1330         uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
1331         struct mlx5_aso_ct_pool *pool;
1332         char out_data[64 * 2];
1333         int ret;
1334
1335         MLX5_ASSERT(ct);
1336         do {
1337                 mlx5_aso_ct_completion_handle(sh->ct_mng);
1338                 ret = mlx5_aso_ct_sq_query_single(sh, ct, out_data);
1339                 if (ret < 0)
1340                         return ret;
1341                 else if (ret > 0)
1342                         goto data_handle;
1343                 /* Waiting for wqe resource or state. */
1344                 else
1345                         rte_delay_us_sleep(10u);
1346         } while (--poll_wqe_times);
1347         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
1348         DRV_LOG(ERR, "Fail to send WQE for ASO CT %d in pool %d",
1349                 ct->offset, pool->index);
1350         return -1;
1351 data_handle:
1352         ret = mlx5_aso_ct_wait_ready(sh, ct);
1353         if (!ret)
1354                 mlx5_aso_ct_obj_analyze(profile, out_data);
1355         return ret;
1356 }
1357
1358 /*
1359  * Make sure the conntrack context is synchronized with hardware before
1360  * creating a flow rule that uses it.
1361  *
1362  * @param[in] sh
1363  *   Pointer to shared device context.
1364  * @param[in] ct
1365  *   Pointer to connection tracking offload object.
1366  *
1367  * @return
1368  *   0 on success, a negative errno value otherwise and rte_errno is set.
1369  */
1370 int
1371 mlx5_aso_ct_available(struct mlx5_dev_ctx_shared *sh,
1372                       struct mlx5_aso_ct_action *ct)
1373 {
1374         struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng;
1375         uint32_t poll_cqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
1376         enum mlx5_aso_ct_state state =
1377                                 __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
1378
1379         if (state == ASO_CONNTRACK_FREE) {
1380                 rte_errno = ENXIO;
1381                 return -rte_errno;
1382         } else if (state == ASO_CONNTRACK_READY ||
1383                    state == ASO_CONNTRACK_QUERY) {
1384                 return 0;
1385         }
1386         do {
1387                 mlx5_aso_ct_completion_handle(mng);
1388                 state = __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
1389                 if (state == ASO_CONNTRACK_READY ||
1390                     state == ASO_CONNTRACK_QUERY)
1391                         return 0;
1392                 /* Waiting for CQE ready, consider should block or sleep. */
1393                 rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
1394         } while (--poll_cqe_times);
1395         rte_errno = EBUSY;
1396         return -rte_errno;
1397 }