drivers/net/ice/base/ice_controlq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2001-2020 Intel Corporation
   3  */
   4
   5 #include "ice_common.h"
   6
   7 #define ICE_CQ_INIT_REGS(qinfo, prefix)                         \
   8 do {                                                            \
   9         (qinfo)->sq.head = prefix##_ATQH;                       \
  10         (qinfo)->sq.tail = prefix##_ATQT;                       \
  11         (qinfo)->sq.len = prefix##_ATQLEN;                      \
  12         (qinfo)->sq.bah = prefix##_ATQBAH;                      \
  13         (qinfo)->sq.bal = prefix##_ATQBAL;                      \
  14         (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M;        \
  15         (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \
  16         (qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M;  \
  17         (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M;           \
  18         (qinfo)->rq.head = prefix##_ARQH;                       \
  19         (qinfo)->rq.tail = prefix##_ARQT;                       \
  20         (qinfo)->rq.len = prefix##_ARQLEN;                      \
  21         (qinfo)->rq.bah = prefix##_ARQBAH;                      \
  22         (qinfo)->rq.bal = prefix##_ARQBAL;                      \
  23         (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M;        \
  24         (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \
  25         (qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M;  \
  26         (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M;           \
  27 } while (0)
  28
  29 /**
  30  * ice_adminq_init_regs - Initialize AdminQ registers
  31  * @hw: pointer to the hardware structure
  32  *
  33  * This assumes the alloc_sq and alloc_rq functions have already been called
  34  */
  35 static void ice_adminq_init_regs(struct ice_hw *hw)
  36 {
  37         struct ice_ctl_q_info *cq = &hw->adminq;
  38
  39         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
  40
  41         ICE_CQ_INIT_REGS(cq, PF_FW);
  42 }
  43
  44 /**
  45  * ice_mailbox_init_regs - Initialize Mailbox registers
  46  * @hw: pointer to the hardware structure
  47  *
  48  * This assumes the alloc_sq and alloc_rq functions have already been called
  49  */
  50 static void ice_mailbox_init_regs(struct ice_hw *hw)
  51 {
  52         struct ice_ctl_q_info *cq = &hw->mailboxq;
  53
  54         ICE_CQ_INIT_REGS(cq, PF_MBX);
  55 }
  56
  57 /**
  58  * ice_check_sq_alive
  59  * @hw: pointer to the HW struct
  60  * @cq: pointer to the specific Control queue
  61  *
  62  * Returns true if Queue is enabled else false.
  63  */
  64 bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  65 {
  66         /* check both queue-length and queue-enable fields */
  67         if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask)
  68                 return (rd32(hw, cq->sq.len) & (cq->sq.len_mask |
  69                                                 cq->sq.len_ena_mask)) ==
  70                         (cq->num_sq_entries | cq->sq.len_ena_mask);
  71
  72         return false;
  73 }
  74
  75 /**
  76  * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings
  77  * @hw: pointer to the hardware structure
  78  * @cq: pointer to the specific Control queue
  79  */
  80 static enum ice_status
  81 ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  82 {
  83         size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc);
  84
  85         cq->sq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->sq.desc_buf, size);
  86         if (!cq->sq.desc_buf.va)
  87                 return ICE_ERR_NO_MEMORY;
  88
  89         cq->sq.cmd_buf = ice_calloc(hw, cq->num_sq_entries,
  90                                     sizeof(struct ice_sq_cd));
  91         if (!cq->sq.cmd_buf) {
  92                 ice_free_dma_mem(hw, &cq->sq.desc_buf);
  93                 return ICE_ERR_NO_MEMORY;
  94         }
  95
  96         return ICE_SUCCESS;
  97 }
  98
  99 /**
 100  * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings
 101  * @hw: pointer to the hardware structure
 102  * @cq: pointer to the specific Control queue
 103  */
 104 static enum ice_status
 105 ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 106 {
 107         size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc);
 108
 109         cq->rq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->rq.desc_buf, size);
 110         if (!cq->rq.desc_buf.va)
 111                 return ICE_ERR_NO_MEMORY;
 112         return ICE_SUCCESS;
 113 }
 114
 115 /**
 116  * ice_free_cq_ring - Free control queue ring
 117  * @hw: pointer to the hardware structure
 118  * @ring: pointer to the specific control queue ring
 119  *
 120  * This assumes the posted buffers have already been cleaned
 121  * and de-allocated
 122  */
 123 static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)
 124 {
 125         ice_free_dma_mem(hw, &ring->desc_buf);
 126 }
 127
 128 /**
 129  * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ
 130  * @hw: pointer to the hardware structure
 131  * @cq: pointer to the specific Control queue
 132  */
 133 static enum ice_status
 134 ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 135 {
 136         int i;
 137
 138         /* We'll be allocating the buffer info memory first, then we can
 139          * allocate the mapped buffers for the event processing
 140          */
 141         cq->rq.dma_head = ice_calloc(hw, cq->num_rq_entries,
 142                                      sizeof(cq->rq.desc_buf));
 143         if (!cq->rq.dma_head)
 144                 return ICE_ERR_NO_MEMORY;
 145         cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head;
 146
 147         /* allocate the mapped buffers */
 148         for (i = 0; i < cq->num_rq_entries; i++) {
 149                 struct ice_aq_desc *desc;
 150                 struct ice_dma_mem *bi;
 151
 152                 bi = &cq->rq.r.rq_bi[i];
 153                 bi->va = ice_alloc_dma_mem(hw, bi, cq->rq_buf_size);
 154                 if (!bi->va)
 155                         goto unwind_alloc_rq_bufs;
 156
 157                 /* now configure the descriptors for use */
 158                 desc = ICE_CTL_Q_DESC(cq->rq, i);
 159
 160                 desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
 161                 if (cq->rq_buf_size > ICE_AQ_LG_BUF)
 162                         desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
 163                 desc->opcode = 0;
 164                 /* This is in accordance with Admin queue design, there is no
 165                  * register for buffer size configuration
 166                  */
 167                 desc->datalen = CPU_TO_LE16(bi->size);
 168                 desc->retval = 0;
 169                 desc->cookie_high = 0;
 170                 desc->cookie_low = 0;
 171                 desc->params.generic.addr_high =
 172                         CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
 173                 desc->params.generic.addr_low =
 174                         CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
 175                 desc->params.generic.param0 = 0;
 176                 desc->params.generic.param1 = 0;
 177         }
 178         return ICE_SUCCESS;
 179
 180 unwind_alloc_rq_bufs:
 181         /* don't try to free the one that failed... */
 182         i--;
 183         for (; i >= 0; i--)
 184                 ice_free_dma_mem(hw, &cq->rq.r.rq_bi[i]);
 185         cq->rq.r.rq_bi = NULL;
 186         ice_free(hw, cq->rq.dma_head);
 187         cq->rq.dma_head = NULL;
 188
 189         return ICE_ERR_NO_MEMORY;
 190 }
 191
 192 /**
 193  * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ
 194  * @hw: pointer to the hardware structure
 195  * @cq: pointer to the specific Control queue
 196  */
 197 static enum ice_status
 198 ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 199 {
 200         int i;
 201
 202         /* No mapped memory needed yet, just the buffer info structures */
 203         cq->sq.dma_head = ice_calloc(hw, cq->num_sq_entries,
 204                                      sizeof(cq->sq.desc_buf));
 205         if (!cq->sq.dma_head)
 206                 return ICE_ERR_NO_MEMORY;
 207         cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head;
 208
 209         /* allocate the mapped buffers */
 210         for (i = 0; i < cq->num_sq_entries; i++) {
 211                 struct ice_dma_mem *bi;
 212
 213                 bi = &cq->sq.r.sq_bi[i];
 214                 bi->va = ice_alloc_dma_mem(hw, bi, cq->sq_buf_size);
 215                 if (!bi->va)
 216                         goto unwind_alloc_sq_bufs;
 217         }
 218         return ICE_SUCCESS;
 219
 220 unwind_alloc_sq_bufs:
 221         /* don't try to free the one that failed... */
 222         i--;
 223         for (; i >= 0; i--)
 224                 ice_free_dma_mem(hw, &cq->sq.r.sq_bi[i]);
 225         cq->sq.r.sq_bi = NULL;
 226         ice_free(hw, cq->sq.dma_head);
 227         cq->sq.dma_head = NULL;
 228
 229         return ICE_ERR_NO_MEMORY;
 230 }
 231
 232 static enum ice_status
 233 ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries)
 234 {
 235         /* Clear Head and Tail */
 236         wr32(hw, ring->head, 0);
 237         wr32(hw, ring->tail, 0);
 238
 239         /* set starting point */
 240         wr32(hw, ring->len, (num_entries | ring->len_ena_mask));
 241         wr32(hw, ring->bal, ICE_LO_DWORD(ring->desc_buf.pa));
 242         wr32(hw, ring->bah, ICE_HI_DWORD(ring->desc_buf.pa));
 243
 244         /* Check one register to verify that config was applied */
 245         if (rd32(hw, ring->bal) != ICE_LO_DWORD(ring->desc_buf.pa))
 246                 return ICE_ERR_AQ_ERROR;
 247
 248         return ICE_SUCCESS;
 249 }
 250
 251 /**
 252  * ice_cfg_sq_regs - configure Control ATQ registers
 253  * @hw: pointer to the hardware structure
 254  * @cq: pointer to the specific Control queue
 255  *
 256  * Configure base address and length registers for the transmit queue
 257  */
 258 static enum ice_status
 259 ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 260 {
 261         return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);
 262 }
 263
 264 /**
 265  * ice_cfg_rq_regs - configure Control ARQ register
 266  * @hw: pointer to the hardware structure
 267  * @cq: pointer to the specific Control queue
 268  *
 269  * Configure base address and length registers for the receive (event queue)
 270  */
 271 static enum ice_status
 272 ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 273 {
 274         enum ice_status status;
 275
 276         status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);
 277         if (status)
 278                 return status;
 279
 280         /* Update tail in the HW to post pre-allocated buffers */
 281         wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1));
 282
 283         return ICE_SUCCESS;
 284 }
 285
 286 #define ICE_FREE_CQ_BUFS(hw, qi, ring)                                  \
 287 do {                                                                    \
 288         /* free descriptors */                                          \
 289         if ((qi)->ring.r.ring##_bi) {                                   \
 290                 int i;                                                  \
 291                                                                         \
 292                 for (i = 0; i < (qi)->num_##ring##_entries; i++)        \
 293                         if ((qi)->ring.r.ring##_bi[i].pa)               \
 294                                 ice_free_dma_mem((hw),                  \
 295                                         &(qi)->ring.r.ring##_bi[i]);    \
 296         }                                                               \
 297         /* free the buffer info list */                                 \
 298         if ((qi)->ring.cmd_buf)                                         \
 299                 ice_free(hw, (qi)->ring.cmd_buf);                       \
 300         /* free DMA head */                                             \
 301         ice_free(hw, (qi)->ring.dma_head);                              \
 302 } while (0)
 303
 304 /**
 305  * ice_init_sq - main initialization routine for Control ATQ
 306  * @hw: pointer to the hardware structure
 307  * @cq: pointer to the specific Control queue
 308  *
 309  * This is the main initialization routine for the Control Send Queue
 310  * Prior to calling this function, the driver *MUST* set the following fields
 311  * in the cq->structure:
 312  *     - cq->num_sq_entries
 313  *     - cq->sq_buf_size
 314  *
 315  * Do *NOT* hold the lock when calling this as the memory allocation routines
 316  * called are not going to be atomic context safe
 317  */
 318 static enum ice_status ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 319 {
 320         enum ice_status ret_code;
 321
 322         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 323
 324         if (cq->sq.count > 0) {
 325                 /* queue already initialized */
 326                 ret_code = ICE_ERR_NOT_READY;
 327                 goto init_ctrlq_exit;
 328         }
 329
 330         /* verify input for valid configuration */
 331         if (!cq->num_sq_entries || !cq->sq_buf_size) {
 332                 ret_code = ICE_ERR_CFG;
 333                 goto init_ctrlq_exit;
 334         }
 335
 336         cq->sq.next_to_use = 0;
 337         cq->sq.next_to_clean = 0;
 338
 339         /* allocate the ring memory */
 340         ret_code = ice_alloc_ctrlq_sq_ring(hw, cq);
 341         if (ret_code)
 342                 goto init_ctrlq_exit;
 343
 344         /* allocate buffers in the rings */
 345         ret_code = ice_alloc_sq_bufs(hw, cq);
 346         if (ret_code)
 347                 goto init_ctrlq_free_rings;
 348
 349         /* initialize base registers */
 350         ret_code = ice_cfg_sq_regs(hw, cq);
 351         if (ret_code)
 352                 goto init_ctrlq_free_rings;
 353
 354         /* success! */
 355         cq->sq.count = cq->num_sq_entries;
 356         goto init_ctrlq_exit;
 357
 358 init_ctrlq_free_rings:
 359         ICE_FREE_CQ_BUFS(hw, cq, sq);
 360         ice_free_cq_ring(hw, &cq->sq);
 361
 362 init_ctrlq_exit:
 363         return ret_code;
 364 }
 365
 366 /**
 367  * ice_init_rq - initialize ARQ
 368  * @hw: pointer to the hardware structure
 369  * @cq: pointer to the specific Control queue
 370  *
 371  * The main initialization routine for the Admin Receive (Event) Queue.
 372  * Prior to calling this function, the driver *MUST* set the following fields
 373  * in the cq->structure:
 374  *     - cq->num_rq_entries
 375  *     - cq->rq_buf_size
 376  *
 377  * Do *NOT* hold the lock when calling this as the memory allocation routines
 378  * called are not going to be atomic context safe
 379  */
 380 static enum ice_status ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 381 {
 382         enum ice_status ret_code;
 383
 384         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 385
 386         if (cq->rq.count > 0) {
 387                 /* queue already initialized */
 388                 ret_code = ICE_ERR_NOT_READY;
 389                 goto init_ctrlq_exit;
 390         }
 391
 392         /* verify input for valid configuration */
 393         if (!cq->num_rq_entries || !cq->rq_buf_size) {
 394                 ret_code = ICE_ERR_CFG;
 395                 goto init_ctrlq_exit;
 396         }
 397
 398         cq->rq.next_to_use = 0;
 399         cq->rq.next_to_clean = 0;
 400
 401         /* allocate the ring memory */
 402         ret_code = ice_alloc_ctrlq_rq_ring(hw, cq);
 403         if (ret_code)
 404                 goto init_ctrlq_exit;
 405
 406         /* allocate buffers in the rings */
 407         ret_code = ice_alloc_rq_bufs(hw, cq);
 408         if (ret_code)
 409                 goto init_ctrlq_free_rings;
 410
 411         /* initialize base registers */
 412         ret_code = ice_cfg_rq_regs(hw, cq);
 413         if (ret_code)
 414                 goto init_ctrlq_free_rings;
 415
 416         /* success! */
 417         cq->rq.count = cq->num_rq_entries;
 418         goto init_ctrlq_exit;
 419
 420 init_ctrlq_free_rings:
 421         ICE_FREE_CQ_BUFS(hw, cq, rq);
 422         ice_free_cq_ring(hw, &cq->rq);
 423
 424 init_ctrlq_exit:
 425         return ret_code;
 426 }
 427
 428 /**
 429  * ice_shutdown_sq - shutdown the Control ATQ
 430  * @hw: pointer to the hardware structure
 431  * @cq: pointer to the specific Control queue
 432  *
 433  * The main shutdown routine for the Control Transmit Queue
 434  */
 435 static enum ice_status
 436 ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 437 {
 438         enum ice_status ret_code = ICE_SUCCESS;
 439
 440         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 441
 442         ice_acquire_lock(&cq->sq_lock);
 443
 444         if (!cq->sq.count) {
 445                 ret_code = ICE_ERR_NOT_READY;
 446                 goto shutdown_sq_out;
 447         }
 448
 449         /* Stop firmware AdminQ processing */
 450         wr32(hw, cq->sq.head, 0);
 451         wr32(hw, cq->sq.tail, 0);
 452         wr32(hw, cq->sq.len, 0);
 453         wr32(hw, cq->sq.bal, 0);
 454         wr32(hw, cq->sq.bah, 0);
 455
 456         cq->sq.count = 0;       /* to indicate uninitialized queue */
 457
 458         /* free ring buffers and the ring itself */
 459         ICE_FREE_CQ_BUFS(hw, cq, sq);
 460         ice_free_cq_ring(hw, &cq->sq);
 461
 462 shutdown_sq_out:
 463         ice_release_lock(&cq->sq_lock);
 464         return ret_code;
 465 }
 466
 467 /**
 468  * ice_aq_ver_check - Check the reported AQ API version.
 469  * @hw: pointer to the hardware structure
 470  *
 471  * Checks if the driver should load on a given AQ API version.
 472  *
 473  * Return: 'true' iff the driver should attempt to load. 'false' otherwise.
 474  */
 475 static bool ice_aq_ver_check(struct ice_hw *hw)
 476 {
 477         if (hw->api_maj_ver > EXP_FW_API_VER_MAJOR) {
 478                 /* Major API version is newer than expected, don't load */
 479                 ice_warn(hw, "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
 480                 return false;
 481         } else if (hw->api_maj_ver == EXP_FW_API_VER_MAJOR) {
 482                 if (hw->api_min_ver > (EXP_FW_API_VER_MINOR + 2))
 483                         ice_info(hw, "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
 484                 else if ((hw->api_min_ver + 2) < EXP_FW_API_VER_MINOR)
 485                         ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
 486         } else {
 487                 /* Major API version is older than expected, log a warning */
 488                 ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
 489         }
 490         return true;
 491 }
 492
 493 /**
 494  * ice_shutdown_rq - shutdown Control ARQ
 495  * @hw: pointer to the hardware structure
 496  * @cq: pointer to the specific Control queue
 497  *
 498  * The main shutdown routine for the Control Receive Queue
 499  */
 500 static enum ice_status
 501 ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 502 {
 503         enum ice_status ret_code = ICE_SUCCESS;
 504
 505         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 506
 507         ice_acquire_lock(&cq->rq_lock);
 508
 509         if (!cq->rq.count) {
 510                 ret_code = ICE_ERR_NOT_READY;
 511                 goto shutdown_rq_out;
 512         }
 513
 514         /* Stop Control Queue processing */
 515         wr32(hw, cq->rq.head, 0);
 516         wr32(hw, cq->rq.tail, 0);
 517         wr32(hw, cq->rq.len, 0);
 518         wr32(hw, cq->rq.bal, 0);
 519         wr32(hw, cq->rq.bah, 0);
 520
 521         /* set rq.count to 0 to indicate uninitialized queue */
 522         cq->rq.count = 0;
 523
 524         /* free ring buffers and the ring itself */
 525         ICE_FREE_CQ_BUFS(hw, cq, rq);
 526         ice_free_cq_ring(hw, &cq->rq);
 527
 528 shutdown_rq_out:
 529         ice_release_lock(&cq->rq_lock);
 530         return ret_code;
 531 }
 532
 533 /**
 534  * ice_init_check_adminq - Check version for Admin Queue to know if its alive
 535  * @hw: pointer to the hardware structure
 536  */
 537 static enum ice_status ice_init_check_adminq(struct ice_hw *hw)
 538 {
 539         struct ice_ctl_q_info *cq = &hw->adminq;
 540         enum ice_status status;
 541
 542         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 543
 544         status = ice_aq_get_fw_ver(hw, NULL);
 545         if (status)
 546                 goto init_ctrlq_free_rq;
 547
 548         if (!ice_aq_ver_check(hw)) {
 549                 status = ICE_ERR_FW_API_VER;
 550                 goto init_ctrlq_free_rq;
 551         }
 552
 553         return ICE_SUCCESS;
 554
 555 init_ctrlq_free_rq:
 556         ice_shutdown_rq(hw, cq);
 557         ice_shutdown_sq(hw, cq);
 558         return status;
 559 }
 560
 561 /**
 562  * ice_init_ctrlq - main initialization routine for any control Queue
 563  * @hw: pointer to the hardware structure
 564  * @q_type: specific Control queue type
 565  *
 566  * Prior to calling this function, the driver *MUST* set the following fields
 567  * in the cq->structure:
 568  *     - cq->num_sq_entries
 569  *     - cq->num_rq_entries
 570  *     - cq->rq_buf_size
 571  *     - cq->sq_buf_size
 572  *
 573  * NOTE: this function does not initialize the controlq locks
 574  */
 575 static enum ice_status ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 576 {
 577         struct ice_ctl_q_info *cq;
 578         enum ice_status ret_code;
 579
 580         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 581
 582         switch (q_type) {
 583         case ICE_CTL_Q_ADMIN:
 584                 ice_adminq_init_regs(hw);
 585                 cq = &hw->adminq;
 586                 break;
 587         case ICE_CTL_Q_MAILBOX:
 588                 ice_mailbox_init_regs(hw);
 589                 cq = &hw->mailboxq;
 590                 break;
 591         default:
 592                 return ICE_ERR_PARAM;
 593         }
 594         cq->qtype = q_type;
 595
 596         /* verify input for valid configuration */
 597         if (!cq->num_rq_entries || !cq->num_sq_entries ||
 598             !cq->rq_buf_size || !cq->sq_buf_size) {
 599                 return ICE_ERR_CFG;
 600         }
 601
 602         /* setup SQ command write back timeout */
 603         cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;
 604
 605         /* allocate the ATQ */
 606         ret_code = ice_init_sq(hw, cq);
 607         if (ret_code)
 608                 return ret_code;
 609
 610         /* allocate the ARQ */
 611         ret_code = ice_init_rq(hw, cq);
 612         if (ret_code)
 613                 goto init_ctrlq_free_sq;
 614
 615         /* success! */
 616         return ICE_SUCCESS;
 617
 618 init_ctrlq_free_sq:
 619         ice_shutdown_sq(hw, cq);
 620         return ret_code;
 621 }
 622
 623 /**
 624  * ice_shutdown_ctrlq - shutdown routine for any control queue
 625  * @hw: pointer to the hardware structure
 626  * @q_type: specific Control queue type
 627  *
 628  * NOTE: this function does not destroy the control queue locks.
 629  */
 630 static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 631 {
 632         struct ice_ctl_q_info *cq;
 633
 634         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 635
 636         switch (q_type) {
 637         case ICE_CTL_Q_ADMIN:
 638                 cq = &hw->adminq;
 639                 if (ice_check_sq_alive(hw, cq))
 640                         ice_aq_q_shutdown(hw, true);
 641                 break;
 642         case ICE_CTL_Q_MAILBOX:
 643                 cq = &hw->mailboxq;
 644                 break;
 645         default:
 646                 return;
 647         }
 648
 649         ice_shutdown_sq(hw, cq);
 650         ice_shutdown_rq(hw, cq);
 651 }
 652
 653 /**
 654  * ice_shutdown_all_ctrlq - shutdown routine for all control queues
 655  * @hw: pointer to the hardware structure
 656  *
 657  * NOTE: this function does not destroy the control queue locks. The driver
 658  * may call this at runtime to shutdown and later restart control queues, such
 659  * as in response to a reset event.
 660  */
 661 void ice_shutdown_all_ctrlq(struct ice_hw *hw)
 662 {
 663         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 664         /* Shutdown FW admin queue */
 665         ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
 666         /* Shutdown PF-VF Mailbox */
 667         ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX);
 668 }
 669
 670 /**
 671  * ice_init_all_ctrlq - main initialization routine for all control queues
 672  * @hw: pointer to the hardware structure
 673  *
 674  * Prior to calling this function, the driver MUST* set the following fields
 675  * in the cq->structure for all control queues:
 676  *     - cq->num_sq_entries
 677  *     - cq->num_rq_entries
 678  *     - cq->rq_buf_size
 679  *     - cq->sq_buf_size
 680  *
 681  * NOTE: this function does not initialize the controlq locks.
 682  */
 683 enum ice_status ice_init_all_ctrlq(struct ice_hw *hw)
 684 {
 685         enum ice_status status;
 686         u32 retry = 0;
 687
 688         ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
 689
 690         /* Init FW admin queue */
 691         do {
 692                 status = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
 693                 if (status)
 694                         return status;
 695
 696                 status = ice_init_check_adminq(hw);
 697                 if (status != ICE_ERR_AQ_FW_CRITICAL)
 698                         break;
 699
 700                 ice_debug(hw, ICE_DBG_AQ_MSG,
 701                           "Retry Admin Queue init due to FW critical error\n");
 702                 ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
 703                 ice_msec_delay(ICE_CTL_Q_ADMIN_INIT_MSEC, true);
 704         } while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT);
 705
 706         if (status)
 707                 return status;
 708         /* Init Mailbox queue */
 709         return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);
 710 }
 711
 712 /**
 713  * ice_init_ctrlq_locks - Initialize locks for a control queue
 714  * @cq: pointer to the control queue
 715  *
 716  * Initializes the send and receive queue locks for a given control queue.
 717  */
 718 static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq)
 719 {
 720         ice_init_lock(&cq->sq_lock);
 721         ice_init_lock(&cq->rq_lock);
 722 }
 723
 724 /**
 725  * ice_create_all_ctrlq - main initialization routine for all control queues
 726  * @hw: pointer to the hardware structure
 727  *
 728  * Prior to calling this function, the driver *MUST* set the following fields
 729  * in the cq->structure for all control queues:
 730  *     - cq->num_sq_entries
 731  *     - cq->num_rq_entries
 732  *     - cq->rq_buf_size
 733  *     - cq->sq_buf_size
 734  *
 735  * This function creates all the control queue locks and then calls
 736  * ice_init_all_ctrlq. It should be called once during driver load. If the
 737  * driver needs to re-initialize control queues at run time it should call
 738  * ice_init_all_ctrlq instead.
 739  */
 740 enum ice_status ice_create_all_ctrlq(struct ice_hw *hw)
 741 {
 742         ice_init_ctrlq_locks(&hw->adminq);
 743         ice_init_ctrlq_locks(&hw->mailboxq);
 744
 745         return ice_init_all_ctrlq(hw);
 746 }
 747
 748 /**
 749  * ice_destroy_ctrlq_locks - Destroy locks for a control queue
 750  * @cq: pointer to the control queue
 751  *
 752  * Destroys the send and receive queue locks for a given control queue.
 753  */
 754 static void ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq)
 755 {
 756         ice_destroy_lock(&cq->sq_lock);
 757         ice_destroy_lock(&cq->rq_lock);
 758 }
 759
 760 /**
 761  * ice_destroy_all_ctrlq - exit routine for all control queues
 762  * @hw: pointer to the hardware structure
 763  *
 764  * This function shuts down all the control queues and then destroys the
 765  * control queue locks. It should be called once during driver unload. The
 766  * driver should call ice_shutdown_all_ctrlq if it needs to shut down and
 767  * reinitialize control queues, such as in response to a reset event.
 768  */
 769 void ice_destroy_all_ctrlq(struct ice_hw *hw)
 770 {
 771         /* shut down all the control queues first */
 772         ice_shutdown_all_ctrlq(hw);
 773
 774         ice_destroy_ctrlq_locks(&hw->adminq);
 775         ice_destroy_ctrlq_locks(&hw->mailboxq);
 776 }
 777
 778 /**
 779  * ice_clean_sq - cleans Admin send queue (ATQ)
 780  * @hw: pointer to the hardware structure
 781  * @cq: pointer to the specific Control queue
 782  *
 783  * returns the number of free desc
 784  */
 785 static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 786 {
 787         struct ice_ctl_q_ring *sq = &cq->sq;
 788         u16 ntc = sq->next_to_clean;
 789         struct ice_sq_cd *details;
 790         struct ice_aq_desc *desc;
 791
 792         desc = ICE_CTL_Q_DESC(*sq, ntc);
 793         details = ICE_CTL_Q_DETAILS(*sq, ntc);
 794
 795         while (rd32(hw, cq->sq.head) != ntc) {
 796                 ice_debug(hw, ICE_DBG_AQ_MSG,
 797                           "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head));
 798                 ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
 799                 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
 800                 ntc++;
 801                 if (ntc == sq->count)
 802                         ntc = 0;
 803                 desc = ICE_CTL_Q_DESC(*sq, ntc);
 804                 details = ICE_CTL_Q_DETAILS(*sq, ntc);
 805         }
 806
 807         sq->next_to_clean = ntc;
 808
 809         return ICE_CTL_Q_DESC_UNUSED(sq);
 810 }
 811
 812 /**
 813  * ice_debug_cq
 814  * @hw: pointer to the hardware structure
 815  * @desc: pointer to control queue descriptor
 816  * @buf: pointer to command buffer
 817  * @buf_len: max length of buf
 818  *
 819  * Dumps debug log about control command with descriptor contents.
 820  */
 821 static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len)
 822 {
 823         struct ice_aq_desc *cq_desc = (struct ice_aq_desc *)desc;
 824         u16 datalen, flags;
 825
 826         if (!((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask))
 827                 return;
 828
 829         if (!desc)
 830                 return;
 831
 832         datalen = LE16_TO_CPU(cq_desc->datalen);
 833         flags = LE16_TO_CPU(cq_desc->flags);
 834
 835         ice_debug(hw, ICE_DBG_AQ_DESC,
 836                   "CQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n",
 837                   LE16_TO_CPU(cq_desc->opcode), flags, datalen,
 838                   LE16_TO_CPU(cq_desc->retval));
 839         ice_debug(hw, ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n",
 840                   LE32_TO_CPU(cq_desc->cookie_high),
 841                   LE32_TO_CPU(cq_desc->cookie_low));
 842         ice_debug(hw, ICE_DBG_AQ_DESC, "\tparam (0,1)  0x%08X 0x%08X\n",
 843                   LE32_TO_CPU(cq_desc->params.generic.param0),
 844                   LE32_TO_CPU(cq_desc->params.generic.param1));
 845         ice_debug(hw, ICE_DBG_AQ_DESC, "\taddr (h,l)   0x%08X 0x%08X\n",
 846                   LE32_TO_CPU(cq_desc->params.generic.addr_high),
 847                   LE32_TO_CPU(cq_desc->params.generic.addr_low));
 848         /* Dump buffer iff 1) one exists and 2) is either a response indicated
 849          * by the DD and/or CMP flag set or a command with the RD flag set.
 850          */
 851         if (buf && cq_desc->datalen != 0 &&
 852             (flags & (ICE_AQ_FLAG_DD | ICE_AQ_FLAG_CMP) ||
 853              flags & ICE_AQ_FLAG_RD)) {
 854                 ice_debug(hw, ICE_DBG_AQ_DESC_BUF, "Buffer:\n");
 855                 ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, (u8 *)buf,
 856                                 MIN_T(u16, buf_len, datalen));
 857         }
 858 }
 859
 860 /**
 861  * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ)
 862  * @hw: pointer to the HW struct
 863  * @cq: pointer to the specific Control queue
 864  *
 865  * Returns true if the firmware has processed all descriptors on the
 866  * admin send queue. Returns false if there are still requests pending.
 867  */
 868 static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 869 {
 870         /* AQ designers suggest use of head for better
 871          * timing reliability than DD bit
 872          */
 873         return rd32(hw, cq->sq.head) == cq->sq.next_to_use;
 874 }
 875
 876 /**
 877  * ice_sq_send_cmd_nolock - send command to Control Queue (ATQ)
 878  * @hw: pointer to the HW struct
 879  * @cq: pointer to the specific Control queue
 880  * @desc: prefilled descriptor describing the command (non DMA mem)
 881  * @buf: buffer to use for indirect commands (or NULL for direct commands)
 882  * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
 883  * @cd: pointer to command details structure
 884  *
 885  * This is the main send command routine for the ATQ. It runs the queue,
 886  * cleans the queue, etc.
 887  */
 888 static enum ice_status
 889 ice_sq_send_cmd_nolock(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 890                        struct ice_aq_desc *desc, void *buf, u16 buf_size,
 891                        struct ice_sq_cd *cd)
 892 {
 893         struct ice_dma_mem *dma_buf = NULL;
 894         struct ice_aq_desc *desc_on_ring;
 895         bool cmd_completed = false;
 896         enum ice_status status = ICE_SUCCESS;
 897         struct ice_sq_cd *details;
 898         u32 total_delay = 0;
 899         u16 retval = 0;
 900         u32 val = 0;
 901
 902         /* if reset is in progress return a soft error */
 903         if (hw->reset_ongoing)
 904                 return ICE_ERR_RESET_ONGOING;
 905
 906         cq->sq_last_status = ICE_AQ_RC_OK;
 907
 908         if (!cq->sq.count) {
 909                 ice_debug(hw, ICE_DBG_AQ_MSG,
 910                           "Control Send queue not initialized.\n");
 911                 status = ICE_ERR_AQ_EMPTY;
 912                 goto sq_send_command_error;
 913         }
 914
 915         if ((buf && !buf_size) || (!buf && buf_size)) {
 916                 status = ICE_ERR_PARAM;
 917                 goto sq_send_command_error;
 918         }
 919
 920         if (buf) {
 921                 if (buf_size > cq->sq_buf_size) {
 922                         ice_debug(hw, ICE_DBG_AQ_MSG,
 923                                   "Invalid buffer size for Control Send queue: %d.\n",
 924                                   buf_size);
 925                         status = ICE_ERR_INVAL_SIZE;
 926                         goto sq_send_command_error;
 927                 }
 928
 929                 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_BUF);
 930                 if (buf_size > ICE_AQ_LG_BUF)
 931                         desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
 932         }
 933
 934         val = rd32(hw, cq->sq.head);
 935         if (val >= cq->num_sq_entries) {
 936                 ice_debug(hw, ICE_DBG_AQ_MSG,
 937                           "head overrun at %d in the Control Send Queue ring\n",
 938                           val);
 939                 status = ICE_ERR_AQ_EMPTY;
 940                 goto sq_send_command_error;
 941         }
 942
 943         details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use);
 944         if (cd)
 945                 *details = *cd;
 946         else
 947                 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
 948
 949         /* Call clean and check queue available function to reclaim the
 950          * descriptors that were processed by FW/MBX; the function returns the
 951          * number of desc available. The clean function called here could be
 952          * called in a separate thread in case of asynchronous completions.
 953          */
 954         if (ice_clean_sq(hw, cq) == 0) {
 955                 ice_debug(hw, ICE_DBG_AQ_MSG,
 956                           "Error: Control Send Queue is full.\n");
 957                 status = ICE_ERR_AQ_FULL;
 958                 goto sq_send_command_error;
 959         }
 960
 961         /* initialize the temp desc pointer with the right desc */
 962         desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use);
 963
 964         /* if the desc is available copy the temp desc to the right place */
 965         ice_memcpy(desc_on_ring, desc, sizeof(*desc_on_ring),
 966                    ICE_NONDMA_TO_DMA);
 967
 968         /* if buf is not NULL assume indirect command */
 969         if (buf) {
 970                 dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use];
 971                 /* copy the user buf into the respective DMA buf */
 972                 ice_memcpy(dma_buf->va, buf, buf_size, ICE_NONDMA_TO_DMA);
 973                 desc_on_ring->datalen = CPU_TO_LE16(buf_size);
 974
 975                 /* Update the address values in the desc with the pa value
 976                  * for respective buffer
 977                  */
 978                 desc_on_ring->params.generic.addr_high =
 979                         CPU_TO_LE32(ICE_HI_DWORD(dma_buf->pa));
 980                 desc_on_ring->params.generic.addr_low =
 981                         CPU_TO_LE32(ICE_LO_DWORD(dma_buf->pa));
 982         }
 983
 984         /* Debug desc and buffer */
 985         ice_debug(hw, ICE_DBG_AQ_DESC,
 986                   "ATQ: Control Send queue desc and buffer:\n");
 987
 988         ice_debug_cq(hw, (void *)desc_on_ring, buf, buf_size);
 989
 990         (cq->sq.next_to_use)++;
 991         if (cq->sq.next_to_use == cq->sq.count)
 992                 cq->sq.next_to_use = 0;
 993         wr32(hw, cq->sq.tail, cq->sq.next_to_use);
 994
 995         do {
 996                 if (ice_sq_done(hw, cq))
 997                         break;
 998
 999                 ice_usec_delay(ICE_CTL_Q_SQ_CMD_USEC, false);
1000                 total_delay++;
1001         } while (total_delay < cq->sq_cmd_timeout);
1002
1003         /* if ready, copy the desc back to temp */
1004         if (ice_sq_done(hw, cq)) {
1005                 ice_memcpy(desc, desc_on_ring, sizeof(*desc),
1006                            ICE_DMA_TO_NONDMA);
1007                 if (buf) {
1008                         /* get returned length to copy */
1009                         u16 copy_size = LE16_TO_CPU(desc->datalen);
1010
1011                         if (copy_size > buf_size) {
1012                                 ice_debug(hw, ICE_DBG_AQ_MSG,
1013                                           "Return len %d > than buf len %d\n",
1014                                           copy_size, buf_size);
1015                                 status = ICE_ERR_AQ_ERROR;
1016                         } else {
1017                                 ice_memcpy(buf, dma_buf->va, copy_size,
1018                                            ICE_DMA_TO_NONDMA);
1019                         }
1020                 }
1021                 retval = LE16_TO_CPU(desc->retval);
1022                 if (retval) {
1023                         ice_debug(hw, ICE_DBG_AQ_MSG,
1024                                   "Control Send Queue command 0x%04X completed with error 0x%X\n",
1025                                   LE16_TO_CPU(desc->opcode),
1026                                   retval);
1027
1028                         /* strip off FW internal code */
1029                         retval &= 0xff;
1030                 }
1031                 cmd_completed = true;
1032                 if (!status && retval != ICE_AQ_RC_OK)
1033                         status = ICE_ERR_AQ_ERROR;
1034                 cq->sq_last_status = (enum ice_aq_err)retval;
1035         }
1036
1037         ice_debug(hw, ICE_DBG_AQ_MSG,
1038                   "ATQ: desc and buffer writeback:\n");
1039
1040         ice_debug_cq(hw, (void *)desc, buf, buf_size);
1041
1042         /* save writeback AQ if requested */
1043         if (details->wb_desc)
1044                 ice_memcpy(details->wb_desc, desc_on_ring,
1045                            sizeof(*details->wb_desc), ICE_DMA_TO_NONDMA);
1046
1047         /* update the error if time out occurred */
1048         if (!cmd_completed) {
1049                 if (rd32(hw, cq->rq.len) & cq->rq.len_crit_mask ||
1050                     rd32(hw, cq->sq.len) & cq->sq.len_crit_mask) {
1051                         ice_debug(hw, ICE_DBG_AQ_MSG, "Critical FW error.\n");
1052                         status = ICE_ERR_AQ_FW_CRITICAL;
1053                 } else {
1054                         ice_debug(hw, ICE_DBG_AQ_MSG,
1055                                   "Control Send Queue Writeback timeout.\n");
1056                         status = ICE_ERR_AQ_TIMEOUT;
1057                 }
1058         }
1059
1060 sq_send_command_error:
1061         return status;
1062 }
1063
1064 /**
1065  * ice_sq_send_cmd - send command to Control Queue (ATQ)
1066  * @hw: pointer to the HW struct
1067  * @cq: pointer to the specific Control queue
1068  * @desc: prefilled descriptor describing the command (non DMA mem)
1069  * @buf: buffer to use for indirect commands (or NULL for direct commands)
1070  * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
1071  * @cd: pointer to command details structure
1072  *
1073  * This is the main send command routine for the ATQ. It runs the queue,
1074  * cleans the queue, etc.
1075  */
1076 enum ice_status
1077 ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
1078                 struct ice_aq_desc *desc, void *buf, u16 buf_size,
1079                 struct ice_sq_cd *cd)
1080 {
1081         enum ice_status status = ICE_SUCCESS;
1082
1083         /* if reset is in progress return a soft error */
1084         if (hw->reset_ongoing)
1085                 return ICE_ERR_RESET_ONGOING;
1086
1087         ice_acquire_lock(&cq->sq_lock);
1088         status = ice_sq_send_cmd_nolock(hw, cq, desc, buf, buf_size, cd);
1089         ice_release_lock(&cq->sq_lock);
1090
1091         return status;
1092 }
1093
1094 /**
1095  * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function
1096  * @desc: pointer to the temp descriptor (non DMA mem)
1097  * @opcode: the opcode can be used to decide which flags to turn off or on
1098  *
1099  * Fill the desc with default values
1100  */
1101 void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode)
1102 {
1103         /* zero out the desc */
1104         ice_memset(desc, 0, sizeof(*desc), ICE_NONDMA_MEM);
1105         desc->opcode = CPU_TO_LE16(opcode);
1106         desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_SI);
1107 }
1108
1109 /**
1110  * ice_clean_rq_elem
1111  * @hw: pointer to the HW struct
1112  * @cq: pointer to the specific Control queue
1113  * @e: event info from the receive descriptor, includes any buffers
1114  * @pending: number of events that could be left to process
1115  *
1116  * This function cleans one Admin Receive Queue element and returns
1117  * the contents through e. It can also return how many events are
1118  * left to process through 'pending'.
1119  */
1120 enum ice_status
1121 ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq,
1122                   struct ice_rq_event_info *e, u16 *pending)
1123 {
1124         u16 ntc = cq->rq.next_to_clean;
1125         enum ice_status ret_code = ICE_SUCCESS;
1126         struct ice_aq_desc *desc;
1127         struct ice_dma_mem *bi;
1128         u16 desc_idx;
1129         u16 datalen;
1130         u16 flags;
1131         u16 ntu;
1132
1133         /* pre-clean the event info */
1134         ice_memset(&e->desc, 0, sizeof(e->desc), ICE_NONDMA_MEM);
1135
1136         /* take the lock before we start messing with the ring */
1137         ice_acquire_lock(&cq->rq_lock);
1138
1139         if (!cq->rq.count) {
1140                 ice_debug(hw, ICE_DBG_AQ_MSG,
1141                           "Control Receive queue not initialized.\n");
1142                 ret_code = ICE_ERR_AQ_EMPTY;
1143                 goto clean_rq_elem_err;
1144         }
1145
1146         /* set next_to_use to head */
1147         ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1148
1149         if (ntu == ntc) {
1150                 /* nothing to do - shouldn't need to update ring's values */
1151                 ret_code = ICE_ERR_AQ_NO_WORK;
1152                 goto clean_rq_elem_out;
1153         }
1154
1155         /* now clean the next descriptor */
1156         desc = ICE_CTL_Q_DESC(cq->rq, ntc);
1157         desc_idx = ntc;
1158
1159         cq->rq_last_status = (enum ice_aq_err)LE16_TO_CPU(desc->retval);
1160         flags = LE16_TO_CPU(desc->flags);
1161         if (flags & ICE_AQ_FLAG_ERR) {
1162                 ret_code = ICE_ERR_AQ_ERROR;
1163                 ice_debug(hw, ICE_DBG_AQ_MSG,
1164                           "Control Receive Queue Event 0x%04X received with error 0x%X\n",
1165                           LE16_TO_CPU(desc->opcode),
1166                           cq->rq_last_status);
1167         }
1168         ice_memcpy(&e->desc, desc, sizeof(e->desc), ICE_DMA_TO_NONDMA);
1169         datalen = LE16_TO_CPU(desc->datalen);
1170         e->msg_len = MIN_T(u16, datalen, e->buf_len);
1171         if (e->msg_buf && e->msg_len)
1172                 ice_memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va,
1173                            e->msg_len, ICE_DMA_TO_NONDMA);
1174
1175         ice_debug(hw, ICE_DBG_AQ_DESC, "ARQ: desc and buffer:\n");
1176
1177         ice_debug_cq(hw, (void *)desc, e->msg_buf, cq->rq_buf_size);
1178
1179         /* Restore the original datalen and buffer address in the desc,
1180          * FW updates datalen to indicate the event message size
1181          */
1182         bi = &cq->rq.r.rq_bi[ntc];
1183         ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
1184
1185         desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
1186         if (cq->rq_buf_size > ICE_AQ_LG_BUF)
1187                 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
1188         desc->datalen = CPU_TO_LE16(bi->size);
1189         desc->params.generic.addr_high = CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
1190         desc->params.generic.addr_low = CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
1191
1192         /* set tail = the last cleaned desc index. */
1193         wr32(hw, cq->rq.tail, ntc);
1194         /* ntc is updated to tail + 1 */
1195         ntc++;
1196         if (ntc == cq->num_rq_entries)
1197                 ntc = 0;
1198         cq->rq.next_to_clean = ntc;
1199         cq->rq.next_to_use = ntu;
1200
1201 clean_rq_elem_out:
1202         /* Set pending if needed, unlock and return */
1203         if (pending) {
1204                 /* re-read HW head to calculate actual pending messages */
1205                 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1206                 *pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc));
1207         }
1208 clean_rq_elem_err:
1209         ice_release_lock(&cq->rq_lock);
1210
1211         return ret_code;
1212 }