drivers/net/ice/base/ice_controlq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2001-2019
   3  */
   4
   5 #include "ice_common.h"
   6
   7
   8 #define ICE_CQ_INIT_REGS(qinfo, prefix)                         \
   9 do {                                                            \
  10         (qinfo)->sq.head = prefix##_ATQH;                       \
  11         (qinfo)->sq.tail = prefix##_ATQT;                       \
  12         (qinfo)->sq.len = prefix##_ATQLEN;                      \
  13         (qinfo)->sq.bah = prefix##_ATQBAH;                      \
  14         (qinfo)->sq.bal = prefix##_ATQBAL;                      \
  15         (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M;        \
  16         (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \
  17         (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M;           \
  18         (qinfo)->rq.head = prefix##_ARQH;                       \
  19         (qinfo)->rq.tail = prefix##_ARQT;                       \
  20         (qinfo)->rq.len = prefix##_ARQLEN;                      \
  21         (qinfo)->rq.bah = prefix##_ARQBAH;                      \
  22         (qinfo)->rq.bal = prefix##_ARQBAL;                      \
  23         (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M;        \
  24         (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \
  25         (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M;           \
  26 } while (0)
  27
  28 /**
  29  * ice_adminq_init_regs - Initialize AdminQ registers
  30  * @hw: pointer to the hardware structure
  31  *
  32  * This assumes the alloc_sq and alloc_rq functions have already been called
  33  */
  34 static void ice_adminq_init_regs(struct ice_hw *hw)
  35 {
  36         struct ice_ctl_q_info *cq = &hw->adminq;
  37
  38         ICE_CQ_INIT_REGS(cq, PF_FW);
  39 }
  40
  41 /**
  42  * ice_mailbox_init_regs - Initialize Mailbox registers
  43  * @hw: pointer to the hardware structure
  44  *
  45  * This assumes the alloc_sq and alloc_rq functions have already been called
  46  */
  47 static void ice_mailbox_init_regs(struct ice_hw *hw)
  48 {
  49         struct ice_ctl_q_info *cq = &hw->mailboxq;
  50
  51         ICE_CQ_INIT_REGS(cq, PF_MBX);
  52 }
  53
  54
  55 /**
  56  * ice_check_sq_alive
  57  * @hw: pointer to the HW struct
  58  * @cq: pointer to the specific Control queue
  59  *
  60  * Returns true if Queue is enabled else false.
  61  */
  62 bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  63 {
  64         /* check both queue-length and queue-enable fields */
  65         if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask)
  66                 return (rd32(hw, cq->sq.len) & (cq->sq.len_mask |
  67                                                 cq->sq.len_ena_mask)) ==
  68                         (cq->num_sq_entries | cq->sq.len_ena_mask);
  69
  70         return false;
  71 }
  72
  73 /**
  74  * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings
  75  * @hw: pointer to the hardware structure
  76  * @cq: pointer to the specific Control queue
  77  */
  78 static enum ice_status
  79 ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  80 {
  81         size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc);
  82
  83         cq->sq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->sq.desc_buf, size);
  84         if (!cq->sq.desc_buf.va)
  85                 return ICE_ERR_NO_MEMORY;
  86
  87         cq->sq.cmd_buf = ice_calloc(hw, cq->num_sq_entries,
  88                                     sizeof(struct ice_sq_cd));
  89         if (!cq->sq.cmd_buf) {
  90                 ice_free_dma_mem(hw, &cq->sq.desc_buf);
  91                 return ICE_ERR_NO_MEMORY;
  92         }
  93
  94         return ICE_SUCCESS;
  95 }
  96
  97 /**
  98  * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings
  99  * @hw: pointer to the hardware structure
 100  * @cq: pointer to the specific Control queue
 101  */
 102 static enum ice_status
 103 ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 104 {
 105         size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc);
 106
 107         cq->rq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->rq.desc_buf, size);
 108         if (!cq->rq.desc_buf.va)
 109                 return ICE_ERR_NO_MEMORY;
 110         return ICE_SUCCESS;
 111 }
 112
 113 /**
 114  * ice_free_cq_ring - Free control queue ring
 115  * @hw: pointer to the hardware structure
 116  * @ring: pointer to the specific control queue ring
 117  *
 118  * This assumes the posted buffers have already been cleaned
 119  * and de-allocated
 120  */
 121 static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)
 122 {
 123         ice_free_dma_mem(hw, &ring->desc_buf);
 124 }
 125
 126 /**
 127  * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ
 128  * @hw: pointer to the hardware structure
 129  * @cq: pointer to the specific Control queue
 130  */
 131 static enum ice_status
 132 ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 133 {
 134         int i;
 135
 136         /* We'll be allocating the buffer info memory first, then we can
 137          * allocate the mapped buffers for the event processing
 138          */
 139         cq->rq.dma_head = ice_calloc(hw, cq->num_rq_entries,
 140                                      sizeof(cq->rq.desc_buf));
 141         if (!cq->rq.dma_head)
 142                 return ICE_ERR_NO_MEMORY;
 143         cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head;
 144
 145         /* allocate the mapped buffers */
 146         for (i = 0; i < cq->num_rq_entries; i++) {
 147                 struct ice_aq_desc *desc;
 148                 struct ice_dma_mem *bi;
 149
 150                 bi = &cq->rq.r.rq_bi[i];
 151                 bi->va = ice_alloc_dma_mem(hw, bi, cq->rq_buf_size);
 152                 if (!bi->va)
 153                         goto unwind_alloc_rq_bufs;
 154
 155                 /* now configure the descriptors for use */
 156                 desc = ICE_CTL_Q_DESC(cq->rq, i);
 157
 158                 desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
 159                 if (cq->rq_buf_size > ICE_AQ_LG_BUF)
 160                         desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
 161                 desc->opcode = 0;
 162                 /* This is in accordance with Admin queue design, there is no
 163                  * register for buffer size configuration
 164                  */
 165                 desc->datalen = CPU_TO_LE16(bi->size);
 166                 desc->retval = 0;
 167                 desc->cookie_high = 0;
 168                 desc->cookie_low = 0;
 169                 desc->params.generic.addr_high =
 170                         CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
 171                 desc->params.generic.addr_low =
 172                         CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
 173                 desc->params.generic.param0 = 0;
 174                 desc->params.generic.param1 = 0;
 175         }
 176         return ICE_SUCCESS;
 177
 178 unwind_alloc_rq_bufs:
 179         /* don't try to free the one that failed... */
 180         i--;
 181         for (; i >= 0; i--)
 182                 ice_free_dma_mem(hw, &cq->rq.r.rq_bi[i]);
 183         ice_free(hw, cq->rq.dma_head);
 184
 185         return ICE_ERR_NO_MEMORY;
 186 }
 187
 188 /**
 189  * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ
 190  * @hw: pointer to the hardware structure
 191  * @cq: pointer to the specific Control queue
 192  */
 193 static enum ice_status
 194 ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 195 {
 196         int i;
 197
 198         /* No mapped memory needed yet, just the buffer info structures */
 199         cq->sq.dma_head = ice_calloc(hw, cq->num_sq_entries,
 200                                      sizeof(cq->sq.desc_buf));
 201         if (!cq->sq.dma_head)
 202                 return ICE_ERR_NO_MEMORY;
 203         cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head;
 204
 205         /* allocate the mapped buffers */
 206         for (i = 0; i < cq->num_sq_entries; i++) {
 207                 struct ice_dma_mem *bi;
 208
 209                 bi = &cq->sq.r.sq_bi[i];
 210                 bi->va = ice_alloc_dma_mem(hw, bi, cq->sq_buf_size);
 211                 if (!bi->va)
 212                         goto unwind_alloc_sq_bufs;
 213         }
 214         return ICE_SUCCESS;
 215
 216 unwind_alloc_sq_bufs:
 217         /* don't try to free the one that failed... */
 218         i--;
 219         for (; i >= 0; i--)
 220                 ice_free_dma_mem(hw, &cq->sq.r.sq_bi[i]);
 221         ice_free(hw, cq->sq.dma_head);
 222
 223         return ICE_ERR_NO_MEMORY;
 224 }
 225
 226 static enum ice_status
 227 ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries)
 228 {
 229         /* Clear Head and Tail */
 230         wr32(hw, ring->head, 0);
 231         wr32(hw, ring->tail, 0);
 232
 233         /* set starting point */
 234         wr32(hw, ring->len, (num_entries | ring->len_ena_mask));
 235         wr32(hw, ring->bal, ICE_LO_DWORD(ring->desc_buf.pa));
 236         wr32(hw, ring->bah, ICE_HI_DWORD(ring->desc_buf.pa));
 237
 238         /* Check one register to verify that config was applied */
 239         if (rd32(hw, ring->bal) != ICE_LO_DWORD(ring->desc_buf.pa))
 240                 return ICE_ERR_AQ_ERROR;
 241
 242         return ICE_SUCCESS;
 243 }
 244
 245 /**
 246  * ice_cfg_sq_regs - configure Control ATQ registers
 247  * @hw: pointer to the hardware structure
 248  * @cq: pointer to the specific Control queue
 249  *
 250  * Configure base address and length registers for the transmit queue
 251  */
 252 static enum ice_status
 253 ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 254 {
 255         return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);
 256 }
 257
 258 /**
 259  * ice_cfg_rq_regs - configure Control ARQ register
 260  * @hw: pointer to the hardware structure
 261  * @cq: pointer to the specific Control queue
 262  *
 263  * Configure base address and length registers for the receive (event q)
 264  */
 265 static enum ice_status
 266 ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 267 {
 268         enum ice_status status;
 269
 270         status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);
 271         if (status)
 272                 return status;
 273
 274         /* Update tail in the HW to post pre-allocated buffers */
 275         wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1));
 276
 277         return ICE_SUCCESS;
 278 }
 279
 280 /**
 281  * ice_init_sq - main initialization routine for Control ATQ
 282  * @hw: pointer to the hardware structure
 283  * @cq: pointer to the specific Control queue
 284  *
 285  * This is the main initialization routine for the Control Send Queue
 286  * Prior to calling this function, the driver *MUST* set the following fields
 287  * in the cq->structure:
 288  *     - cq->num_sq_entries
 289  *     - cq->sq_buf_size
 290  *
 291  * Do *NOT* hold the lock when calling this as the memory allocation routines
 292  * called are not going to be atomic context safe
 293  */
 294 static enum ice_status ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 295 {
 296         enum ice_status ret_code;
 297
 298         if (cq->sq.count > 0) {
 299                 /* queue already initialized */
 300                 ret_code = ICE_ERR_NOT_READY;
 301                 goto init_ctrlq_exit;
 302         }
 303
 304         /* verify input for valid configuration */
 305         if (!cq->num_sq_entries || !cq->sq_buf_size) {
 306                 ret_code = ICE_ERR_CFG;
 307                 goto init_ctrlq_exit;
 308         }
 309
 310         cq->sq.next_to_use = 0;
 311         cq->sq.next_to_clean = 0;
 312
 313         /* allocate the ring memory */
 314         ret_code = ice_alloc_ctrlq_sq_ring(hw, cq);
 315         if (ret_code)
 316                 goto init_ctrlq_exit;
 317
 318         /* allocate buffers in the rings */
 319         ret_code = ice_alloc_sq_bufs(hw, cq);
 320         if (ret_code)
 321                 goto init_ctrlq_free_rings;
 322
 323         /* initialize base registers */
 324         ret_code = ice_cfg_sq_regs(hw, cq);
 325         if (ret_code)
 326                 goto init_ctrlq_free_rings;
 327
 328         /* success! */
 329         cq->sq.count = cq->num_sq_entries;
 330         goto init_ctrlq_exit;
 331
 332 init_ctrlq_free_rings:
 333         ice_free_cq_ring(hw, &cq->sq);
 334
 335 init_ctrlq_exit:
 336         return ret_code;
 337 }
 338
 339 /**
 340  * ice_init_rq - initialize ARQ
 341  * @hw: pointer to the hardware structure
 342  * @cq: pointer to the specific Control queue
 343  *
 344  * The main initialization routine for the Admin Receive (Event) Queue.
 345  * Prior to calling this function, the driver *MUST* set the following fields
 346  * in the cq->structure:
 347  *     - cq->num_rq_entries
 348  *     - cq->rq_buf_size
 349  *
 350  * Do *NOT* hold the lock when calling this as the memory allocation routines
 351  * called are not going to be atomic context safe
 352  */
 353 static enum ice_status ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 354 {
 355         enum ice_status ret_code;
 356
 357         if (cq->rq.count > 0) {
 358                 /* queue already initialized */
 359                 ret_code = ICE_ERR_NOT_READY;
 360                 goto init_ctrlq_exit;
 361         }
 362
 363         /* verify input for valid configuration */
 364         if (!cq->num_rq_entries || !cq->rq_buf_size) {
 365                 ret_code = ICE_ERR_CFG;
 366                 goto init_ctrlq_exit;
 367         }
 368
 369         cq->rq.next_to_use = 0;
 370         cq->rq.next_to_clean = 0;
 371
 372         /* allocate the ring memory */
 373         ret_code = ice_alloc_ctrlq_rq_ring(hw, cq);
 374         if (ret_code)
 375                 goto init_ctrlq_exit;
 376
 377         /* allocate buffers in the rings */
 378         ret_code = ice_alloc_rq_bufs(hw, cq);
 379         if (ret_code)
 380                 goto init_ctrlq_free_rings;
 381
 382         /* initialize base registers */
 383         ret_code = ice_cfg_rq_regs(hw, cq);
 384         if (ret_code)
 385                 goto init_ctrlq_free_rings;
 386
 387         /* success! */
 388         cq->rq.count = cq->num_rq_entries;
 389         goto init_ctrlq_exit;
 390
 391 init_ctrlq_free_rings:
 392         ice_free_cq_ring(hw, &cq->rq);
 393
 394 init_ctrlq_exit:
 395         return ret_code;
 396 }
 397
 398 #define ICE_FREE_CQ_BUFS(hw, qi, ring)                                  \
 399 do {                                                                    \
 400         int i;                                                          \
 401         /* free descriptors */                                          \
 402         for (i = 0; i < (qi)->num_##ring##_entries; i++)                \
 403                 if ((qi)->ring.r.ring##_bi[i].pa)                       \
 404                         ice_free_dma_mem((hw),                          \
 405                                          &(qi)->ring.r.ring##_bi[i]);   \
 406         /* free the buffer info list */                                 \
 407         if ((qi)->ring.cmd_buf)                                         \
 408                 ice_free(hw, (qi)->ring.cmd_buf);                       \
 409         /* free dma head */                                             \
 410         ice_free(hw, (qi)->ring.dma_head);                              \
 411 } while (0)
 412
 413 /**
 414  * ice_shutdown_sq - shutdown the Control ATQ
 415  * @hw: pointer to the hardware structure
 416  * @cq: pointer to the specific Control queue
 417  *
 418  * The main shutdown routine for the Control Transmit Queue
 419  */
 420 static enum ice_status
 421 ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 422 {
 423         enum ice_status ret_code = ICE_SUCCESS;
 424
 425         ice_acquire_lock(&cq->sq_lock);
 426
 427         if (!cq->sq.count) {
 428                 ret_code = ICE_ERR_NOT_READY;
 429                 goto shutdown_sq_out;
 430         }
 431
 432         /* Stop firmware AdminQ processing */
 433         wr32(hw, cq->sq.head, 0);
 434         wr32(hw, cq->sq.tail, 0);
 435         wr32(hw, cq->sq.len, 0);
 436         wr32(hw, cq->sq.bal, 0);
 437         wr32(hw, cq->sq.bah, 0);
 438
 439         cq->sq.count = 0;       /* to indicate uninitialized queue */
 440
 441         /* free ring buffers and the ring itself */
 442         ICE_FREE_CQ_BUFS(hw, cq, sq);
 443         ice_free_cq_ring(hw, &cq->sq);
 444
 445 shutdown_sq_out:
 446         ice_release_lock(&cq->sq_lock);
 447         return ret_code;
 448 }
 449
 450 /**
 451  * ice_aq_ver_check - Check the reported AQ API version.
 452  * @hw: pointer to the hardware structure
 453  *
 454  * Checks if the driver should load on a given AQ API version.
 455  *
 456  * Return: 'true' iff the driver should attempt to load. 'false' otherwise.
 457  */
 458 static bool ice_aq_ver_check(struct ice_hw *hw)
 459 {
 460         if (hw->api_maj_ver > EXP_FW_API_VER_MAJOR) {
 461                 /* Major API version is newer than expected, don't load */
 462                 ice_warn(hw, "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
 463                 return false;
 464         } else if (hw->api_maj_ver == EXP_FW_API_VER_MAJOR) {
 465                 if (hw->api_min_ver > (EXP_FW_API_VER_MINOR + 2))
 466                         ice_info(hw, "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
 467                 else if ((hw->api_min_ver + 2) < EXP_FW_API_VER_MINOR)
 468                         ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
 469         } else {
 470                 /* Major API version is older than expected, log a warning */
 471                 ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
 472         }
 473         return true;
 474 }
 475
 476 /**
 477  * ice_shutdown_rq - shutdown Control ARQ
 478  * @hw: pointer to the hardware structure
 479  * @cq: pointer to the specific Control queue
 480  *
 481  * The main shutdown routine for the Control Receive Queue
 482  */
 483 static enum ice_status
 484 ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 485 {
 486         enum ice_status ret_code = ICE_SUCCESS;
 487
 488         ice_acquire_lock(&cq->rq_lock);
 489
 490         if (!cq->rq.count) {
 491                 ret_code = ICE_ERR_NOT_READY;
 492                 goto shutdown_rq_out;
 493         }
 494
 495         /* Stop Control Queue processing */
 496         wr32(hw, cq->rq.head, 0);
 497         wr32(hw, cq->rq.tail, 0);
 498         wr32(hw, cq->rq.len, 0);
 499         wr32(hw, cq->rq.bal, 0);
 500         wr32(hw, cq->rq.bah, 0);
 501
 502         /* set rq.count to 0 to indicate uninitialized queue */
 503         cq->rq.count = 0;
 504
 505         /* free ring buffers and the ring itself */
 506         ICE_FREE_CQ_BUFS(hw, cq, rq);
 507         ice_free_cq_ring(hw, &cq->rq);
 508
 509 shutdown_rq_out:
 510         ice_release_lock(&cq->rq_lock);
 511         return ret_code;
 512 }
 513
 514
 515 /**
 516  * ice_init_check_adminq - Check version for Admin Queue to know if its alive
 517  * @hw: pointer to the hardware structure
 518  */
 519 static enum ice_status ice_init_check_adminq(struct ice_hw *hw)
 520 {
 521         struct ice_ctl_q_info *cq = &hw->adminq;
 522         enum ice_status status;
 523
 524
 525         status = ice_aq_get_fw_ver(hw, NULL);
 526         if (status)
 527                 goto init_ctrlq_free_rq;
 528
 529
 530         if (!ice_aq_ver_check(hw)) {
 531                 status = ICE_ERR_FW_API_VER;
 532                 goto init_ctrlq_free_rq;
 533         }
 534
 535         return ICE_SUCCESS;
 536
 537 init_ctrlq_free_rq:
 538         ice_shutdown_rq(hw, cq);
 539         ice_shutdown_sq(hw, cq);
 540         return status;
 541 }
 542
 543 /**
 544  * ice_init_ctrlq - main initialization routine for any control Queue
 545  * @hw: pointer to the hardware structure
 546  * @q_type: specific Control queue type
 547  *
 548  * Prior to calling this function, the driver *MUST* set the following fields
 549  * in the cq->structure:
 550  *     - cq->num_sq_entries
 551  *     - cq->num_rq_entries
 552  *     - cq->rq_buf_size
 553  *     - cq->sq_buf_size
 554  *
 555  * NOTE: this function does not initialize the controlq locks
 556  */
 557 static enum ice_status ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 558 {
 559         struct ice_ctl_q_info *cq;
 560         enum ice_status ret_code;
 561
 562         switch (q_type) {
 563         case ICE_CTL_Q_ADMIN:
 564                 ice_adminq_init_regs(hw);
 565                 cq = &hw->adminq;
 566                 break;
 567         case ICE_CTL_Q_MAILBOX:
 568                 ice_mailbox_init_regs(hw);
 569                 cq = &hw->mailboxq;
 570                 break;
 571         default:
 572                 return ICE_ERR_PARAM;
 573         }
 574         cq->qtype = q_type;
 575
 576         /* verify input for valid configuration */
 577         if (!cq->num_rq_entries || !cq->num_sq_entries ||
 578             !cq->rq_buf_size || !cq->sq_buf_size) {
 579                 return ICE_ERR_CFG;
 580         }
 581
 582         /* setup SQ command write back timeout */
 583         cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;
 584
 585         /* allocate the ATQ */
 586         ret_code = ice_init_sq(hw, cq);
 587         if (ret_code)
 588                 return ret_code;
 589
 590         /* allocate the ARQ */
 591         ret_code = ice_init_rq(hw, cq);
 592         if (ret_code)
 593                 goto init_ctrlq_free_sq;
 594
 595         /* success! */
 596         return ICE_SUCCESS;
 597
 598 init_ctrlq_free_sq:
 599         ice_shutdown_sq(hw, cq);
 600         return ret_code;
 601 }
 602
 603 /**
 604  * ice_init_all_ctrlq - main initialization routine for all control queues
 605  * @hw: pointer to the hardware structure
 606  *
 607  * Prior to calling this function, the driver MUST* set the following fields
 608  * in the cq->structure for all control queues:
 609  *     - cq->num_sq_entries
 610  *     - cq->num_rq_entries
 611  *     - cq->rq_buf_size
 612  *     - cq->sq_buf_size
 613  *
 614  * NOTE: this function does not initialize the controlq locks.
 615  */
 616 enum ice_status ice_init_all_ctrlq(struct ice_hw *hw)
 617 {
 618         enum ice_status ret_code;
 619
 620
 621         /* Init FW admin queue */
 622         ret_code = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
 623         if (ret_code)
 624                 return ret_code;
 625
 626         ret_code = ice_init_check_adminq(hw);
 627         if (ret_code)
 628                 return ret_code;
 629         /* Init Mailbox queue */
 630         return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);
 631 }
 632
 633 /**
 634  * ice_init_ctrlq_locks - Initialize locks for a control queue
 635  * @cq: pointer to the control queue
 636  *
 637  * Initializes the send and receive queue locks for a given control queue.
 638  */
 639 static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq)
 640 {
 641         ice_init_lock(&cq->sq_lock);
 642         ice_init_lock(&cq->rq_lock);
 643 }
 644
 645 /**
 646  * ice_create_all_ctrlq - main initialization routine for all control queues
 647  * @hw: pointer to the hardware structure
 648  *
 649  * Prior to calling this function, the driver *MUST* set the following fields
 650  * in the cq->structure for all control queues:
 651  *     - cq->num_sq_entries
 652  *     - cq->num_rq_entries
 653  *     - cq->rq_buf_size
 654  *     - cq->sq_buf_size
 655  *
 656  * This function creates all the control queue locks and then calls
 657  * ice_init_all_ctrlq. It should be called once during driver load. If the
 658  * driver needs to re-initialize control queues at run time it should call
 659  * ice_init_all_ctrlq instead.
 660  */
 661 enum ice_status ice_create_all_ctrlq(struct ice_hw *hw)
 662 {
 663         ice_init_ctrlq_locks(&hw->adminq);
 664         ice_init_ctrlq_locks(&hw->mailboxq);
 665
 666         return ice_init_all_ctrlq(hw);
 667 }
 668
 669 /**
 670  * ice_shutdown_ctrlq - shutdown routine for any control queue
 671  * @hw: pointer to the hardware structure
 672  * @q_type: specific Control queue type
 673  *
 674  * NOTE: this function does not destroy the control queue locks.
 675  */
 676 static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 677 {
 678         struct ice_ctl_q_info *cq;
 679
 680         switch (q_type) {
 681         case ICE_CTL_Q_ADMIN:
 682                 cq = &hw->adminq;
 683                 if (ice_check_sq_alive(hw, cq))
 684                         ice_aq_q_shutdown(hw, true);
 685                 break;
 686         case ICE_CTL_Q_MAILBOX:
 687                 cq = &hw->mailboxq;
 688                 break;
 689         default:
 690                 return;
 691         }
 692
 693         ice_shutdown_sq(hw, cq);
 694         ice_shutdown_rq(hw, cq);
 695 }
 696
 697 /**
 698  * ice_shutdown_all_ctrlq - shutdown routine for all control queues
 699  * @hw: pointer to the hardware structure
 700  *
 701  * NOTE: this function does not destroy the control queue locks. The driver
 702  * may call this at runtime to shutdown and later restart control queues, such
 703  * as in response to a reset event.
 704  */
 705 void ice_shutdown_all_ctrlq(struct ice_hw *hw)
 706 {
 707         /* Shutdown FW admin queue */
 708         ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
 709         /* Shutdown PF-VF Mailbox */
 710         ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX);
 711 }
 712
 713 /**
 714  * ice_destroy_ctrlq_locks - Destroy locks for a control queue
 715  * @cq: pointer to the control queue
 716  *
 717  * Destroys the send and receive queue locks for a given control queue.
 718  */
 719 static void
 720 ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq)
 721 {
 722         ice_destroy_lock(&cq->sq_lock);
 723         ice_destroy_lock(&cq->rq_lock);
 724 }
 725
 726 /**
 727  * ice_destroy_all_ctrlq - exit routine for all control queues
 728  * @hw: pointer to the hardware structure
 729  *
 730  * This function shuts down all the control queues and then destroys the
 731  * control queue locks. It should be called once during driver unload. The
 732  * driver should call ice_shutdown_all_ctrlq if it needs to shut down and
 733  * reinitialize control queues, such as in response to a reset event.
 734  */
 735 void ice_destroy_all_ctrlq(struct ice_hw *hw)
 736 {
 737         /* shut down all the control queues first */
 738         ice_shutdown_all_ctrlq(hw);
 739
 740         ice_destroy_ctrlq_locks(&hw->adminq);
 741         ice_destroy_ctrlq_locks(&hw->mailboxq);
 742 }
 743
 744 /**
 745  * ice_clean_sq - cleans Admin send queue (ATQ)
 746  * @hw: pointer to the hardware structure
 747  * @cq: pointer to the specific Control queue
 748  *
 749  * returns the number of free desc
 750  */
 751 static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 752 {
 753         struct ice_ctl_q_ring *sq = &cq->sq;
 754         u16 ntc = sq->next_to_clean;
 755         struct ice_sq_cd *details;
 756 #if 0
 757         struct ice_aq_desc desc_cb;
 758 #endif
 759         struct ice_aq_desc *desc;
 760
 761         desc = ICE_CTL_Q_DESC(*sq, ntc);
 762         details = ICE_CTL_Q_DETAILS(*sq, ntc);
 763
 764         while (rd32(hw, cq->sq.head) != ntc) {
 765                 ice_debug(hw, ICE_DBG_AQ_MSG,
 766                           "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head));
 767 #if 0
 768                 if (details->callback) {
 769                         ICE_CTL_Q_CALLBACK cb_func =
 770                                 (ICE_CTL_Q_CALLBACK)details->callback;
 771                         ice_memcpy(&desc_cb, desc, sizeof(desc_cb),
 772                                    ICE_DMA_TO_DMA);
 773                         cb_func(hw, &desc_cb);
 774                 }
 775 #endif
 776                 ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
 777                 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
 778                 ntc++;
 779                 if (ntc == sq->count)
 780                         ntc = 0;
 781                 desc = ICE_CTL_Q_DESC(*sq, ntc);
 782                 details = ICE_CTL_Q_DETAILS(*sq, ntc);
 783         }
 784
 785         sq->next_to_clean = ntc;
 786
 787         return ICE_CTL_Q_DESC_UNUSED(sq);
 788 }
 789
 790 /**
 791  * ice_debug_cq
 792  * @hw: pointer to the hardware structure
 793  * @desc: pointer to control queue descriptor
 794  * @buf: pointer to command buffer
 795  * @buf_len: max length of buf
 796  *
 797  * Dumps debug log about control command with descriptor contents.
 798  */
 799 static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len)
 800 {
 801         struct ice_aq_desc *cq_desc = (struct ice_aq_desc *)desc;
 802         u16 datalen, flags;
 803
 804         if (!((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask))
 805                 return;
 806
 807         if (!desc)
 808                 return;
 809
 810         datalen = LE16_TO_CPU(cq_desc->datalen);
 811         flags = LE16_TO_CPU(cq_desc->flags);
 812
 813         ice_debug(hw, ICE_DBG_AQ_DESC,
 814                   "CQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n",
 815                   LE16_TO_CPU(cq_desc->opcode), flags, datalen,
 816                   LE16_TO_CPU(cq_desc->retval));
 817         ice_debug(hw, ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n",
 818                   LE32_TO_CPU(cq_desc->cookie_high),
 819                   LE32_TO_CPU(cq_desc->cookie_low));
 820         ice_debug(hw, ICE_DBG_AQ_DESC, "\tparam (0,1)  0x%08X 0x%08X\n",
 821                   LE32_TO_CPU(cq_desc->params.generic.param0),
 822                   LE32_TO_CPU(cq_desc->params.generic.param1));
 823         ice_debug(hw, ICE_DBG_AQ_DESC, "\taddr (h,l)   0x%08X 0x%08X\n",
 824                   LE32_TO_CPU(cq_desc->params.generic.addr_high),
 825                   LE32_TO_CPU(cq_desc->params.generic.addr_low));
 826         /* Dump buffer iff 1) one exists and 2) is either a response indicated
 827          * by the DD and/or CMP flag set or a command with the RD flag set.
 828          */
 829         if (buf && cq_desc->datalen != 0 &&
 830             (flags & (ICE_AQ_FLAG_DD | ICE_AQ_FLAG_CMP) ||
 831              flags & ICE_AQ_FLAG_RD)) {
 832                 ice_debug(hw, ICE_DBG_AQ_DESC_BUF, "Buffer:\n");
 833                 ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, (u8 *)buf,
 834                                 min(buf_len, datalen));
 835         }
 836 }
 837
 838 /**
 839  * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ)
 840  * @hw: pointer to the HW struct
 841  * @cq: pointer to the specific Control queue
 842  *
 843  * Returns true if the firmware has processed all descriptors on the
 844  * admin send queue. Returns false if there are still requests pending.
 845  */
 846 static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 847 {
 848         /* AQ designers suggest use of head for better
 849          * timing reliability than DD bit
 850          */
 851         return rd32(hw, cq->sq.head) == cq->sq.next_to_use;
 852 }
 853
 854 /**
 855  * ice_sq_send_cmd - send command to Control Queue (ATQ)
 856  * @hw: pointer to the HW struct
 857  * @cq: pointer to the specific Control queue
 858  * @desc: prefilled descriptor describing the command (non DMA mem)
 859  * @buf: buffer to use for indirect commands (or NULL for direct commands)
 860  * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
 861  * @cd: pointer to command details structure
 862  *
 863  * This is the main send command routine for the ATQ. It runs the queue,
 864  * cleans the queue, etc.
 865  */
 866 enum ice_status
 867 ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 868                 struct ice_aq_desc *desc, void *buf, u16 buf_size,
 869                 struct ice_sq_cd *cd)
 870 {
 871         struct ice_dma_mem *dma_buf = NULL;
 872         struct ice_aq_desc *desc_on_ring;
 873         bool cmd_completed = false;
 874         enum ice_status status = ICE_SUCCESS;
 875         struct ice_sq_cd *details;
 876         u32 total_delay = 0;
 877         u16 retval = 0;
 878         u32 val = 0;
 879
 880         /* if reset is in progress return a soft error */
 881         if (hw->reset_ongoing)
 882                 return ICE_ERR_RESET_ONGOING;
 883         ice_acquire_lock(&cq->sq_lock);
 884
 885         cq->sq_last_status = ICE_AQ_RC_OK;
 886
 887         if (!cq->sq.count) {
 888                 ice_debug(hw, ICE_DBG_AQ_MSG,
 889                           "Control Send queue not initialized.\n");
 890                 status = ICE_ERR_AQ_EMPTY;
 891                 goto sq_send_command_error;
 892         }
 893
 894         if ((buf && !buf_size) || (!buf && buf_size)) {
 895                 status = ICE_ERR_PARAM;
 896                 goto sq_send_command_error;
 897         }
 898
 899         if (buf) {
 900                 if (buf_size > cq->sq_buf_size) {
 901                         ice_debug(hw, ICE_DBG_AQ_MSG,
 902                                   "Invalid buffer size for Control Send queue: %d.\n",
 903                                   buf_size);
 904                         status = ICE_ERR_INVAL_SIZE;
 905                         goto sq_send_command_error;
 906                 }
 907
 908                 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_BUF);
 909                 if (buf_size > ICE_AQ_LG_BUF)
 910                         desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
 911         }
 912
 913         val = rd32(hw, cq->sq.head);
 914         if (val >= cq->num_sq_entries) {
 915                 ice_debug(hw, ICE_DBG_AQ_MSG,
 916                           "head overrun at %d in the Control Send Queue ring\n",
 917                           val);
 918                 status = ICE_ERR_AQ_EMPTY;
 919                 goto sq_send_command_error;
 920         }
 921
 922         details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use);
 923         if (cd)
 924                 *details = *cd;
 925 #if 0
 926                 /* FIXME: if/when this block gets enabled (when the #if 0
 927                  * is removed), add braces to both branches of the surrounding
 928                  * conditional expression. The braces have been removed to
 929                  * prevent checkpatch complaining.
 930                  */
 931
 932                 /* If the command details are defined copy the cookie. The
 933                  * CPU_TO_LE32 is not needed here because the data is ignored
 934                  * by the FW, only used by the driver
 935                  */
 936                 if (details->cookie) {
 937                         desc->cookie_high =
 938                                 CPU_TO_LE32(ICE_HI_DWORD(details->cookie));
 939                         desc->cookie_low =
 940                                 CPU_TO_LE32(ICE_LO_DWORD(details->cookie));
 941                 }
 942 #endif
 943         else
 944                 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
 945 #if 0
 946         /* clear requested flags and then set additional flags if defined */
 947         desc->flags &= ~CPU_TO_LE16(details->flags_dis);
 948         desc->flags |= CPU_TO_LE16(details->flags_ena);
 949
 950         if (details->postpone && !details->async) {
 951                 ice_debug(hw, ICE_DBG_AQ_MSG,
 952                           "Async flag not set along with postpone flag\n");
 953                 status = ICE_ERR_PARAM;
 954                 goto sq_send_command_error;
 955         }
 956 #endif
 957
 958         /* Call clean and check queue available function to reclaim the
 959          * descriptors that were processed by FW/MBX; the function returns the
 960          * number of desc available. The clean function called here could be
 961          * called in a separate thread in case of asynchronous completions.
 962          */
 963         if (ice_clean_sq(hw, cq) == 0) {
 964                 ice_debug(hw, ICE_DBG_AQ_MSG,
 965                           "Error: Control Send Queue is full.\n");
 966                 status = ICE_ERR_AQ_FULL;
 967                 goto sq_send_command_error;
 968         }
 969
 970         /* initialize the temp desc pointer with the right desc */
 971         desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use);
 972
 973         /* if the desc is available copy the temp desc to the right place */
 974         ice_memcpy(desc_on_ring, desc, sizeof(*desc_on_ring),
 975                    ICE_NONDMA_TO_DMA);
 976
 977         /* if buf is not NULL assume indirect command */
 978         if (buf) {
 979                 dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use];
 980                 /* copy the user buf into the respective DMA buf */
 981                 ice_memcpy(dma_buf->va, buf, buf_size, ICE_NONDMA_TO_DMA);
 982                 desc_on_ring->datalen = CPU_TO_LE16(buf_size);
 983
 984                 /* Update the address values in the desc with the pa value
 985                  * for respective buffer
 986                  */
 987                 desc_on_ring->params.generic.addr_high =
 988                         CPU_TO_LE32(ICE_HI_DWORD(dma_buf->pa));
 989                 desc_on_ring->params.generic.addr_low =
 990                         CPU_TO_LE32(ICE_LO_DWORD(dma_buf->pa));
 991         }
 992
 993         /* Debug desc and buffer */
 994         ice_debug(hw, ICE_DBG_AQ_MSG,
 995                   "ATQ: Control Send queue desc and buffer:\n");
 996
 997         ice_debug_cq(hw, (void *)desc_on_ring, buf, buf_size);
 998
 999
1000         (cq->sq.next_to_use)++;
1001         if (cq->sq.next_to_use == cq->sq.count)
1002                 cq->sq.next_to_use = 0;
1003 #if 0
1004         /* FIXME - handle this case? */
1005         if (!details->postpone)
1006 #endif
1007         wr32(hw, cq->sq.tail, cq->sq.next_to_use);
1008
1009 #if 0
1010         /* if command details are not defined or async flag is not set,
1011          * we need to wait for desc write back
1012          */
1013         if (!details->async && !details->postpone) {
1014                 /* FIXME - handle this case? */
1015         }
1016 #endif
1017         do {
1018                 if (ice_sq_done(hw, cq))
1019                         break;
1020
1021                 ice_msec_delay(1, false);
1022                 total_delay++;
1023         } while (total_delay < cq->sq_cmd_timeout);
1024
1025         /* if ready, copy the desc back to temp */
1026         if (ice_sq_done(hw, cq)) {
1027                 ice_memcpy(desc, desc_on_ring, sizeof(*desc),
1028                            ICE_DMA_TO_NONDMA);
1029                 if (buf) {
1030                         /* get returned length to copy */
1031                         u16 copy_size = LE16_TO_CPU(desc->datalen);
1032
1033                         if (copy_size > buf_size) {
1034                                 ice_debug(hw, ICE_DBG_AQ_MSG,
1035                                           "Return len %d > than buf len %d\n",
1036                                           copy_size, buf_size);
1037                                 status = ICE_ERR_AQ_ERROR;
1038                         } else {
1039                                 ice_memcpy(buf, dma_buf->va, copy_size,
1040                                            ICE_DMA_TO_NONDMA);
1041                         }
1042                 }
1043                 retval = LE16_TO_CPU(desc->retval);
1044                 if (retval) {
1045                         ice_debug(hw, ICE_DBG_AQ_MSG,
1046                                   "Control Send Queue command completed with error 0x%x\n",
1047                                   retval);
1048
1049                         /* strip off FW internal code */
1050                         retval &= 0xff;
1051                 }
1052                 cmd_completed = true;
1053                 if (!status && retval != ICE_AQ_RC_OK)
1054                         status = ICE_ERR_AQ_ERROR;
1055                 cq->sq_last_status = (enum ice_aq_err)retval;
1056         }
1057
1058         ice_debug(hw, ICE_DBG_AQ_MSG,
1059                   "ATQ: desc and buffer writeback:\n");
1060
1061         ice_debug_cq(hw, (void *)desc, buf, buf_size);
1062
1063
1064         /* save writeback AQ if requested */
1065         if (details->wb_desc)
1066                 ice_memcpy(details->wb_desc, desc_on_ring,
1067                            sizeof(*details->wb_desc), ICE_DMA_TO_NONDMA);
1068
1069         /* update the error if time out occurred */
1070         if (!cmd_completed) {
1071 #if 0
1072             (!details->async && !details->postpone)) {
1073 #endif
1074                 ice_debug(hw, ICE_DBG_AQ_MSG,
1075                           "Control Send Queue Writeback timeout.\n");
1076                 status = ICE_ERR_AQ_TIMEOUT;
1077         }
1078
1079 sq_send_command_error:
1080         ice_release_lock(&cq->sq_lock);
1081         return status;
1082 }
1083
1084 /**
1085  * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function
1086  * @desc: pointer to the temp descriptor (non DMA mem)
1087  * @opcode: the opcode can be used to decide which flags to turn off or on
1088  *
1089  * Fill the desc with default values
1090  */
1091 void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode)
1092 {
1093         /* zero out the desc */
1094         ice_memset(desc, 0, sizeof(*desc), ICE_NONDMA_MEM);
1095         desc->opcode = CPU_TO_LE16(opcode);
1096         desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_SI);
1097 }
1098
1099 /**
1100  * ice_clean_rq_elem
1101  * @hw: pointer to the HW struct
1102  * @cq: pointer to the specific Control queue
1103  * @e: event info from the receive descriptor, includes any buffers
1104  * @pending: number of events that could be left to process
1105  *
1106  * This function cleans one Admin Receive Queue element and returns
1107  * the contents through e. It can also return how many events are
1108  * left to process through 'pending'.
1109  */
1110 enum ice_status
1111 ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq,
1112                   struct ice_rq_event_info *e, u16 *pending)
1113 {
1114         u16 ntc = cq->rq.next_to_clean;
1115         enum ice_status ret_code = ICE_SUCCESS;
1116         struct ice_aq_desc *desc;
1117         struct ice_dma_mem *bi;
1118         u16 desc_idx;
1119         u16 datalen;
1120         u16 flags;
1121         u16 ntu;
1122
1123         /* pre-clean the event info */
1124         ice_memset(&e->desc, 0, sizeof(e->desc), ICE_NONDMA_MEM);
1125
1126         /* take the lock before we start messing with the ring */
1127         ice_acquire_lock(&cq->rq_lock);
1128
1129         if (!cq->rq.count) {
1130                 ice_debug(hw, ICE_DBG_AQ_MSG,
1131                           "Control Receive queue not initialized.\n");
1132                 ret_code = ICE_ERR_AQ_EMPTY;
1133                 goto clean_rq_elem_err;
1134         }
1135
1136         /* set next_to_use to head */
1137         ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1138
1139         if (ntu == ntc) {
1140                 /* nothing to do - shouldn't need to update ring's values */
1141                 ret_code = ICE_ERR_AQ_NO_WORK;
1142                 goto clean_rq_elem_out;
1143         }
1144
1145         /* now clean the next descriptor */
1146         desc = ICE_CTL_Q_DESC(cq->rq, ntc);
1147         desc_idx = ntc;
1148
1149         cq->rq_last_status = (enum ice_aq_err)LE16_TO_CPU(desc->retval);
1150         flags = LE16_TO_CPU(desc->flags);
1151         if (flags & ICE_AQ_FLAG_ERR) {
1152                 ret_code = ICE_ERR_AQ_ERROR;
1153                 ice_debug(hw, ICE_DBG_AQ_MSG,
1154                           "Control Receive Queue Event received with error 0x%x\n",
1155                           cq->rq_last_status);
1156         }
1157         ice_memcpy(&e->desc, desc, sizeof(e->desc), ICE_DMA_TO_NONDMA);
1158         datalen = LE16_TO_CPU(desc->datalen);
1159         e->msg_len = min(datalen, e->buf_len);
1160         if (e->msg_buf && e->msg_len)
1161                 ice_memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va,
1162                            e->msg_len, ICE_DMA_TO_NONDMA);
1163
1164         ice_debug(hw, ICE_DBG_AQ_MSG, "ARQ: desc and buffer:\n");
1165
1166         ice_debug_cq(hw, (void *)desc, e->msg_buf,
1167                      cq->rq_buf_size);
1168
1169
1170         /* Restore the original datalen and buffer address in the desc,
1171          * FW updates datalen to indicate the event message size
1172          */
1173         bi = &cq->rq.r.rq_bi[ntc];
1174         ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
1175
1176         desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
1177         if (cq->rq_buf_size > ICE_AQ_LG_BUF)
1178                 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
1179         desc->datalen = CPU_TO_LE16(bi->size);
1180         desc->params.generic.addr_high = CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
1181         desc->params.generic.addr_low = CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
1182
1183         /* set tail = the last cleaned desc index. */
1184         wr32(hw, cq->rq.tail, ntc);
1185         /* ntc is updated to tail + 1 */
1186         ntc++;
1187         if (ntc == cq->num_rq_entries)
1188                 ntc = 0;
1189         cq->rq.next_to_clean = ntc;
1190         cq->rq.next_to_use = ntu;
1191
1192 #if 0
1193         ice_nvmupd_check_wait_event(hw, LE16_TO_CPU(e->desc.opcode));
1194 #endif
1195 clean_rq_elem_out:
1196         /* Set pending if needed, unlock and return */
1197         if (pending) {
1198                 /* re-read HW head to calculate actual pending messages */
1199                 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1200                 *pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc));
1201         }
1202 clean_rq_elem_err:
1203         ice_release_lock(&cq->rq_lock);
1204
1205         return ret_code;
1206 }