drivers/net/ice/base/ice_controlq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2001-2019
   3  */
   4
   5 #include "ice_common.h"
   6
   7
   8 #define ICE_CQ_INIT_REGS(qinfo, prefix)                         \
   9 do {                                                            \
  10         (qinfo)->sq.head = prefix##_ATQH;                       \
  11         (qinfo)->sq.tail = prefix##_ATQT;                       \
  12         (qinfo)->sq.len = prefix##_ATQLEN;                      \
  13         (qinfo)->sq.bah = prefix##_ATQBAH;                      \
  14         (qinfo)->sq.bal = prefix##_ATQBAL;                      \
  15         (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M;        \
  16         (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \
  17         (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M;           \
  18         (qinfo)->rq.head = prefix##_ARQH;                       \
  19         (qinfo)->rq.tail = prefix##_ARQT;                       \
  20         (qinfo)->rq.len = prefix##_ARQLEN;                      \
  21         (qinfo)->rq.bah = prefix##_ARQBAH;                      \
  22         (qinfo)->rq.bal = prefix##_ARQBAL;                      \
  23         (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M;        \
  24         (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \
  25         (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M;           \
  26 } while (0)
  27
  28 /**
  29  * ice_adminq_init_regs - Initialize AdminQ registers
  30  * @hw: pointer to the hardware structure
  31  *
  32  * This assumes the alloc_sq and alloc_rq functions have already been called
  33  */
  34 static void ice_adminq_init_regs(struct ice_hw *hw)
  35 {
  36         struct ice_ctl_q_info *cq = &hw->adminq;
  37
  38         ICE_CQ_INIT_REGS(cq, PF_FW);
  39 }
  40
  41 /**
  42  * ice_mailbox_init_regs - Initialize Mailbox registers
  43  * @hw: pointer to the hardware structure
  44  *
  45  * This assumes the alloc_sq and alloc_rq functions have already been called
  46  */
  47 static void ice_mailbox_init_regs(struct ice_hw *hw)
  48 {
  49         struct ice_ctl_q_info *cq = &hw->mailboxq;
  50
  51         ICE_CQ_INIT_REGS(cq, PF_MBX);
  52 }
  53
  54
  55 /**
  56  * ice_check_sq_alive
  57  * @hw: pointer to the HW struct
  58  * @cq: pointer to the specific Control queue
  59  *
  60  * Returns true if Queue is enabled else false.
  61  */
  62 bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  63 {
  64         /* check both queue-length and queue-enable fields */
  65         if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask)
  66                 return (rd32(hw, cq->sq.len) & (cq->sq.len_mask |
  67                                                 cq->sq.len_ena_mask)) ==
  68                         (cq->num_sq_entries | cq->sq.len_ena_mask);
  69
  70         return false;
  71 }
  72
  73 /**
  74  * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings
  75  * @hw: pointer to the hardware structure
  76  * @cq: pointer to the specific Control queue
  77  */
  78 static enum ice_status
  79 ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  80 {
  81         size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc);
  82
  83         cq->sq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->sq.desc_buf, size);
  84         if (!cq->sq.desc_buf.va)
  85                 return ICE_ERR_NO_MEMORY;
  86
  87         cq->sq.cmd_buf = ice_calloc(hw, cq->num_sq_entries,
  88                                     sizeof(struct ice_sq_cd));
  89         if (!cq->sq.cmd_buf) {
  90                 ice_free_dma_mem(hw, &cq->sq.desc_buf);
  91                 return ICE_ERR_NO_MEMORY;
  92         }
  93
  94         return ICE_SUCCESS;
  95 }
  96
  97 /**
  98  * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings
  99  * @hw: pointer to the hardware structure
 100  * @cq: pointer to the specific Control queue
 101  */
 102 static enum ice_status
 103 ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 104 {
 105         size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc);
 106
 107         cq->rq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->rq.desc_buf, size);
 108         if (!cq->rq.desc_buf.va)
 109                 return ICE_ERR_NO_MEMORY;
 110         return ICE_SUCCESS;
 111 }
 112
 113 /**
 114  * ice_free_cq_ring - Free control queue ring
 115  * @hw: pointer to the hardware structure
 116  * @ring: pointer to the specific control queue ring
 117  *
 118  * This assumes the posted buffers have already been cleaned
 119  * and de-allocated
 120  */
 121 static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)
 122 {
 123         ice_free_dma_mem(hw, &ring->desc_buf);
 124 }
 125
 126 /**
 127  * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ
 128  * @hw: pointer to the hardware structure
 129  * @cq: pointer to the specific Control queue
 130  */
 131 static enum ice_status
 132 ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 133 {
 134         int i;
 135
 136         /* We'll be allocating the buffer info memory first, then we can
 137          * allocate the mapped buffers for the event processing
 138          */
 139         cq->rq.dma_head = ice_calloc(hw, cq->num_rq_entries,
 140                                      sizeof(cq->rq.desc_buf));
 141         if (!cq->rq.dma_head)
 142                 return ICE_ERR_NO_MEMORY;
 143         cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head;
 144
 145         /* allocate the mapped buffers */
 146         for (i = 0; i < cq->num_rq_entries; i++) {
 147                 struct ice_aq_desc *desc;
 148                 struct ice_dma_mem *bi;
 149
 150                 bi = &cq->rq.r.rq_bi[i];
 151                 bi->va = ice_alloc_dma_mem(hw, bi, cq->rq_buf_size);
 152                 if (!bi->va)
 153                         goto unwind_alloc_rq_bufs;
 154
 155                 /* now configure the descriptors for use */
 156                 desc = ICE_CTL_Q_DESC(cq->rq, i);
 157
 158                 desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
 159                 if (cq->rq_buf_size > ICE_AQ_LG_BUF)
 160                         desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
 161                 desc->opcode = 0;
 162                 /* This is in accordance with Admin queue design, there is no
 163                  * register for buffer size configuration
 164                  */
 165                 desc->datalen = CPU_TO_LE16(bi->size);
 166                 desc->retval = 0;
 167                 desc->cookie_high = 0;
 168                 desc->cookie_low = 0;
 169                 desc->params.generic.addr_high =
 170                         CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
 171                 desc->params.generic.addr_low =
 172                         CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
 173                 desc->params.generic.param0 = 0;
 174                 desc->params.generic.param1 = 0;
 175         }
 176         return ICE_SUCCESS;
 177
 178 unwind_alloc_rq_bufs:
 179         /* don't try to free the one that failed... */
 180         i--;
 181         for (; i >= 0; i--)
 182                 ice_free_dma_mem(hw, &cq->rq.r.rq_bi[i]);
 183         ice_free(hw, cq->rq.dma_head);
 184
 185         return ICE_ERR_NO_MEMORY;
 186 }
 187
 188 /**
 189  * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ
 190  * @hw: pointer to the hardware structure
 191  * @cq: pointer to the specific Control queue
 192  */
 193 static enum ice_status
 194 ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 195 {
 196         int i;
 197
 198         /* No mapped memory needed yet, just the buffer info structures */
 199         cq->sq.dma_head = ice_calloc(hw, cq->num_sq_entries,
 200                                      sizeof(cq->sq.desc_buf));
 201         if (!cq->sq.dma_head)
 202                 return ICE_ERR_NO_MEMORY;
 203         cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head;
 204
 205         /* allocate the mapped buffers */
 206         for (i = 0; i < cq->num_sq_entries; i++) {
 207                 struct ice_dma_mem *bi;
 208
 209                 bi = &cq->sq.r.sq_bi[i];
 210                 bi->va = ice_alloc_dma_mem(hw, bi, cq->sq_buf_size);
 211                 if (!bi->va)
 212                         goto unwind_alloc_sq_bufs;
 213         }
 214         return ICE_SUCCESS;
 215
 216 unwind_alloc_sq_bufs:
 217         /* don't try to free the one that failed... */
 218         i--;
 219         for (; i >= 0; i--)
 220                 ice_free_dma_mem(hw, &cq->sq.r.sq_bi[i]);
 221         ice_free(hw, cq->sq.dma_head);
 222
 223         return ICE_ERR_NO_MEMORY;
 224 }
 225
 226 static enum ice_status
 227 ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries)
 228 {
 229         /* Clear Head and Tail */
 230         wr32(hw, ring->head, 0);
 231         wr32(hw, ring->tail, 0);
 232
 233         /* set starting point */
 234         wr32(hw, ring->len, (num_entries | ring->len_ena_mask));
 235         wr32(hw, ring->bal, ICE_LO_DWORD(ring->desc_buf.pa));
 236         wr32(hw, ring->bah, ICE_HI_DWORD(ring->desc_buf.pa));
 237
 238         /* Check one register to verify that config was applied */
 239         if (rd32(hw, ring->bal) != ICE_LO_DWORD(ring->desc_buf.pa))
 240                 return ICE_ERR_AQ_ERROR;
 241
 242         return ICE_SUCCESS;
 243 }
 244
 245 /**
 246  * ice_cfg_sq_regs - configure Control ATQ registers
 247  * @hw: pointer to the hardware structure
 248  * @cq: pointer to the specific Control queue
 249  *
 250  * Configure base address and length registers for the transmit queue
 251  */
 252 static enum ice_status
 253 ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 254 {
 255         return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);
 256 }
 257
 258 /**
 259  * ice_cfg_rq_regs - configure Control ARQ register
 260  * @hw: pointer to the hardware structure
 261  * @cq: pointer to the specific Control queue
 262  *
 263  * Configure base address and length registers for the receive (event q)
 264  */
 265 static enum ice_status
 266 ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 267 {
 268         enum ice_status status;
 269
 270         status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);
 271         if (status)
 272                 return status;
 273
 274         /* Update tail in the HW to post pre-allocated buffers */
 275         wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1));
 276
 277         return ICE_SUCCESS;
 278 }
 279
 280 /**
 281  * ice_init_sq - main initialization routine for Control ATQ
 282  * @hw: pointer to the hardware structure
 283  * @cq: pointer to the specific Control queue
 284  *
 285  * This is the main initialization routine for the Control Send Queue
 286  * Prior to calling this function, drivers *MUST* set the following fields
 287  * in the cq->structure:
 288  *     - cq->num_sq_entries
 289  *     - cq->sq_buf_size
 290  *
 291  * Do *NOT* hold the lock when calling this as the memory allocation routines
 292  * called are not going to be atomic context safe
 293  */
 294 static enum ice_status ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 295 {
 296         enum ice_status ret_code;
 297
 298         if (cq->sq.count > 0) {
 299                 /* queue already initialized */
 300                 ret_code = ICE_ERR_NOT_READY;
 301                 goto init_ctrlq_exit;
 302         }
 303
 304         /* verify input for valid configuration */
 305         if (!cq->num_sq_entries || !cq->sq_buf_size) {
 306                 ret_code = ICE_ERR_CFG;
 307                 goto init_ctrlq_exit;
 308         }
 309
 310         cq->sq.next_to_use = 0;
 311         cq->sq.next_to_clean = 0;
 312
 313         /* allocate the ring memory */
 314         ret_code = ice_alloc_ctrlq_sq_ring(hw, cq);
 315         if (ret_code)
 316                 goto init_ctrlq_exit;
 317
 318         /* allocate buffers in the rings */
 319         ret_code = ice_alloc_sq_bufs(hw, cq);
 320         if (ret_code)
 321                 goto init_ctrlq_free_rings;
 322
 323         /* initialize base registers */
 324         ret_code = ice_cfg_sq_regs(hw, cq);
 325         if (ret_code)
 326                 goto init_ctrlq_free_rings;
 327
 328         /* success! */
 329         cq->sq.count = cq->num_sq_entries;
 330         goto init_ctrlq_exit;
 331
 332 init_ctrlq_free_rings:
 333         ice_free_cq_ring(hw, &cq->sq);
 334
 335 init_ctrlq_exit:
 336         return ret_code;
 337 }
 338
 339 /**
 340  * ice_init_rq - initialize ARQ
 341  * @hw: pointer to the hardware structure
 342  * @cq: pointer to the specific Control queue
 343  *
 344  * The main initialization routine for the Admin Receive (Event) Queue.
 345  * Prior to calling this function, drivers *MUST* set the following fields
 346  * in the cq->structure:
 347  *     - cq->num_rq_entries
 348  *     - cq->rq_buf_size
 349  *
 350  * Do *NOT* hold the lock when calling this as the memory allocation routines
 351  * called are not going to be atomic context safe
 352  */
 353 static enum ice_status ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 354 {
 355         enum ice_status ret_code;
 356
 357         if (cq->rq.count > 0) {
 358                 /* queue already initialized */
 359                 ret_code = ICE_ERR_NOT_READY;
 360                 goto init_ctrlq_exit;
 361         }
 362
 363         /* verify input for valid configuration */
 364         if (!cq->num_rq_entries || !cq->rq_buf_size) {
 365                 ret_code = ICE_ERR_CFG;
 366                 goto init_ctrlq_exit;
 367         }
 368
 369         cq->rq.next_to_use = 0;
 370         cq->rq.next_to_clean = 0;
 371
 372         /* allocate the ring memory */
 373         ret_code = ice_alloc_ctrlq_rq_ring(hw, cq);
 374         if (ret_code)
 375                 goto init_ctrlq_exit;
 376
 377         /* allocate buffers in the rings */
 378         ret_code = ice_alloc_rq_bufs(hw, cq);
 379         if (ret_code)
 380                 goto init_ctrlq_free_rings;
 381
 382         /* initialize base registers */
 383         ret_code = ice_cfg_rq_regs(hw, cq);
 384         if (ret_code)
 385                 goto init_ctrlq_free_rings;
 386
 387         /* success! */
 388         cq->rq.count = cq->num_rq_entries;
 389         goto init_ctrlq_exit;
 390
 391 init_ctrlq_free_rings:
 392         ice_free_cq_ring(hw, &cq->rq);
 393
 394 init_ctrlq_exit:
 395         return ret_code;
 396 }
 397
 398 #define ICE_FREE_CQ_BUFS(hw, qi, ring)                                  \
 399 do {                                                                    \
 400         int i;                                                          \
 401         /* free descriptors */                                          \
 402         for (i = 0; i < (qi)->num_##ring##_entries; i++)                \
 403                 if ((qi)->ring.r.ring##_bi[i].pa)                       \
 404                         ice_free_dma_mem((hw),                          \
 405                                          &(qi)->ring.r.ring##_bi[i]);   \
 406         /* free the buffer info list */                                 \
 407         if ((qi)->ring.cmd_buf)                                         \
 408                 ice_free(hw, (qi)->ring.cmd_buf);                       \
 409         /* free dma head */                                             \
 410         ice_free(hw, (qi)->ring.dma_head);                              \
 411 } while (0)
 412
 413 /**
 414  * ice_shutdown_sq - shutdown the Control ATQ
 415  * @hw: pointer to the hardware structure
 416  * @cq: pointer to the specific Control queue
 417  *
 418  * The main shutdown routine for the Control Transmit Queue
 419  */
 420 static enum ice_status
 421 ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 422 {
 423         enum ice_status ret_code = ICE_SUCCESS;
 424
 425         ice_acquire_lock(&cq->sq_lock);
 426
 427         if (!cq->sq.count) {
 428                 ret_code = ICE_ERR_NOT_READY;
 429                 goto shutdown_sq_out;
 430         }
 431
 432         /* Stop firmware AdminQ processing */
 433         wr32(hw, cq->sq.head, 0);
 434         wr32(hw, cq->sq.tail, 0);
 435         wr32(hw, cq->sq.len, 0);
 436         wr32(hw, cq->sq.bal, 0);
 437         wr32(hw, cq->sq.bah, 0);
 438
 439         cq->sq.count = 0;       /* to indicate uninitialized queue */
 440
 441         /* free ring buffers and the ring itself */
 442         ICE_FREE_CQ_BUFS(hw, cq, sq);
 443         ice_free_cq_ring(hw, &cq->sq);
 444
 445 shutdown_sq_out:
 446         ice_release_lock(&cq->sq_lock);
 447         return ret_code;
 448 }
 449
 450 /**
 451  * ice_aq_ver_check - Check the reported AQ API version.
 452  * @hw: pointer to the hardware structure
 453  *
 454  * Checks if the driver should load on a given AQ API version.
 455  *
 456  * Return: 'true' iff the driver should attempt to load. 'false' otherwise.
 457  */
 458 static bool ice_aq_ver_check(struct ice_hw *hw)
 459 {
 460         if (hw->api_maj_ver > EXP_FW_API_VER_MAJOR) {
 461                 /* Major API version is newer than expected, don't load */
 462                 ice_warn(hw, "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
 463                 return false;
 464         } else if (hw->api_maj_ver == EXP_FW_API_VER_MAJOR) {
 465                 if (hw->api_min_ver > (EXP_FW_API_VER_MINOR + 2))
 466                         ice_info(hw, "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
 467                 else if ((hw->api_min_ver + 2) < EXP_FW_API_VER_MINOR)
 468                         ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
 469         } else {
 470                 /* Major API version is older than expected, log a warning */
 471                 ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
 472         }
 473         return true;
 474 }
 475
 476 /**
 477  * ice_shutdown_rq - shutdown Control ARQ
 478  * @hw: pointer to the hardware structure
 479  * @cq: pointer to the specific Control queue
 480  *
 481  * The main shutdown routine for the Control Receive Queue
 482  */
 483 static enum ice_status
 484 ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 485 {
 486         enum ice_status ret_code = ICE_SUCCESS;
 487
 488         ice_acquire_lock(&cq->rq_lock);
 489
 490         if (!cq->rq.count) {
 491                 ret_code = ICE_ERR_NOT_READY;
 492                 goto shutdown_rq_out;
 493         }
 494
 495         /* Stop Control Queue processing */
 496         wr32(hw, cq->rq.head, 0);
 497         wr32(hw, cq->rq.tail, 0);
 498         wr32(hw, cq->rq.len, 0);
 499         wr32(hw, cq->rq.bal, 0);
 500         wr32(hw, cq->rq.bah, 0);
 501
 502         /* set rq.count to 0 to indicate uninitialized queue */
 503         cq->rq.count = 0;
 504
 505         /* free ring buffers and the ring itself */
 506         ICE_FREE_CQ_BUFS(hw, cq, rq);
 507         ice_free_cq_ring(hw, &cq->rq);
 508
 509 shutdown_rq_out:
 510         ice_release_lock(&cq->rq_lock);
 511         return ret_code;
 512 }
 513
 514
 515 /**
 516  * ice_init_check_adminq - Check version for Admin Queue to know if its alive
 517  * @hw: pointer to the hardware structure
 518  */
 519 static enum ice_status ice_init_check_adminq(struct ice_hw *hw)
 520 {
 521         struct ice_ctl_q_info *cq = &hw->adminq;
 522         enum ice_status status;
 523
 524
 525         status = ice_aq_get_fw_ver(hw, NULL);
 526         if (status)
 527                 goto init_ctrlq_free_rq;
 528
 529
 530         if (!ice_aq_ver_check(hw)) {
 531                 status = ICE_ERR_FW_API_VER;
 532                 goto init_ctrlq_free_rq;
 533         }
 534
 535         return ICE_SUCCESS;
 536
 537 init_ctrlq_free_rq:
 538         if (cq->rq.count) {
 539                 ice_shutdown_rq(hw, cq);
 540                 ice_destroy_lock(&cq->rq_lock);
 541         }
 542         if (cq->sq.count) {
 543                 ice_shutdown_sq(hw, cq);
 544                 ice_destroy_lock(&cq->sq_lock);
 545         }
 546         return status;
 547 }
 548
 549 /**
 550  * ice_init_ctrlq - main initialization routine for any control Queue
 551  * @hw: pointer to the hardware structure
 552  * @q_type: specific Control queue type
 553  *
 554  * Prior to calling this function, drivers *MUST* set the following fields
 555  * in the cq->structure:
 556  *     - cq->num_sq_entries
 557  *     - cq->num_rq_entries
 558  *     - cq->rq_buf_size
 559  *     - cq->sq_buf_size
 560  */
 561 static enum ice_status ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 562 {
 563         struct ice_ctl_q_info *cq;
 564         enum ice_status ret_code;
 565
 566         switch (q_type) {
 567         case ICE_CTL_Q_ADMIN:
 568                 ice_adminq_init_regs(hw);
 569                 cq = &hw->adminq;
 570                 break;
 571         case ICE_CTL_Q_MAILBOX:
 572                 ice_mailbox_init_regs(hw);
 573                 cq = &hw->mailboxq;
 574                 break;
 575         default:
 576                 return ICE_ERR_PARAM;
 577         }
 578         cq->qtype = q_type;
 579
 580         /* verify input for valid configuration */
 581         if (!cq->num_rq_entries || !cq->num_sq_entries ||
 582             !cq->rq_buf_size || !cq->sq_buf_size) {
 583                 return ICE_ERR_CFG;
 584         }
 585         ice_init_lock(&cq->sq_lock);
 586         ice_init_lock(&cq->rq_lock);
 587
 588         /* setup SQ command write back timeout */
 589         cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;
 590
 591         /* allocate the ATQ */
 592         ret_code = ice_init_sq(hw, cq);
 593         if (ret_code)
 594                 goto init_ctrlq_destroy_locks;
 595
 596         /* allocate the ARQ */
 597         ret_code = ice_init_rq(hw, cq);
 598         if (ret_code)
 599                 goto init_ctrlq_free_sq;
 600
 601         /* success! */
 602         return ICE_SUCCESS;
 603
 604 init_ctrlq_free_sq:
 605         ice_shutdown_sq(hw, cq);
 606 init_ctrlq_destroy_locks:
 607         ice_destroy_lock(&cq->sq_lock);
 608         ice_destroy_lock(&cq->rq_lock);
 609         return ret_code;
 610 }
 611
 612 /**
 613  * ice_init_all_ctrlq - main initialization routine for all control queues
 614  * @hw: pointer to the hardware structure
 615  *
 616  * Prior to calling this function, drivers *MUST* set the following fields
 617  * in the cq->structure for all control queues:
 618  *     - cq->num_sq_entries
 619  *     - cq->num_rq_entries
 620  *     - cq->rq_buf_size
 621  *     - cq->sq_buf_size
 622  */
 623 enum ice_status ice_init_all_ctrlq(struct ice_hw *hw)
 624 {
 625         enum ice_status ret_code;
 626
 627
 628         /* Init FW admin queue */
 629         ret_code = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
 630         if (ret_code)
 631                 return ret_code;
 632
 633         ret_code = ice_init_check_adminq(hw);
 634         if (ret_code)
 635                 return ret_code;
 636         /* Init Mailbox queue */
 637         return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);
 638 }
 639
 640 /**
 641  * ice_shutdown_ctrlq - shutdown routine for any control queue
 642  * @hw: pointer to the hardware structure
 643  * @q_type: specific Control queue type
 644  */
 645 static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 646 {
 647         struct ice_ctl_q_info *cq;
 648
 649         switch (q_type) {
 650         case ICE_CTL_Q_ADMIN:
 651                 cq = &hw->adminq;
 652                 if (ice_check_sq_alive(hw, cq))
 653                         ice_aq_q_shutdown(hw, true);
 654                 break;
 655         case ICE_CTL_Q_MAILBOX:
 656                 cq = &hw->mailboxq;
 657                 break;
 658         default:
 659                 return;
 660         }
 661
 662         if (cq->sq.count) {
 663                 ice_shutdown_sq(hw, cq);
 664                 ice_destroy_lock(&cq->sq_lock);
 665         }
 666         if (cq->rq.count) {
 667                 ice_shutdown_rq(hw, cq);
 668                 ice_destroy_lock(&cq->rq_lock);
 669         }
 670 }
 671
 672 /**
 673  * ice_shutdown_all_ctrlq - shutdown routine for all control queues
 674  * @hw: pointer to the hardware structure
 675  */
 676 void ice_shutdown_all_ctrlq(struct ice_hw *hw)
 677 {
 678         /* Shutdown FW admin queue */
 679         ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
 680         /* Shutdown PF-VF Mailbox */
 681         ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX);
 682 }
 683
 684 /**
 685  * ice_clean_sq - cleans Admin send queue (ATQ)
 686  * @hw: pointer to the hardware structure
 687  * @cq: pointer to the specific Control queue
 688  *
 689  * returns the number of free desc
 690  */
 691 static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 692 {
 693         struct ice_ctl_q_ring *sq = &cq->sq;
 694         u16 ntc = sq->next_to_clean;
 695         struct ice_sq_cd *details;
 696 #if 0
 697         struct ice_aq_desc desc_cb;
 698 #endif
 699         struct ice_aq_desc *desc;
 700
 701         desc = ICE_CTL_Q_DESC(*sq, ntc);
 702         details = ICE_CTL_Q_DETAILS(*sq, ntc);
 703
 704         while (rd32(hw, cq->sq.head) != ntc) {
 705                 ice_debug(hw, ICE_DBG_AQ_MSG,
 706                           "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head));
 707 #if 0
 708                 if (details->callback) {
 709                         ICE_CTL_Q_CALLBACK cb_func =
 710                                 (ICE_CTL_Q_CALLBACK)details->callback;
 711                         ice_memcpy(&desc_cb, desc, sizeof(desc_cb),
 712                                    ICE_DMA_TO_DMA);
 713                         cb_func(hw, &desc_cb);
 714                 }
 715 #endif
 716                 ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
 717                 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
 718                 ntc++;
 719                 if (ntc == sq->count)
 720                         ntc = 0;
 721                 desc = ICE_CTL_Q_DESC(*sq, ntc);
 722                 details = ICE_CTL_Q_DETAILS(*sq, ntc);
 723         }
 724
 725         sq->next_to_clean = ntc;
 726
 727         return ICE_CTL_Q_DESC_UNUSED(sq);
 728 }
 729
 730 /**
 731  * ice_debug_cq
 732  * @hw: pointer to the hardware structure
 733  * @desc: pointer to control queue descriptor
 734  * @buf: pointer to command buffer
 735  * @buf_len: max length of buf
 736  *
 737  * Dumps debug log about control command with descriptor contents.
 738  */
 739 static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len)
 740 {
 741         struct ice_aq_desc *cq_desc = (struct ice_aq_desc *)desc;
 742         u16 datalen, flags;
 743
 744         if (!((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask))
 745                 return;
 746
 747         if (!desc)
 748                 return;
 749
 750         datalen = LE16_TO_CPU(cq_desc->datalen);
 751         flags = LE16_TO_CPU(cq_desc->flags);
 752
 753         ice_debug(hw, ICE_DBG_AQ_DESC,
 754                   "CQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n",
 755                   LE16_TO_CPU(cq_desc->opcode), flags, datalen,
 756                   LE16_TO_CPU(cq_desc->retval));
 757         ice_debug(hw, ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n",
 758                   LE32_TO_CPU(cq_desc->cookie_high),
 759                   LE32_TO_CPU(cq_desc->cookie_low));
 760         ice_debug(hw, ICE_DBG_AQ_DESC, "\tparam (0,1)  0x%08X 0x%08X\n",
 761                   LE32_TO_CPU(cq_desc->params.generic.param0),
 762                   LE32_TO_CPU(cq_desc->params.generic.param1));
 763         ice_debug(hw, ICE_DBG_AQ_DESC, "\taddr (h,l)   0x%08X 0x%08X\n",
 764                   LE32_TO_CPU(cq_desc->params.generic.addr_high),
 765                   LE32_TO_CPU(cq_desc->params.generic.addr_low));
 766         /* Dump buffer iff 1) one exists and 2) is either a response indicated
 767          * by the DD and/or CMP flag set or a command with the RD flag set.
 768          */
 769         if (buf && cq_desc->datalen != 0 &&
 770             (flags & (ICE_AQ_FLAG_DD | ICE_AQ_FLAG_CMP) ||
 771              flags & ICE_AQ_FLAG_RD)) {
 772                 ice_debug(hw, ICE_DBG_AQ_DESC_BUF, "Buffer:\n");
 773                 ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, (u8 *)buf,
 774                                 min(buf_len, datalen));
 775         }
 776 }
 777
 778 /**
 779  * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ)
 780  * @hw: pointer to the HW struct
 781  * @cq: pointer to the specific Control queue
 782  *
 783  * Returns true if the firmware has processed all descriptors on the
 784  * admin send queue. Returns false if there are still requests pending.
 785  */
 786 static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 787 {
 788         /* AQ designers suggest use of head for better
 789          * timing reliability than DD bit
 790          */
 791         return rd32(hw, cq->sq.head) == cq->sq.next_to_use;
 792 }
 793
 794 /**
 795  * ice_sq_send_cmd - send command to Control Queue (ATQ)
 796  * @hw: pointer to the HW struct
 797  * @cq: pointer to the specific Control queue
 798  * @desc: prefilled descriptor describing the command (non DMA mem)
 799  * @buf: buffer to use for indirect commands (or NULL for direct commands)
 800  * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
 801  * @cd: pointer to command details structure
 802  *
 803  * This is the main send command routine for the ATQ. It runs the queue,
 804  * cleans the queue, etc.
 805  */
 806 enum ice_status
 807 ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 808                 struct ice_aq_desc *desc, void *buf, u16 buf_size,
 809                 struct ice_sq_cd *cd)
 810 {
 811         struct ice_dma_mem *dma_buf = NULL;
 812         struct ice_aq_desc *desc_on_ring;
 813         bool cmd_completed = false;
 814         enum ice_status status = ICE_SUCCESS;
 815         struct ice_sq_cd *details;
 816         u32 total_delay = 0;
 817         u16 retval = 0;
 818         u32 val = 0;
 819
 820         /* if reset is in progress return a soft error */
 821         if (hw->reset_ongoing)
 822                 return ICE_ERR_RESET_ONGOING;
 823         ice_acquire_lock(&cq->sq_lock);
 824
 825         cq->sq_last_status = ICE_AQ_RC_OK;
 826
 827         if (!cq->sq.count) {
 828                 ice_debug(hw, ICE_DBG_AQ_MSG,
 829                           "Control Send queue not initialized.\n");
 830                 status = ICE_ERR_AQ_EMPTY;
 831                 goto sq_send_command_error;
 832         }
 833
 834         if ((buf && !buf_size) || (!buf && buf_size)) {
 835                 status = ICE_ERR_PARAM;
 836                 goto sq_send_command_error;
 837         }
 838
 839         if (buf) {
 840                 if (buf_size > cq->sq_buf_size) {
 841                         ice_debug(hw, ICE_DBG_AQ_MSG,
 842                                   "Invalid buffer size for Control Send queue: %d.\n",
 843                                   buf_size);
 844                         status = ICE_ERR_INVAL_SIZE;
 845                         goto sq_send_command_error;
 846                 }
 847
 848                 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_BUF);
 849                 if (buf_size > ICE_AQ_LG_BUF)
 850                         desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
 851         }
 852
 853         val = rd32(hw, cq->sq.head);
 854         if (val >= cq->num_sq_entries) {
 855                 ice_debug(hw, ICE_DBG_AQ_MSG,
 856                           "head overrun at %d in the Control Send Queue ring\n",
 857                           val);
 858                 status = ICE_ERR_AQ_EMPTY;
 859                 goto sq_send_command_error;
 860         }
 861
 862         details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use);
 863         if (cd)
 864                 *details = *cd;
 865 #if 0
 866                 /* FIXME: if/when this block gets enabled (when the #if 0
 867                  * is removed), add braces to both branches of the surrounding
 868                  * conditional expression. The braces have been removed to
 869                  * prevent checkpatch complaining.
 870                  */
 871
 872                 /* If the command details are defined copy the cookie. The
 873                  * CPU_TO_LE32 is not needed here because the data is ignored
 874                  * by the FW, only used by the driver
 875                  */
 876                 if (details->cookie) {
 877                         desc->cookie_high =
 878                                 CPU_TO_LE32(ICE_HI_DWORD(details->cookie));
 879                         desc->cookie_low =
 880                                 CPU_TO_LE32(ICE_LO_DWORD(details->cookie));
 881                 }
 882 #endif
 883         else
 884                 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
 885 #if 0
 886         /* clear requested flags and then set additional flags if defined */
 887         desc->flags &= ~CPU_TO_LE16(details->flags_dis);
 888         desc->flags |= CPU_TO_LE16(details->flags_ena);
 889
 890         if (details->postpone && !details->async) {
 891                 ice_debug(hw, ICE_DBG_AQ_MSG,
 892                           "Async flag not set along with postpone flag\n");
 893                 status = ICE_ERR_PARAM;
 894                 goto sq_send_command_error;
 895         }
 896 #endif
 897
 898         /* Call clean and check queue available function to reclaim the
 899          * descriptors that were processed by FW/MBX; the function returns the
 900          * number of desc available. The clean function called here could be
 901          * called in a separate thread in case of asynchronous completions.
 902          */
 903         if (ice_clean_sq(hw, cq) == 0) {
 904                 ice_debug(hw, ICE_DBG_AQ_MSG,
 905                           "Error: Control Send Queue is full.\n");
 906                 status = ICE_ERR_AQ_FULL;
 907                 goto sq_send_command_error;
 908         }
 909
 910         /* initialize the temp desc pointer with the right desc */
 911         desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use);
 912
 913         /* if the desc is available copy the temp desc to the right place */
 914         ice_memcpy(desc_on_ring, desc, sizeof(*desc_on_ring),
 915                    ICE_NONDMA_TO_DMA);
 916
 917         /* if buf is not NULL assume indirect command */
 918         if (buf) {
 919                 dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use];
 920                 /* copy the user buf into the respective DMA buf */
 921                 ice_memcpy(dma_buf->va, buf, buf_size, ICE_NONDMA_TO_DMA);
 922                 desc_on_ring->datalen = CPU_TO_LE16(buf_size);
 923
 924                 /* Update the address values in the desc with the pa value
 925                  * for respective buffer
 926                  */
 927                 desc_on_ring->params.generic.addr_high =
 928                         CPU_TO_LE32(ICE_HI_DWORD(dma_buf->pa));
 929                 desc_on_ring->params.generic.addr_low =
 930                         CPU_TO_LE32(ICE_LO_DWORD(dma_buf->pa));
 931         }
 932
 933         /* Debug desc and buffer */
 934         ice_debug(hw, ICE_DBG_AQ_MSG,
 935                   "ATQ: Control Send queue desc and buffer:\n");
 936
 937         ice_debug_cq(hw, (void *)desc_on_ring, buf, buf_size);
 938
 939
 940         (cq->sq.next_to_use)++;
 941         if (cq->sq.next_to_use == cq->sq.count)
 942                 cq->sq.next_to_use = 0;
 943 #if 0
 944         /* FIXME - handle this case? */
 945         if (!details->postpone)
 946 #endif
 947         wr32(hw, cq->sq.tail, cq->sq.next_to_use);
 948
 949 #if 0
 950         /* if command details are not defined or async flag is not set,
 951          * we need to wait for desc write back
 952          */
 953         if (!details->async && !details->postpone) {
 954                 /* FIXME - handle this case? */
 955         }
 956 #endif
 957         do {
 958                 if (ice_sq_done(hw, cq))
 959                         break;
 960
 961                 ice_msec_delay(1, false);
 962                 total_delay++;
 963         } while (total_delay < cq->sq_cmd_timeout);
 964
 965         /* if ready, copy the desc back to temp */
 966         if (ice_sq_done(hw, cq)) {
 967                 ice_memcpy(desc, desc_on_ring, sizeof(*desc),
 968                            ICE_DMA_TO_NONDMA);
 969                 if (buf) {
 970                         /* get returned length to copy */
 971                         u16 copy_size = LE16_TO_CPU(desc->datalen);
 972
 973                         if (copy_size > buf_size) {
 974                                 ice_debug(hw, ICE_DBG_AQ_MSG,
 975                                           "Return len %d > than buf len %d\n",
 976                                           copy_size, buf_size);
 977                                 status = ICE_ERR_AQ_ERROR;
 978                         } else {
 979                                 ice_memcpy(buf, dma_buf->va, copy_size,
 980                                            ICE_DMA_TO_NONDMA);
 981                         }
 982                 }
 983                 retval = LE16_TO_CPU(desc->retval);
 984                 if (retval) {
 985                         ice_debug(hw, ICE_DBG_AQ_MSG,
 986                                   "Control Send Queue command completed with error 0x%x\n",
 987                                   retval);
 988
 989                         /* strip off FW internal code */
 990                         retval &= 0xff;
 991                 }
 992                 cmd_completed = true;
 993                 if (!status && retval != ICE_AQ_RC_OK)
 994                         status = ICE_ERR_AQ_ERROR;
 995                 cq->sq_last_status = (enum ice_aq_err)retval;
 996         }
 997
 998         ice_debug(hw, ICE_DBG_AQ_MSG,
 999                   "ATQ: desc and buffer writeback:\n");
1000
1001         ice_debug_cq(hw, (void *)desc, buf, buf_size);
1002
1003
1004         /* save writeback AQ if requested */
1005         if (details->wb_desc)
1006                 ice_memcpy(details->wb_desc, desc_on_ring,
1007                            sizeof(*details->wb_desc), ICE_DMA_TO_NONDMA);
1008
1009         /* update the error if time out occurred */
1010         if (!cmd_completed) {
1011 #if 0
1012             (!details->async && !details->postpone)) {
1013 #endif
1014                 ice_debug(hw, ICE_DBG_AQ_MSG,
1015                           "Control Send Queue Writeback timeout.\n");
1016                 status = ICE_ERR_AQ_TIMEOUT;
1017         }
1018
1019 sq_send_command_error:
1020         ice_release_lock(&cq->sq_lock);
1021         return status;
1022 }
1023
1024 /**
1025  * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function
1026  * @desc: pointer to the temp descriptor (non DMA mem)
1027  * @opcode: the opcode can be used to decide which flags to turn off or on
1028  *
1029  * Fill the desc with default values
1030  */
1031 void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode)
1032 {
1033         /* zero out the desc */
1034         ice_memset(desc, 0, sizeof(*desc), ICE_NONDMA_MEM);
1035         desc->opcode = CPU_TO_LE16(opcode);
1036         desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_SI);
1037 }
1038
1039 /**
1040  * ice_clean_rq_elem
1041  * @hw: pointer to the HW struct
1042  * @cq: pointer to the specific Control queue
1043  * @e: event info from the receive descriptor, includes any buffers
1044  * @pending: number of events that could be left to process
1045  *
1046  * This function cleans one Admin Receive Queue element and returns
1047  * the contents through e. It can also return how many events are
1048  * left to process through 'pending'.
1049  */
1050 enum ice_status
1051 ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq,
1052                   struct ice_rq_event_info *e, u16 *pending)
1053 {
1054         u16 ntc = cq->rq.next_to_clean;
1055         enum ice_status ret_code = ICE_SUCCESS;
1056         struct ice_aq_desc *desc;
1057         struct ice_dma_mem *bi;
1058         u16 desc_idx;
1059         u16 datalen;
1060         u16 flags;
1061         u16 ntu;
1062
1063         /* pre-clean the event info */
1064         ice_memset(&e->desc, 0, sizeof(e->desc), ICE_NONDMA_MEM);
1065
1066         /* take the lock before we start messing with the ring */
1067         ice_acquire_lock(&cq->rq_lock);
1068
1069         if (!cq->rq.count) {
1070                 ice_debug(hw, ICE_DBG_AQ_MSG,
1071                           "Control Receive queue not initialized.\n");
1072                 ret_code = ICE_ERR_AQ_EMPTY;
1073                 goto clean_rq_elem_err;
1074         }
1075
1076         /* set next_to_use to head */
1077         ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1078
1079         if (ntu == ntc) {
1080                 /* nothing to do - shouldn't need to update ring's values */
1081                 ret_code = ICE_ERR_AQ_NO_WORK;
1082                 goto clean_rq_elem_out;
1083         }
1084
1085         /* now clean the next descriptor */
1086         desc = ICE_CTL_Q_DESC(cq->rq, ntc);
1087         desc_idx = ntc;
1088
1089         cq->rq_last_status = (enum ice_aq_err)LE16_TO_CPU(desc->retval);
1090         flags = LE16_TO_CPU(desc->flags);
1091         if (flags & ICE_AQ_FLAG_ERR) {
1092                 ret_code = ICE_ERR_AQ_ERROR;
1093                 ice_debug(hw, ICE_DBG_AQ_MSG,
1094                           "Control Receive Queue Event received with error 0x%x\n",
1095                           cq->rq_last_status);
1096         }
1097         ice_memcpy(&e->desc, desc, sizeof(e->desc), ICE_DMA_TO_NONDMA);
1098         datalen = LE16_TO_CPU(desc->datalen);
1099         e->msg_len = min(datalen, e->buf_len);
1100         if (e->msg_buf && e->msg_len)
1101                 ice_memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va,
1102                            e->msg_len, ICE_DMA_TO_NONDMA);
1103
1104         ice_debug(hw, ICE_DBG_AQ_MSG, "ARQ: desc and buffer:\n");
1105
1106         ice_debug_cq(hw, (void *)desc, e->msg_buf,
1107                      cq->rq_buf_size);
1108
1109
1110         /* Restore the original datalen and buffer address in the desc,
1111          * FW updates datalen to indicate the event message size
1112          */
1113         bi = &cq->rq.r.rq_bi[ntc];
1114         ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
1115
1116         desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
1117         if (cq->rq_buf_size > ICE_AQ_LG_BUF)
1118                 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
1119         desc->datalen = CPU_TO_LE16(bi->size);
1120         desc->params.generic.addr_high = CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
1121         desc->params.generic.addr_low = CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
1122
1123         /* set tail = the last cleaned desc index. */
1124         wr32(hw, cq->rq.tail, ntc);
1125         /* ntc is updated to tail + 1 */
1126         ntc++;
1127         if (ntc == cq->num_rq_entries)
1128                 ntc = 0;
1129         cq->rq.next_to_clean = ntc;
1130         cq->rq.next_to_use = ntu;
1131
1132 #if 0
1133         ice_nvmupd_check_wait_event(hw, LE16_TO_CPU(e->desc.opcode));
1134 #endif
1135 clean_rq_elem_out:
1136         /* Set pending if needed, unlock and return */
1137         if (pending) {
1138                 /* re-read HW head to calculate actual pending messages */
1139                 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1140                 *pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc));
1141         }
1142 clean_rq_elem_err:
1143         ice_release_lock(&cq->rq_lock);
1144
1145         return ret_code;
1146 }