drivers/net/ice/base/ice_controlq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2001-2018
   3  */
   4
   5 #include "ice_common.h"
   6
   7
   8 #define ICE_CQ_INIT_REGS(qinfo, prefix)                         \
   9 do {                                                            \
  10         (qinfo)->sq.head = prefix##_ATQH;                       \
  11         (qinfo)->sq.tail = prefix##_ATQT;                       \
  12         (qinfo)->sq.len = prefix##_ATQLEN;                      \
  13         (qinfo)->sq.bah = prefix##_ATQBAH;                      \
  14         (qinfo)->sq.bal = prefix##_ATQBAL;                      \
  15         (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M;        \
  16         (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \
  17         (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M;           \
  18         (qinfo)->rq.head = prefix##_ARQH;                       \
  19         (qinfo)->rq.tail = prefix##_ARQT;                       \
  20         (qinfo)->rq.len = prefix##_ARQLEN;                      \
  21         (qinfo)->rq.bah = prefix##_ARQBAH;                      \
  22         (qinfo)->rq.bal = prefix##_ARQBAL;                      \
  23         (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M;        \
  24         (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \
  25         (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M;           \
  26 } while (0)
  27
  28 /**
  29  * ice_adminq_init_regs - Initialize AdminQ registers
  30  * @hw: pointer to the hardware structure
  31  *
  32  * This assumes the alloc_sq and alloc_rq functions have already been called
  33  */
  34 static void ice_adminq_init_regs(struct ice_hw *hw)
  35 {
  36         struct ice_ctl_q_info *cq = &hw->adminq;
  37
  38         ICE_CQ_INIT_REGS(cq, PF_FW);
  39 }
  40
  41 /**
  42  * ice_mailbox_init_regs - Initialize Mailbox registers
  43  * @hw: pointer to the hardware structure
  44  *
  45  * This assumes the alloc_sq and alloc_rq functions have already been called
  46  */
  47 static void ice_mailbox_init_regs(struct ice_hw *hw)
  48 {
  49         struct ice_ctl_q_info *cq = &hw->mailboxq;
  50
  51         ICE_CQ_INIT_REGS(cq, PF_MBX);
  52 }
  53
  54
  55 /**
  56  * ice_check_sq_alive
  57  * @hw: pointer to the hw struct
  58  * @cq: pointer to the specific Control queue
  59  *
  60  * Returns true if Queue is enabled else false.
  61  */
  62 bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  63 {
  64         /* check both queue-length and queue-enable fields */
  65         if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask)
  66                 return (rd32(hw, cq->sq.len) & (cq->sq.len_mask |
  67                                                 cq->sq.len_ena_mask)) ==
  68                         (cq->num_sq_entries | cq->sq.len_ena_mask);
  69
  70         return false;
  71 }
  72
  73 /**
  74  * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings
  75  * @hw: pointer to the hardware structure
  76  * @cq: pointer to the specific Control queue
  77  */
  78 static enum ice_status
  79 ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  80 {
  81         size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc);
  82
  83         cq->sq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->sq.desc_buf, size);
  84         if (!cq->sq.desc_buf.va)
  85                 return ICE_ERR_NO_MEMORY;
  86
  87         cq->sq.cmd_buf = ice_calloc(hw, cq->num_sq_entries,
  88                                     sizeof(struct ice_sq_cd));
  89         if (!cq->sq.cmd_buf) {
  90                 ice_free_dma_mem(hw, &cq->sq.desc_buf);
  91                 return ICE_ERR_NO_MEMORY;
  92         }
  93
  94         return ICE_SUCCESS;
  95 }
  96
  97 /**
  98  * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings
  99  * @hw: pointer to the hardware structure
 100  * @cq: pointer to the specific Control queue
 101  */
 102 static enum ice_status
 103 ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 104 {
 105         size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc);
 106
 107         cq->rq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->rq.desc_buf, size);
 108         if (!cq->rq.desc_buf.va)
 109                 return ICE_ERR_NO_MEMORY;
 110         return ICE_SUCCESS;
 111 }
 112
 113 /**
 114  * ice_free_cq_ring - Free control queue ring
 115  * @hw: pointer to the hardware structure
 116  * @ring: pointer to the specific control queue ring
 117  *
 118  * This assumes the posted buffers have already been cleaned
 119  * and de-allocated
 120  */
 121 static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)
 122 {
 123         ice_free_dma_mem(hw, &ring->desc_buf);
 124 }
 125
 126 /**
 127  * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ
 128  * @hw: pointer to the hardware structure
 129  * @cq: pointer to the specific Control queue
 130  */
 131 static enum ice_status
 132 ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 133 {
 134         int i;
 135
 136         /* We'll be allocating the buffer info memory first, then we can
 137          * allocate the mapped buffers for the event processing
 138          */
 139         cq->rq.dma_head = ice_calloc(hw, cq->num_rq_entries,
 140                                      sizeof(cq->rq.desc_buf));
 141         if (!cq->rq.dma_head)
 142                 return ICE_ERR_NO_MEMORY;
 143         cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head;
 144
 145         /* allocate the mapped buffers */
 146         for (i = 0; i < cq->num_rq_entries; i++) {
 147                 struct ice_aq_desc *desc;
 148                 struct ice_dma_mem *bi;
 149
 150                 bi = &cq->rq.r.rq_bi[i];
 151                 bi->va = ice_alloc_dma_mem(hw, bi, cq->rq_buf_size);
 152                 if (!bi->va)
 153                         goto unwind_alloc_rq_bufs;
 154
 155                 /* now configure the descriptors for use */
 156                 desc = ICE_CTL_Q_DESC(cq->rq, i);
 157
 158                 desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
 159                 if (cq->rq_buf_size > ICE_AQ_LG_BUF)
 160                         desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
 161                 desc->opcode = 0;
 162                 /* This is in accordance with Admin queue design, there is no
 163                  * register for buffer size configuration
 164                  */
 165                 desc->datalen = CPU_TO_LE16(bi->size);
 166                 desc->retval = 0;
 167                 desc->cookie_high = 0;
 168                 desc->cookie_low = 0;
 169                 desc->params.generic.addr_high =
 170                         CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
 171                 desc->params.generic.addr_low =
 172                         CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
 173                 desc->params.generic.param0 = 0;
 174                 desc->params.generic.param1 = 0;
 175         }
 176         return ICE_SUCCESS;
 177
 178 unwind_alloc_rq_bufs:
 179         /* don't try to free the one that failed... */
 180         i--;
 181         for (; i >= 0; i--)
 182                 ice_free_dma_mem(hw, &cq->rq.r.rq_bi[i]);
 183         ice_free(hw, cq->rq.dma_head);
 184
 185         return ICE_ERR_NO_MEMORY;
 186 }
 187
 188 /**
 189  * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ
 190  * @hw: pointer to the hardware structure
 191  * @cq: pointer to the specific Control queue
 192  */
 193 static enum ice_status
 194 ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 195 {
 196         int i;
 197
 198         /* No mapped memory needed yet, just the buffer info structures */
 199         cq->sq.dma_head = ice_calloc(hw, cq->num_sq_entries,
 200                                      sizeof(cq->sq.desc_buf));
 201         if (!cq->sq.dma_head)
 202                 return ICE_ERR_NO_MEMORY;
 203         cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head;
 204
 205         /* allocate the mapped buffers */
 206         for (i = 0; i < cq->num_sq_entries; i++) {
 207                 struct ice_dma_mem *bi;
 208
 209                 bi = &cq->sq.r.sq_bi[i];
 210                 bi->va = ice_alloc_dma_mem(hw, bi, cq->sq_buf_size);
 211                 if (!bi->va)
 212                         goto unwind_alloc_sq_bufs;
 213         }
 214         return ICE_SUCCESS;
 215
 216 unwind_alloc_sq_bufs:
 217         /* don't try to free the one that failed... */
 218         i--;
 219         for (; i >= 0; i--)
 220                 ice_free_dma_mem(hw, &cq->sq.r.sq_bi[i]);
 221         ice_free(hw, cq->sq.dma_head);
 222
 223         return ICE_ERR_NO_MEMORY;
 224 }
 225
 226 static enum ice_status
 227 ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries)
 228 {
 229         /* Clear Head and Tail */
 230         wr32(hw, ring->head, 0);
 231         wr32(hw, ring->tail, 0);
 232
 233         /* set starting point */
 234         wr32(hw, ring->len, (num_entries | ring->len_ena_mask));
 235         wr32(hw, ring->bal, ICE_LO_DWORD(ring->desc_buf.pa));
 236         wr32(hw, ring->bah, ICE_HI_DWORD(ring->desc_buf.pa));
 237
 238         /* Check one register to verify that config was applied */
 239         if (rd32(hw, ring->bal) != ICE_LO_DWORD(ring->desc_buf.pa))
 240                 return ICE_ERR_AQ_ERROR;
 241
 242         return ICE_SUCCESS;
 243 }
 244
 245 /**
 246  * ice_cfg_sq_regs - configure Control ATQ registers
 247  * @hw: pointer to the hardware structure
 248  * @cq: pointer to the specific Control queue
 249  *
 250  * Configure base address and length registers for the transmit queue
 251  */
 252 static enum ice_status
 253 ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 254 {
 255         return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);
 256 }
 257
 258 /**
 259  * ice_cfg_rq_regs - configure Control ARQ register
 260  * @hw: pointer to the hardware structure
 261  * @cq: pointer to the specific Control queue
 262  *
 263  * Configure base address and length registers for the receive (event q)
 264  */
 265 static enum ice_status
 266 ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 267 {
 268         enum ice_status status;
 269
 270         status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);
 271         if (status)
 272                 return status;
 273
 274         /* Update tail in the HW to post pre-allocated buffers */
 275         wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1));
 276
 277         return ICE_SUCCESS;
 278 }
 279
 280 /**
 281  * ice_init_sq - main initialization routine for Control ATQ
 282  * @hw: pointer to the hardware structure
 283  * @cq: pointer to the specific Control queue
 284  *
 285  * This is the main initialization routine for the Control Send Queue
 286  * Prior to calling this function, drivers *MUST* set the following fields
 287  * in the cq->structure:
 288  *     - cq->num_sq_entries
 289  *     - cq->sq_buf_size
 290  *
 291  * Do *NOT* hold the lock when calling this as the memory allocation routines
 292  * called are not going to be atomic context safe
 293  */
 294 static enum ice_status ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 295 {
 296         enum ice_status ret_code;
 297
 298         if (cq->sq.count > 0) {
 299                 /* queue already initialized */
 300                 ret_code = ICE_ERR_NOT_READY;
 301                 goto init_ctrlq_exit;
 302         }
 303
 304         /* verify input for valid configuration */
 305         if (!cq->num_sq_entries || !cq->sq_buf_size) {
 306                 ret_code = ICE_ERR_CFG;
 307                 goto init_ctrlq_exit;
 308         }
 309
 310         cq->sq.next_to_use = 0;
 311         cq->sq.next_to_clean = 0;
 312
 313         /* allocate the ring memory */
 314         ret_code = ice_alloc_ctrlq_sq_ring(hw, cq);
 315         if (ret_code)
 316                 goto init_ctrlq_exit;
 317
 318         /* allocate buffers in the rings */
 319         ret_code = ice_alloc_sq_bufs(hw, cq);
 320         if (ret_code)
 321                 goto init_ctrlq_free_rings;
 322
 323         /* initialize base registers */
 324         ret_code = ice_cfg_sq_regs(hw, cq);
 325         if (ret_code)
 326                 goto init_ctrlq_free_rings;
 327
 328         /* success! */
 329         cq->sq.count = cq->num_sq_entries;
 330         goto init_ctrlq_exit;
 331
 332 init_ctrlq_free_rings:
 333         ice_free_cq_ring(hw, &cq->sq);
 334
 335 init_ctrlq_exit:
 336         return ret_code;
 337 }
 338
 339 /**
 340  * ice_init_rq - initialize ARQ
 341  * @hw: pointer to the hardware structure
 342  * @cq: pointer to the specific Control queue
 343  *
 344  * The main initialization routine for the Admin Receive (Event) Queue.
 345  * Prior to calling this function, drivers *MUST* set the following fields
 346  * in the cq->structure:
 347  *     - cq->num_rq_entries
 348  *     - cq->rq_buf_size
 349  *
 350  * Do *NOT* hold the lock when calling this as the memory allocation routines
 351  * called are not going to be atomic context safe
 352  */
 353 static enum ice_status ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 354 {
 355         enum ice_status ret_code;
 356
 357         if (cq->rq.count > 0) {
 358                 /* queue already initialized */
 359                 ret_code = ICE_ERR_NOT_READY;
 360                 goto init_ctrlq_exit;
 361         }
 362
 363         /* verify input for valid configuration */
 364         if (!cq->num_rq_entries || !cq->rq_buf_size) {
 365                 ret_code = ICE_ERR_CFG;
 366                 goto init_ctrlq_exit;
 367         }
 368
 369         cq->rq.next_to_use = 0;
 370         cq->rq.next_to_clean = 0;
 371
 372         /* allocate the ring memory */
 373         ret_code = ice_alloc_ctrlq_rq_ring(hw, cq);
 374         if (ret_code)
 375                 goto init_ctrlq_exit;
 376
 377         /* allocate buffers in the rings */
 378         ret_code = ice_alloc_rq_bufs(hw, cq);
 379         if (ret_code)
 380                 goto init_ctrlq_free_rings;
 381
 382         /* initialize base registers */
 383         ret_code = ice_cfg_rq_regs(hw, cq);
 384         if (ret_code)
 385                 goto init_ctrlq_free_rings;
 386
 387         /* success! */
 388         cq->rq.count = cq->num_rq_entries;
 389         goto init_ctrlq_exit;
 390
 391 init_ctrlq_free_rings:
 392         ice_free_cq_ring(hw, &cq->rq);
 393
 394 init_ctrlq_exit:
 395         return ret_code;
 396 }
 397
 398 #define ICE_FREE_CQ_BUFS(hw, qi, ring)                                  \
 399 do {                                                                    \
 400         int i;                                                          \
 401         /* free descriptors */                                          \
 402         for (i = 0; i < (qi)->num_##ring##_entries; i++)                \
 403                 if ((qi)->ring.r.ring##_bi[i].pa)                       \
 404                         ice_free_dma_mem((hw),                          \
 405                                          &(qi)->ring.r.ring##_bi[i]);   \
 406         /* free the buffer info list */                                 \
 407         if ((qi)->ring.cmd_buf)                                         \
 408                 ice_free(hw, (qi)->ring.cmd_buf);                       \
 409         /* free dma head */                                             \
 410         ice_free(hw, (qi)->ring.dma_head);                              \
 411 } while (0)
 412
 413 /**
 414  * ice_shutdown_sq - shutdown the Control ATQ
 415  * @hw: pointer to the hardware structure
 416  * @cq: pointer to the specific Control queue
 417  *
 418  * The main shutdown routine for the Control Transmit Queue
 419  */
 420 static enum ice_status
 421 ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 422 {
 423         enum ice_status ret_code = ICE_SUCCESS;
 424
 425         ice_acquire_lock(&cq->sq_lock);
 426
 427         if (!cq->sq.count) {
 428                 ret_code = ICE_ERR_NOT_READY;
 429                 goto shutdown_sq_out;
 430         }
 431
 432         /* Stop firmware AdminQ processing */
 433         wr32(hw, cq->sq.head, 0);
 434         wr32(hw, cq->sq.tail, 0);
 435         wr32(hw, cq->sq.len, 0);
 436         wr32(hw, cq->sq.bal, 0);
 437         wr32(hw, cq->sq.bah, 0);
 438
 439         cq->sq.count = 0;       /* to indicate uninitialized queue */
 440
 441         /* free ring buffers and the ring itself */
 442         ICE_FREE_CQ_BUFS(hw, cq, sq);
 443         ice_free_cq_ring(hw, &cq->sq);
 444
 445 shutdown_sq_out:
 446         ice_release_lock(&cq->sq_lock);
 447         return ret_code;
 448 }
 449
 450 /**
 451  * ice_aq_ver_check - Check the reported AQ API version.
 452  * @hw: pointer to the hardware structure
 453  *
 454  * Checks if the driver should load on a given AQ API version.
 455  *
 456  * Return: 'true' iff the driver should attempt to load. 'false' otherwise.
 457  */
 458 static bool ice_aq_ver_check(struct ice_hw *hw)
 459 {
 460         if (hw->api_maj_ver > EXP_FW_API_VER_MAJOR) {
 461                 /* Major API version is newer than expected, don't load */
 462                 ice_warn(hw, "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
 463                 return false;
 464         } else if (hw->api_maj_ver == EXP_FW_API_VER_MAJOR) {
 465                 if (hw->api_min_ver > (EXP_FW_API_VER_MINOR + 2))
 466                         ice_info(hw, "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
 467                 else if ((hw->api_min_ver + 2) < EXP_FW_API_VER_MINOR)
 468                         ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
 469         } else {
 470                 /* Major API version is older than expected, log a warning */
 471                 ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
 472         }
 473         return true;
 474 }
 475
 476 /**
 477  * ice_shutdown_rq - shutdown Control ARQ
 478  * @hw: pointer to the hardware structure
 479  * @cq: pointer to the specific Control queue
 480  *
 481  * The main shutdown routine for the Control Receive Queue
 482  */
 483 static enum ice_status
 484 ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 485 {
 486         enum ice_status ret_code = ICE_SUCCESS;
 487
 488         ice_acquire_lock(&cq->rq_lock);
 489
 490         if (!cq->rq.count) {
 491                 ret_code = ICE_ERR_NOT_READY;
 492                 goto shutdown_rq_out;
 493         }
 494
 495         /* Stop Control Queue processing */
 496         wr32(hw, cq->rq.head, 0);
 497         wr32(hw, cq->rq.tail, 0);
 498         wr32(hw, cq->rq.len, 0);
 499         wr32(hw, cq->rq.bal, 0);
 500         wr32(hw, cq->rq.bah, 0);
 501
 502         /* set rq.count to 0 to indicate uninitialized queue */
 503         cq->rq.count = 0;
 504
 505         /* free ring buffers and the ring itself */
 506         ICE_FREE_CQ_BUFS(hw, cq, rq);
 507         ice_free_cq_ring(hw, &cq->rq);
 508
 509 shutdown_rq_out:
 510         ice_release_lock(&cq->rq_lock);
 511         return ret_code;
 512 }
 513
 514
 515 /**
 516  * ice_init_check_adminq - Check version for Admin Queue to know if its alive
 517  * @hw: pointer to the hardware structure
 518  */
 519 static enum ice_status ice_init_check_adminq(struct ice_hw *hw)
 520 {
 521         struct ice_ctl_q_info *cq = &hw->adminq;
 522         enum ice_status status;
 523
 524
 525         status = ice_aq_get_fw_ver(hw, NULL);
 526         if (status)
 527                 goto init_ctrlq_free_rq;
 528
 529
 530         if (!ice_aq_ver_check(hw)) {
 531                 status = ICE_ERR_FW_API_VER;
 532                 goto init_ctrlq_free_rq;
 533         }
 534
 535         return ICE_SUCCESS;
 536
 537 init_ctrlq_free_rq:
 538         if (cq->rq.count) {
 539                 ice_shutdown_rq(hw, cq);
 540                 ice_destroy_lock(&cq->rq_lock);
 541         }
 542         if (cq->sq.count) {
 543                 ice_shutdown_sq(hw, cq);
 544                 ice_destroy_lock(&cq->sq_lock);
 545         }
 546         return status;
 547 }
 548
 549 /**
 550  * ice_init_ctrlq - main initialization routine for any control Queue
 551  * @hw: pointer to the hardware structure
 552  * @q_type: specific Control queue type
 553  *
 554  * Prior to calling this function, drivers *MUST* set the following fields
 555  * in the cq->structure:
 556  *     - cq->num_sq_entries
 557  *     - cq->num_rq_entries
 558  *     - cq->rq_buf_size
 559  *     - cq->sq_buf_size
 560  */
 561 static enum ice_status ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 562 {
 563         struct ice_ctl_q_info *cq;
 564         enum ice_status ret_code;
 565
 566         switch (q_type) {
 567         case ICE_CTL_Q_ADMIN:
 568                 ice_adminq_init_regs(hw);
 569                 cq = &hw->adminq;
 570                 break;
 571         case ICE_CTL_Q_MAILBOX:
 572                 ice_mailbox_init_regs(hw);
 573                 cq = &hw->mailboxq;
 574                 break;
 575         default:
 576                 return ICE_ERR_PARAM;
 577         }
 578         cq->qtype = q_type;
 579
 580         /* verify input for valid configuration */
 581         if (!cq->num_rq_entries || !cq->num_sq_entries ||
 582             !cq->rq_buf_size || !cq->sq_buf_size) {
 583                 return ICE_ERR_CFG;
 584         }
 585         ice_init_lock(&cq->sq_lock);
 586         ice_init_lock(&cq->rq_lock);
 587
 588         /* setup SQ command write back timeout */
 589         cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;
 590
 591         /* allocate the ATQ */
 592         ret_code = ice_init_sq(hw, cq);
 593         if (ret_code)
 594                 goto init_ctrlq_destroy_locks;
 595
 596         /* allocate the ARQ */
 597         ret_code = ice_init_rq(hw, cq);
 598         if (ret_code)
 599                 goto init_ctrlq_free_sq;
 600
 601         /* success! */
 602         return ICE_SUCCESS;
 603
 604 init_ctrlq_free_sq:
 605         ice_shutdown_sq(hw, cq);
 606 init_ctrlq_destroy_locks:
 607         ice_destroy_lock(&cq->sq_lock);
 608         ice_destroy_lock(&cq->rq_lock);
 609         return ret_code;
 610 }
 611
 612 /**
 613  * ice_init_all_ctrlq - main initialization routine for all control queues
 614  * @hw: pointer to the hardware structure
 615  *
 616  * Prior to calling this function, drivers *MUST* set the following fields
 617  * in the cq->structure for all control queues:
 618  *     - cq->num_sq_entries
 619  *     - cq->num_rq_entries
 620  *     - cq->rq_buf_size
 621  *     - cq->sq_buf_size
 622  */
 623 enum ice_status ice_init_all_ctrlq(struct ice_hw *hw)
 624 {
 625         enum ice_status ret_code;
 626
 627
 628         /* Init FW admin queue */
 629         ret_code = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
 630         if (ret_code)
 631                 return ret_code;
 632
 633         ret_code = ice_init_check_adminq(hw);
 634         if (ret_code)
 635                 return ret_code;
 636         /* Init Mailbox queue */
 637         return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);
 638 }
 639
 640 /**
 641  * ice_shutdown_ctrlq - shutdown routine for any control queue
 642  * @hw: pointer to the hardware structure
 643  * @q_type: specific Control queue type
 644  */
 645 static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 646 {
 647         struct ice_ctl_q_info *cq;
 648
 649         switch (q_type) {
 650         case ICE_CTL_Q_ADMIN:
 651                 cq = &hw->adminq;
 652                 if (ice_check_sq_alive(hw, cq))
 653                         ice_aq_q_shutdown(hw, true);
 654                 break;
 655         case ICE_CTL_Q_MAILBOX:
 656                 cq = &hw->mailboxq;
 657                 break;
 658         default:
 659                 return;
 660         }
 661
 662         if (cq->sq.count) {
 663                 ice_shutdown_sq(hw, cq);
 664                 ice_destroy_lock(&cq->sq_lock);
 665         }
 666         if (cq->rq.count) {
 667                 ice_shutdown_rq(hw, cq);
 668                 ice_destroy_lock(&cq->rq_lock);
 669         }
 670 }
 671
 672 /**
 673  * ice_shutdown_all_ctrlq - shutdown routine for all control queues
 674  * @hw: pointer to the hardware structure
 675  */
 676 void ice_shutdown_all_ctrlq(struct ice_hw *hw)
 677 {
 678         /* Shutdown FW admin queue */
 679         ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
 680         /* Shutdown PF-VF Mailbox */
 681         ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX);
 682 }
 683
 684 /**
 685  * ice_clean_sq - cleans Admin send queue (ATQ)
 686  * @hw: pointer to the hardware structure
 687  * @cq: pointer to the specific Control queue
 688  *
 689  * returns the number of free desc
 690  */
 691 static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 692 {
 693         struct ice_ctl_q_ring *sq = &cq->sq;
 694         u16 ntc = sq->next_to_clean;
 695         struct ice_sq_cd *details;
 696 #if 0
 697         struct ice_aq_desc desc_cb;
 698 #endif
 699         struct ice_aq_desc *desc;
 700
 701         desc = ICE_CTL_Q_DESC(*sq, ntc);
 702         details = ICE_CTL_Q_DETAILS(*sq, ntc);
 703
 704         while (rd32(hw, cq->sq.head) != ntc) {
 705                 ice_debug(hw, ICE_DBG_AQ_MSG,
 706                           "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head));
 707 #if 0
 708                 if (details->callback) {
 709                         ICE_CTL_Q_CALLBACK cb_func =
 710                                 (ICE_CTL_Q_CALLBACK)details->callback;
 711                         ice_memcpy(&desc_cb, desc, sizeof(desc_cb),
 712                                    ICE_DMA_TO_DMA);
 713                         cb_func(hw, &desc_cb);
 714                 }
 715 #endif
 716                 ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
 717                 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
 718                 ntc++;
 719                 if (ntc == sq->count)
 720                         ntc = 0;
 721                 desc = ICE_CTL_Q_DESC(*sq, ntc);
 722                 details = ICE_CTL_Q_DETAILS(*sq, ntc);
 723         }
 724
 725         sq->next_to_clean = ntc;
 726
 727         return ICE_CTL_Q_DESC_UNUSED(sq);
 728 }
 729
 730 /**
 731  * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ)
 732  * @hw: pointer to the hw struct
 733  * @cq: pointer to the specific Control queue
 734  *
 735  * Returns true if the firmware has processed all descriptors on the
 736  * admin send queue. Returns false if there are still requests pending.
 737  */
 738 static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 739 {
 740         /* AQ designers suggest use of head for better
 741          * timing reliability than DD bit
 742          */
 743         return rd32(hw, cq->sq.head) == cq->sq.next_to_use;
 744 }
 745
 746 /**
 747  * ice_sq_send_cmd - send command to Control Queue (ATQ)
 748  * @hw: pointer to the hw struct
 749  * @cq: pointer to the specific Control queue
 750  * @desc: prefilled descriptor describing the command (non DMA mem)
 751  * @buf: buffer to use for indirect commands (or NULL for direct commands)
 752  * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
 753  * @cd: pointer to command details structure
 754  *
 755  * This is the main send command routine for the ATQ. It runs the queue,
 756  * cleans the queue, etc.
 757  */
 758 enum ice_status
 759 ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 760                 struct ice_aq_desc *desc, void *buf, u16 buf_size,
 761                 struct ice_sq_cd *cd)
 762 {
 763         struct ice_dma_mem *dma_buf = NULL;
 764         struct ice_aq_desc *desc_on_ring;
 765         bool cmd_completed = false;
 766         enum ice_status status = ICE_SUCCESS;
 767         struct ice_sq_cd *details;
 768         u32 total_delay = 0;
 769         u16 retval = 0;
 770         u32 val = 0;
 771
 772         /* if reset is in progress return a soft error */
 773         if (hw->reset_ongoing)
 774                 return ICE_ERR_RESET_ONGOING;
 775         ice_acquire_lock(&cq->sq_lock);
 776
 777         cq->sq_last_status = ICE_AQ_RC_OK;
 778
 779         if (!cq->sq.count) {
 780                 ice_debug(hw, ICE_DBG_AQ_MSG,
 781                           "Control Send queue not initialized.\n");
 782                 status = ICE_ERR_AQ_EMPTY;
 783                 goto sq_send_command_error;
 784         }
 785
 786         if ((buf && !buf_size) || (!buf && buf_size)) {
 787                 status = ICE_ERR_PARAM;
 788                 goto sq_send_command_error;
 789         }
 790
 791         if (buf) {
 792                 if (buf_size > cq->sq_buf_size) {
 793                         ice_debug(hw, ICE_DBG_AQ_MSG,
 794                                   "Invalid buffer size for Control Send queue: %d.\n",
 795                                   buf_size);
 796                         status = ICE_ERR_INVAL_SIZE;
 797                         goto sq_send_command_error;
 798                 }
 799
 800                 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_BUF);
 801                 if (buf_size > ICE_AQ_LG_BUF)
 802                         desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
 803         }
 804
 805         val = rd32(hw, cq->sq.head);
 806         if (val >= cq->num_sq_entries) {
 807                 ice_debug(hw, ICE_DBG_AQ_MSG,
 808                           "head overrun at %d in the Control Send Queue ring\n",
 809                           val);
 810                 status = ICE_ERR_AQ_EMPTY;
 811                 goto sq_send_command_error;
 812         }
 813
 814         details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use);
 815         if (cd)
 816                 *details = *cd;
 817 #if 0
 818                 /* FIXME: if/when this block gets enabled (when the #if 0
 819                  * is removed), add braces to both branches of the surrounding
 820                  * conditional expression. The braces have been removed to
 821                  * prevent checkpatch complaining.
 822                  */
 823
 824                 /* If the command details are defined copy the cookie. The
 825                  * CPU_TO_LE32 is not needed here because the data is ignored
 826                  * by the FW, only used by the driver
 827                  */
 828                 if (details->cookie) {
 829                         desc->cookie_high =
 830                                 CPU_TO_LE32(ICE_HI_DWORD(details->cookie));
 831                         desc->cookie_low =
 832                                 CPU_TO_LE32(ICE_LO_DWORD(details->cookie));
 833                 }
 834 #endif
 835         else
 836                 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
 837 #if 0
 838         /* clear requested flags and then set additional flags if defined */
 839         desc->flags &= ~CPU_TO_LE16(details->flags_dis);
 840         desc->flags |= CPU_TO_LE16(details->flags_ena);
 841
 842         if (details->postpone && !details->async) {
 843                 ice_debug(hw, ICE_DBG_AQ_MSG,
 844                           "Async flag not set along with postpone flag\n");
 845                 status = ICE_ERR_PARAM;
 846                 goto sq_send_command_error;
 847         }
 848 #endif
 849
 850         /* Call clean and check queue available function to reclaim the
 851          * descriptors that were processed by FW/MBX; the function returns the
 852          * number of desc available. The clean function called here could be
 853          * called in a separate thread in case of asynchronous completions.
 854          */
 855         if (ice_clean_sq(hw, cq) == 0) {
 856                 ice_debug(hw, ICE_DBG_AQ_MSG,
 857                           "Error: Control Send Queue is full.\n");
 858                 status = ICE_ERR_AQ_FULL;
 859                 goto sq_send_command_error;
 860         }
 861
 862         /* initialize the temp desc pointer with the right desc */
 863         desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use);
 864
 865         /* if the desc is available copy the temp desc to the right place */
 866         ice_memcpy(desc_on_ring, desc, sizeof(*desc_on_ring),
 867                    ICE_NONDMA_TO_DMA);
 868
 869         /* if buf is not NULL assume indirect command */
 870         if (buf) {
 871                 dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use];
 872                 /* copy the user buf into the respective DMA buf */
 873                 ice_memcpy(dma_buf->va, buf, buf_size, ICE_NONDMA_TO_DMA);
 874                 desc_on_ring->datalen = CPU_TO_LE16(buf_size);
 875
 876                 /* Update the address values in the desc with the pa value
 877                  * for respective buffer
 878                  */
 879                 desc_on_ring->params.generic.addr_high =
 880                         CPU_TO_LE32(ICE_HI_DWORD(dma_buf->pa));
 881                 desc_on_ring->params.generic.addr_low =
 882                         CPU_TO_LE32(ICE_LO_DWORD(dma_buf->pa));
 883         }
 884
 885         /* Debug desc and buffer */
 886         ice_debug(hw, ICE_DBG_AQ_MSG,
 887                   "ATQ: Control Send queue desc and buffer:\n");
 888
 889         ice_debug_cq(hw, ICE_DBG_AQ_CMD, (void *)desc_on_ring, buf, buf_size);
 890
 891
 892         (cq->sq.next_to_use)++;
 893         if (cq->sq.next_to_use == cq->sq.count)
 894                 cq->sq.next_to_use = 0;
 895 #if 0
 896         /* FIXME - handle this case? */
 897         if (!details->postpone)
 898 #endif
 899         wr32(hw, cq->sq.tail, cq->sq.next_to_use);
 900
 901 #if 0
 902         /* if command details are not defined or async flag is not set,
 903          * we need to wait for desc write back
 904          */
 905         if (!details->async && !details->postpone) {
 906                 /* FIXME - handle this case? */
 907         }
 908 #endif
 909         do {
 910                 if (ice_sq_done(hw, cq))
 911                         break;
 912
 913                 ice_msec_delay(1, false);
 914                 total_delay++;
 915         } while (total_delay < cq->sq_cmd_timeout);
 916
 917         /* if ready, copy the desc back to temp */
 918         if (ice_sq_done(hw, cq)) {
 919                 ice_memcpy(desc, desc_on_ring, sizeof(*desc),
 920                            ICE_DMA_TO_NONDMA);
 921                 if (buf) {
 922                         /* get returned length to copy */
 923                         u16 copy_size = LE16_TO_CPU(desc->datalen);
 924
 925                         if (copy_size > buf_size) {
 926                                 ice_debug(hw, ICE_DBG_AQ_MSG,
 927                                           "Return len %d > than buf len %d\n",
 928                                           copy_size, buf_size);
 929                                 status = ICE_ERR_AQ_ERROR;
 930                         } else {
 931                                 ice_memcpy(buf, dma_buf->va, copy_size,
 932                                            ICE_DMA_TO_NONDMA);
 933                         }
 934                 }
 935                 retval = LE16_TO_CPU(desc->retval);
 936                 if (retval) {
 937                         ice_debug(hw, ICE_DBG_AQ_MSG,
 938                                   "Control Send Queue command completed with error 0x%x\n",
 939                                   retval);
 940
 941                         /* strip off FW internal code */
 942                         retval &= 0xff;
 943                 }
 944                 cmd_completed = true;
 945                 if (!status && retval != ICE_AQ_RC_OK)
 946                         status = ICE_ERR_AQ_ERROR;
 947                 cq->sq_last_status = (enum ice_aq_err)retval;
 948         }
 949
 950         ice_debug(hw, ICE_DBG_AQ_MSG,
 951                   "ATQ: desc and buffer writeback:\n");
 952
 953         ice_debug_cq(hw, ICE_DBG_AQ_CMD, (void *)desc, buf, buf_size);
 954
 955
 956         /* save writeback AQ if requested */
 957         if (details->wb_desc)
 958                 ice_memcpy(details->wb_desc, desc_on_ring,
 959                            sizeof(*details->wb_desc), ICE_DMA_TO_NONDMA);
 960
 961         /* update the error if time out occurred */
 962         if (!cmd_completed) {
 963 #if 0
 964             (!details->async && !details->postpone)) {
 965 #endif
 966                 ice_debug(hw, ICE_DBG_AQ_MSG,
 967                           "Control Send Queue Writeback timeout.\n");
 968                 status = ICE_ERR_AQ_TIMEOUT;
 969         }
 970
 971 sq_send_command_error:
 972         ice_release_lock(&cq->sq_lock);
 973         return status;
 974 }
 975
 976 /**
 977  * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function
 978  * @desc: pointer to the temp descriptor (non DMA mem)
 979  * @opcode: the opcode can be used to decide which flags to turn off or on
 980  *
 981  * Fill the desc with default values
 982  */
 983 void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode)
 984 {
 985         /* zero out the desc */
 986         ice_memset(desc, 0, sizeof(*desc), ICE_NONDMA_MEM);
 987         desc->opcode = CPU_TO_LE16(opcode);
 988         desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_SI);
 989 }
 990
 991 /**
 992  * ice_clean_rq_elem
 993  * @hw: pointer to the hw struct
 994  * @cq: pointer to the specific Control queue
 995  * @e: event info from the receive descriptor, includes any buffers
 996  * @pending: number of events that could be left to process
 997  *
 998  * This function cleans one Admin Receive Queue element and returns
 999  * the contents through e. It can also return how many events are
1000  * left to process through 'pending'.
1001  */
1002 enum ice_status
1003 ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq,
1004                   struct ice_rq_event_info *e, u16 *pending)
1005 {
1006         u16 ntc = cq->rq.next_to_clean;
1007         enum ice_status ret_code = ICE_SUCCESS;
1008         struct ice_aq_desc *desc;
1009         struct ice_dma_mem *bi;
1010         u16 desc_idx;
1011         u16 datalen;
1012         u16 flags;
1013         u16 ntu;
1014
1015         /* pre-clean the event info */
1016         ice_memset(&e->desc, 0, sizeof(e->desc), ICE_NONDMA_MEM);
1017
1018         /* take the lock before we start messing with the ring */
1019         ice_acquire_lock(&cq->rq_lock);
1020
1021         if (!cq->rq.count) {
1022                 ice_debug(hw, ICE_DBG_AQ_MSG,
1023                           "Control Receive queue not initialized.\n");
1024                 ret_code = ICE_ERR_AQ_EMPTY;
1025                 goto clean_rq_elem_err;
1026         }
1027
1028         /* set next_to_use to head */
1029         ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1030
1031         if (ntu == ntc) {
1032                 /* nothing to do - shouldn't need to update ring's values */
1033                 ret_code = ICE_ERR_AQ_NO_WORK;
1034                 goto clean_rq_elem_out;
1035         }
1036
1037         /* now clean the next descriptor */
1038         desc = ICE_CTL_Q_DESC(cq->rq, ntc);
1039         desc_idx = ntc;
1040
1041         cq->rq_last_status = (enum ice_aq_err)LE16_TO_CPU(desc->retval);
1042         flags = LE16_TO_CPU(desc->flags);
1043         if (flags & ICE_AQ_FLAG_ERR) {
1044                 ret_code = ICE_ERR_AQ_ERROR;
1045                 ice_debug(hw, ICE_DBG_AQ_MSG,
1046                           "Control Receive Queue Event received with error 0x%x\n",
1047                           cq->rq_last_status);
1048         }
1049         ice_memcpy(&e->desc, desc, sizeof(e->desc), ICE_DMA_TO_NONDMA);
1050         datalen = LE16_TO_CPU(desc->datalen);
1051         e->msg_len = min(datalen, e->buf_len);
1052         if (e->msg_buf && e->msg_len)
1053                 ice_memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va,
1054                            e->msg_len, ICE_DMA_TO_NONDMA);
1055
1056         ice_debug(hw, ICE_DBG_AQ_MSG, "ARQ: desc and buffer:\n");
1057
1058         ice_debug_cq(hw, ICE_DBG_AQ_CMD, (void *)desc, e->msg_buf,
1059                      cq->rq_buf_size);
1060
1061
1062         /* Restore the original datalen and buffer address in the desc,
1063          * FW updates datalen to indicate the event message size
1064          */
1065         bi = &cq->rq.r.rq_bi[ntc];
1066         ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
1067
1068         desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
1069         if (cq->rq_buf_size > ICE_AQ_LG_BUF)
1070                 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
1071         desc->datalen = CPU_TO_LE16(bi->size);
1072         desc->params.generic.addr_high = CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
1073         desc->params.generic.addr_low = CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
1074
1075         /* set tail = the last cleaned desc index. */
1076         wr32(hw, cq->rq.tail, ntc);
1077         /* ntc is updated to tail + 1 */
1078         ntc++;
1079         if (ntc == cq->num_rq_entries)
1080                 ntc = 0;
1081         cq->rq.next_to_clean = ntc;
1082         cq->rq.next_to_use = ntu;
1083
1084 #if 0
1085         ice_nvmupd_check_wait_event(hw, LE16_TO_CPU(e->desc.opcode));
1086 #endif
1087 clean_rq_elem_out:
1088         /* Set pending if needed, unlock and return */
1089         if (pending) {
1090                 /* re-read HW head to calculate actual pending messages */
1091                 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1092                 *pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc));
1093         }
1094 clean_rq_elem_err:
1095         ice_release_lock(&cq->rq_lock);
1096
1097         return ret_code;
1098 }