1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2001-2021 Intel Corporation
5 #include "ice_common.h"
7 #define ICE_CQ_INIT_REGS(qinfo, prefix) \
9 (qinfo)->sq.head = prefix##_ATQH; \
10 (qinfo)->sq.tail = prefix##_ATQT; \
11 (qinfo)->sq.len = prefix##_ATQLEN; \
12 (qinfo)->sq.bah = prefix##_ATQBAH; \
13 (qinfo)->sq.bal = prefix##_ATQBAL; \
14 (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M; \
15 (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \
16 (qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M; \
17 (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M; \
18 (qinfo)->rq.head = prefix##_ARQH; \
19 (qinfo)->rq.tail = prefix##_ARQT; \
20 (qinfo)->rq.len = prefix##_ARQLEN; \
21 (qinfo)->rq.bah = prefix##_ARQBAH; \
22 (qinfo)->rq.bal = prefix##_ARQBAL; \
23 (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M; \
24 (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \
25 (qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M; \
26 (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M; \
30 * ice_adminq_init_regs - Initialize AdminQ registers
31 * @hw: pointer to the hardware structure
33 * This assumes the alloc_sq and alloc_rq functions have already been called
35 static void ice_adminq_init_regs(struct ice_hw *hw)
37 struct ice_ctl_q_info *cq = &hw->adminq;
39 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
41 ICE_CQ_INIT_REGS(cq, PF_FW);
45 * ice_mailbox_init_regs - Initialize Mailbox registers
46 * @hw: pointer to the hardware structure
48 * This assumes the alloc_sq and alloc_rq functions have already been called
50 static void ice_mailbox_init_regs(struct ice_hw *hw)
52 struct ice_ctl_q_info *cq = &hw->mailboxq;
54 ICE_CQ_INIT_REGS(cq, PF_MBX);
59 * @hw: pointer to the HW struct
60 * @cq: pointer to the specific Control queue
62 * Returns true if Queue is enabled else false.
64 bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq)
66 /* check both queue-length and queue-enable fields */
67 if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask)
68 return (rd32(hw, cq->sq.len) & (cq->sq.len_mask |
69 cq->sq.len_ena_mask)) ==
70 (cq->num_sq_entries | cq->sq.len_ena_mask);
76 * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings
77 * @hw: pointer to the hardware structure
78 * @cq: pointer to the specific Control queue
80 static enum ice_status
81 ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
83 size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc);
85 cq->sq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->sq.desc_buf, size);
86 if (!cq->sq.desc_buf.va)
87 return ICE_ERR_NO_MEMORY;
89 cq->sq.cmd_buf = ice_calloc(hw, cq->num_sq_entries,
90 sizeof(struct ice_sq_cd));
91 if (!cq->sq.cmd_buf) {
92 ice_free_dma_mem(hw, &cq->sq.desc_buf);
93 return ICE_ERR_NO_MEMORY;
100 * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings
101 * @hw: pointer to the hardware structure
102 * @cq: pointer to the specific Control queue
104 static enum ice_status
105 ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
107 size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc);
109 cq->rq.desc_buf.va = ice_alloc_dma_mem(hw, &cq->rq.desc_buf, size);
110 if (!cq->rq.desc_buf.va)
111 return ICE_ERR_NO_MEMORY;
116 * ice_free_cq_ring - Free control queue ring
117 * @hw: pointer to the hardware structure
118 * @ring: pointer to the specific control queue ring
120 * This assumes the posted buffers have already been cleaned
123 static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)
125 ice_free_dma_mem(hw, &ring->desc_buf);
129 * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ
130 * @hw: pointer to the hardware structure
131 * @cq: pointer to the specific Control queue
133 static enum ice_status
134 ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
138 /* We'll be allocating the buffer info memory first, then we can
139 * allocate the mapped buffers for the event processing
141 cq->rq.dma_head = ice_calloc(hw, cq->num_rq_entries,
142 sizeof(cq->rq.desc_buf));
143 if (!cq->rq.dma_head)
144 return ICE_ERR_NO_MEMORY;
145 cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head;
147 /* allocate the mapped buffers */
148 for (i = 0; i < cq->num_rq_entries; i++) {
149 struct ice_aq_desc *desc;
150 struct ice_dma_mem *bi;
152 bi = &cq->rq.r.rq_bi[i];
153 bi->va = ice_alloc_dma_mem(hw, bi, cq->rq_buf_size);
155 goto unwind_alloc_rq_bufs;
157 /* now configure the descriptors for use */
158 desc = ICE_CTL_Q_DESC(cq->rq, i);
160 desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
161 if (cq->rq_buf_size > ICE_AQ_LG_BUF)
162 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
164 /* This is in accordance with Admin queue design, there is no
165 * register for buffer size configuration
167 desc->datalen = CPU_TO_LE16(bi->size);
169 desc->cookie_high = 0;
170 desc->cookie_low = 0;
171 desc->params.generic.addr_high =
172 CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
173 desc->params.generic.addr_low =
174 CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
175 desc->params.generic.param0 = 0;
176 desc->params.generic.param1 = 0;
180 unwind_alloc_rq_bufs:
181 /* don't try to free the one that failed... */
184 ice_free_dma_mem(hw, &cq->rq.r.rq_bi[i]);
185 cq->rq.r.rq_bi = NULL;
186 ice_free(hw, cq->rq.dma_head);
187 cq->rq.dma_head = NULL;
189 return ICE_ERR_NO_MEMORY;
193 * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ
194 * @hw: pointer to the hardware structure
195 * @cq: pointer to the specific Control queue
197 static enum ice_status
198 ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
202 /* No mapped memory needed yet, just the buffer info structures */
203 cq->sq.dma_head = ice_calloc(hw, cq->num_sq_entries,
204 sizeof(cq->sq.desc_buf));
205 if (!cq->sq.dma_head)
206 return ICE_ERR_NO_MEMORY;
207 cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head;
209 /* allocate the mapped buffers */
210 for (i = 0; i < cq->num_sq_entries; i++) {
211 struct ice_dma_mem *bi;
213 bi = &cq->sq.r.sq_bi[i];
214 bi->va = ice_alloc_dma_mem(hw, bi, cq->sq_buf_size);
216 goto unwind_alloc_sq_bufs;
220 unwind_alloc_sq_bufs:
221 /* don't try to free the one that failed... */
224 ice_free_dma_mem(hw, &cq->sq.r.sq_bi[i]);
225 cq->sq.r.sq_bi = NULL;
226 ice_free(hw, cq->sq.dma_head);
227 cq->sq.dma_head = NULL;
229 return ICE_ERR_NO_MEMORY;
232 static enum ice_status
233 ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries)
235 /* Clear Head and Tail */
236 wr32(hw, ring->head, 0);
237 wr32(hw, ring->tail, 0);
239 /* set starting point */
240 wr32(hw, ring->len, (num_entries | ring->len_ena_mask));
241 wr32(hw, ring->bal, ICE_LO_DWORD(ring->desc_buf.pa));
242 wr32(hw, ring->bah, ICE_HI_DWORD(ring->desc_buf.pa));
244 /* Check one register to verify that config was applied */
245 if (rd32(hw, ring->bal) != ICE_LO_DWORD(ring->desc_buf.pa))
246 return ICE_ERR_AQ_ERROR;
252 * ice_cfg_sq_regs - configure Control ATQ registers
253 * @hw: pointer to the hardware structure
254 * @cq: pointer to the specific Control queue
256 * Configure base address and length registers for the transmit queue
258 static enum ice_status
259 ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
261 return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);
265 * ice_cfg_rq_regs - configure Control ARQ register
266 * @hw: pointer to the hardware structure
267 * @cq: pointer to the specific Control queue
269 * Configure base address and length registers for the receive (event queue)
271 static enum ice_status
272 ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
274 enum ice_status status;
276 status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);
280 /* Update tail in the HW to post pre-allocated buffers */
281 wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1));
286 #define ICE_FREE_CQ_BUFS(hw, qi, ring) \
288 /* free descriptors */ \
289 if ((qi)->ring.r.ring##_bi) { \
292 for (i = 0; i < (qi)->num_##ring##_entries; i++) \
293 if ((qi)->ring.r.ring##_bi[i].pa) \
294 ice_free_dma_mem((hw), \
295 &(qi)->ring.r.ring##_bi[i]); \
297 /* free the buffer info list */ \
298 if ((qi)->ring.cmd_buf) \
299 ice_free(hw, (qi)->ring.cmd_buf); \
300 /* free DMA head */ \
301 ice_free(hw, (qi)->ring.dma_head); \
305 * ice_init_sq - main initialization routine for Control ATQ
306 * @hw: pointer to the hardware structure
307 * @cq: pointer to the specific Control queue
309 * This is the main initialization routine for the Control Send Queue
310 * Prior to calling this function, the driver *MUST* set the following fields
311 * in the cq->structure:
312 * - cq->num_sq_entries
315 * Do *NOT* hold the lock when calling this as the memory allocation routines
316 * called are not going to be atomic context safe
318 static enum ice_status ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
320 enum ice_status ret_code;
322 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
324 if (cq->sq.count > 0) {
325 /* queue already initialized */
326 ret_code = ICE_ERR_NOT_READY;
327 goto init_ctrlq_exit;
330 /* verify input for valid configuration */
331 if (!cq->num_sq_entries || !cq->sq_buf_size) {
332 ret_code = ICE_ERR_CFG;
333 goto init_ctrlq_exit;
336 cq->sq.next_to_use = 0;
337 cq->sq.next_to_clean = 0;
339 /* allocate the ring memory */
340 ret_code = ice_alloc_ctrlq_sq_ring(hw, cq);
342 goto init_ctrlq_exit;
344 /* allocate buffers in the rings */
345 ret_code = ice_alloc_sq_bufs(hw, cq);
347 goto init_ctrlq_free_rings;
349 /* initialize base registers */
350 ret_code = ice_cfg_sq_regs(hw, cq);
352 goto init_ctrlq_free_rings;
355 cq->sq.count = cq->num_sq_entries;
356 goto init_ctrlq_exit;
358 init_ctrlq_free_rings:
359 ICE_FREE_CQ_BUFS(hw, cq, sq);
360 ice_free_cq_ring(hw, &cq->sq);
367 * ice_init_rq - initialize ARQ
368 * @hw: pointer to the hardware structure
369 * @cq: pointer to the specific Control queue
371 * The main initialization routine for the Admin Receive (Event) Queue.
372 * Prior to calling this function, the driver *MUST* set the following fields
373 * in the cq->structure:
374 * - cq->num_rq_entries
377 * Do *NOT* hold the lock when calling this as the memory allocation routines
378 * called are not going to be atomic context safe
380 static enum ice_status ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
382 enum ice_status ret_code;
384 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
386 if (cq->rq.count > 0) {
387 /* queue already initialized */
388 ret_code = ICE_ERR_NOT_READY;
389 goto init_ctrlq_exit;
392 /* verify input for valid configuration */
393 if (!cq->num_rq_entries || !cq->rq_buf_size) {
394 ret_code = ICE_ERR_CFG;
395 goto init_ctrlq_exit;
398 cq->rq.next_to_use = 0;
399 cq->rq.next_to_clean = 0;
401 /* allocate the ring memory */
402 ret_code = ice_alloc_ctrlq_rq_ring(hw, cq);
404 goto init_ctrlq_exit;
406 /* allocate buffers in the rings */
407 ret_code = ice_alloc_rq_bufs(hw, cq);
409 goto init_ctrlq_free_rings;
411 /* initialize base registers */
412 ret_code = ice_cfg_rq_regs(hw, cq);
414 goto init_ctrlq_free_rings;
417 cq->rq.count = cq->num_rq_entries;
418 goto init_ctrlq_exit;
420 init_ctrlq_free_rings:
421 ICE_FREE_CQ_BUFS(hw, cq, rq);
422 ice_free_cq_ring(hw, &cq->rq);
429 * ice_shutdown_sq - shutdown the Control ATQ
430 * @hw: pointer to the hardware structure
431 * @cq: pointer to the specific Control queue
433 * The main shutdown routine for the Control Transmit Queue
435 static enum ice_status
436 ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
438 enum ice_status ret_code = ICE_SUCCESS;
440 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
442 ice_acquire_lock(&cq->sq_lock);
445 ret_code = ICE_ERR_NOT_READY;
446 goto shutdown_sq_out;
449 /* Stop firmware AdminQ processing */
450 wr32(hw, cq->sq.head, 0);
451 wr32(hw, cq->sq.tail, 0);
452 wr32(hw, cq->sq.len, 0);
453 wr32(hw, cq->sq.bal, 0);
454 wr32(hw, cq->sq.bah, 0);
456 cq->sq.count = 0; /* to indicate uninitialized queue */
458 /* free ring buffers and the ring itself */
459 ICE_FREE_CQ_BUFS(hw, cq, sq);
460 ice_free_cq_ring(hw, &cq->sq);
463 ice_release_lock(&cq->sq_lock);
468 * ice_aq_ver_check - Check the reported AQ API version.
469 * @hw: pointer to the hardware structure
471 * Checks if the driver should load on a given AQ API version.
473 * Return: 'true' iff the driver should attempt to load. 'false' otherwise.
475 static bool ice_aq_ver_check(struct ice_hw *hw)
477 if (hw->api_maj_ver > EXP_FW_API_VER_MAJOR) {
478 /* Major API version is newer than expected, don't load */
479 ice_warn(hw, "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
481 } else if (hw->api_maj_ver == EXP_FW_API_VER_MAJOR) {
482 if (hw->api_min_ver > (EXP_FW_API_VER_MINOR + 2))
483 ice_info(hw, "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
484 else if ((hw->api_min_ver + 2) < EXP_FW_API_VER_MINOR)
485 ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
487 /* Major API version is older than expected, log a warning */
488 ice_info(hw, "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
494 * ice_shutdown_rq - shutdown Control ARQ
495 * @hw: pointer to the hardware structure
496 * @cq: pointer to the specific Control queue
498 * The main shutdown routine for the Control Receive Queue
500 static enum ice_status
501 ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
503 enum ice_status ret_code = ICE_SUCCESS;
505 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
507 ice_acquire_lock(&cq->rq_lock);
510 ret_code = ICE_ERR_NOT_READY;
511 goto shutdown_rq_out;
514 /* Stop Control Queue processing */
515 wr32(hw, cq->rq.head, 0);
516 wr32(hw, cq->rq.tail, 0);
517 wr32(hw, cq->rq.len, 0);
518 wr32(hw, cq->rq.bal, 0);
519 wr32(hw, cq->rq.bah, 0);
521 /* set rq.count to 0 to indicate uninitialized queue */
524 /* free ring buffers and the ring itself */
525 ICE_FREE_CQ_BUFS(hw, cq, rq);
526 ice_free_cq_ring(hw, &cq->rq);
529 ice_release_lock(&cq->rq_lock);
534 * ice_init_check_adminq - Check version for Admin Queue to know if its alive
535 * @hw: pointer to the hardware structure
537 static enum ice_status ice_init_check_adminq(struct ice_hw *hw)
539 struct ice_ctl_q_info *cq = &hw->adminq;
540 enum ice_status status;
542 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
544 status = ice_aq_get_fw_ver(hw, NULL);
546 goto init_ctrlq_free_rq;
548 if (!ice_aq_ver_check(hw)) {
549 status = ICE_ERR_FW_API_VER;
550 goto init_ctrlq_free_rq;
556 ice_shutdown_rq(hw, cq);
557 ice_shutdown_sq(hw, cq);
562 * ice_init_ctrlq - main initialization routine for any control Queue
563 * @hw: pointer to the hardware structure
564 * @q_type: specific Control queue type
566 * Prior to calling this function, the driver *MUST* set the following fields
567 * in the cq->structure:
568 * - cq->num_sq_entries
569 * - cq->num_rq_entries
573 * NOTE: this function does not initialize the controlq locks
575 static enum ice_status ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
577 struct ice_ctl_q_info *cq;
578 enum ice_status ret_code;
580 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
583 case ICE_CTL_Q_ADMIN:
584 ice_adminq_init_regs(hw);
587 case ICE_CTL_Q_MAILBOX:
588 ice_mailbox_init_regs(hw);
592 return ICE_ERR_PARAM;
596 /* verify input for valid configuration */
597 if (!cq->num_rq_entries || !cq->num_sq_entries ||
598 !cq->rq_buf_size || !cq->sq_buf_size) {
602 /* setup SQ command write back timeout */
603 cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;
605 /* allocate the ATQ */
606 ret_code = ice_init_sq(hw, cq);
610 /* allocate the ARQ */
611 ret_code = ice_init_rq(hw, cq);
613 goto init_ctrlq_free_sq;
619 ice_shutdown_sq(hw, cq);
624 * ice_shutdown_ctrlq - shutdown routine for any control queue
625 * @hw: pointer to the hardware structure
626 * @q_type: specific Control queue type
628 * NOTE: this function does not destroy the control queue locks.
630 static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
632 struct ice_ctl_q_info *cq;
634 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
637 case ICE_CTL_Q_ADMIN:
639 if (ice_check_sq_alive(hw, cq))
640 ice_aq_q_shutdown(hw, true);
642 case ICE_CTL_Q_MAILBOX:
649 ice_shutdown_sq(hw, cq);
650 ice_shutdown_rq(hw, cq);
654 * ice_shutdown_all_ctrlq - shutdown routine for all control queues
655 * @hw: pointer to the hardware structure
657 * NOTE: this function does not destroy the control queue locks. The driver
658 * may call this at runtime to shutdown and later restart control queues, such
659 * as in response to a reset event.
661 void ice_shutdown_all_ctrlq(struct ice_hw *hw)
663 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
664 /* Shutdown FW admin queue */
665 ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
666 /* Shutdown PF-VF Mailbox */
667 ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX);
671 * ice_init_all_ctrlq - main initialization routine for all control queues
672 * @hw: pointer to the hardware structure
674 * Prior to calling this function, the driver MUST* set the following fields
675 * in the cq->structure for all control queues:
676 * - cq->num_sq_entries
677 * - cq->num_rq_entries
681 * NOTE: this function does not initialize the controlq locks.
683 enum ice_status ice_init_all_ctrlq(struct ice_hw *hw)
685 enum ice_status status;
688 ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__);
690 /* Init FW admin queue */
692 status = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
696 status = ice_init_check_adminq(hw);
697 if (status != ICE_ERR_AQ_FW_CRITICAL)
700 ice_debug(hw, ICE_DBG_AQ_MSG, "Retry Admin Queue init due to FW critical error\n");
701 ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
702 ice_msec_delay(ICE_CTL_Q_ADMIN_INIT_MSEC, true);
703 } while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT);
707 /* Init Mailbox queue */
708 return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);
712 * ice_init_ctrlq_locks - Initialize locks for a control queue
713 * @cq: pointer to the control queue
715 * Initializes the send and receive queue locks for a given control queue.
717 static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq)
719 ice_init_lock(&cq->sq_lock);
720 ice_init_lock(&cq->rq_lock);
724 * ice_create_all_ctrlq - main initialization routine for all control queues
725 * @hw: pointer to the hardware structure
727 * Prior to calling this function, the driver *MUST* set the following fields
728 * in the cq->structure for all control queues:
729 * - cq->num_sq_entries
730 * - cq->num_rq_entries
734 * This function creates all the control queue locks and then calls
735 * ice_init_all_ctrlq. It should be called once during driver load. If the
736 * driver needs to re-initialize control queues at run time it should call
737 * ice_init_all_ctrlq instead.
739 enum ice_status ice_create_all_ctrlq(struct ice_hw *hw)
741 ice_init_ctrlq_locks(&hw->adminq);
742 ice_init_ctrlq_locks(&hw->mailboxq);
744 return ice_init_all_ctrlq(hw);
748 * ice_destroy_ctrlq_locks - Destroy locks for a control queue
749 * @cq: pointer to the control queue
751 * Destroys the send and receive queue locks for a given control queue.
753 static void ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq)
755 ice_destroy_lock(&cq->sq_lock);
756 ice_destroy_lock(&cq->rq_lock);
760 * ice_destroy_all_ctrlq - exit routine for all control queues
761 * @hw: pointer to the hardware structure
763 * This function shuts down all the control queues and then destroys the
764 * control queue locks. It should be called once during driver unload. The
765 * driver should call ice_shutdown_all_ctrlq if it needs to shut down and
766 * reinitialize control queues, such as in response to a reset event.
768 void ice_destroy_all_ctrlq(struct ice_hw *hw)
770 /* shut down all the control queues first */
771 ice_shutdown_all_ctrlq(hw);
773 ice_destroy_ctrlq_locks(&hw->adminq);
774 ice_destroy_ctrlq_locks(&hw->mailboxq);
778 * ice_clean_sq - cleans Admin send queue (ATQ)
779 * @hw: pointer to the hardware structure
780 * @cq: pointer to the specific Control queue
782 * returns the number of free desc
784 static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
786 struct ice_ctl_q_ring *sq = &cq->sq;
787 u16 ntc = sq->next_to_clean;
788 struct ice_sq_cd *details;
789 struct ice_aq_desc *desc;
791 desc = ICE_CTL_Q_DESC(*sq, ntc);
792 details = ICE_CTL_Q_DETAILS(*sq, ntc);
794 while (rd32(hw, cq->sq.head) != ntc) {
795 ice_debug(hw, ICE_DBG_AQ_MSG, "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head));
796 ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
797 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
799 if (ntc == sq->count)
801 desc = ICE_CTL_Q_DESC(*sq, ntc);
802 details = ICE_CTL_Q_DETAILS(*sq, ntc);
805 sq->next_to_clean = ntc;
807 return ICE_CTL_Q_DESC_UNUSED(sq);
812 * @hw: pointer to the hardware structure
813 * @desc: pointer to control queue descriptor
814 * @buf: pointer to command buffer
815 * @buf_len: max length of buf
817 * Dumps debug log about control command with descriptor contents.
819 static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len)
821 struct ice_aq_desc *cq_desc = (struct ice_aq_desc *)desc;
824 if (!((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask))
830 datalen = LE16_TO_CPU(cq_desc->datalen);
831 flags = LE16_TO_CPU(cq_desc->flags);
833 ice_debug(hw, ICE_DBG_AQ_DESC, "CQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n",
834 LE16_TO_CPU(cq_desc->opcode), flags, datalen,
835 LE16_TO_CPU(cq_desc->retval));
836 ice_debug(hw, ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n",
837 LE32_TO_CPU(cq_desc->cookie_high),
838 LE32_TO_CPU(cq_desc->cookie_low));
839 ice_debug(hw, ICE_DBG_AQ_DESC, "\tparam (0,1) 0x%08X 0x%08X\n",
840 LE32_TO_CPU(cq_desc->params.generic.param0),
841 LE32_TO_CPU(cq_desc->params.generic.param1));
842 ice_debug(hw, ICE_DBG_AQ_DESC, "\taddr (h,l) 0x%08X 0x%08X\n",
843 LE32_TO_CPU(cq_desc->params.generic.addr_high),
844 LE32_TO_CPU(cq_desc->params.generic.addr_low));
845 /* Dump buffer iff 1) one exists and 2) is either a response indicated
846 * by the DD and/or CMP flag set or a command with the RD flag set.
848 if (buf && cq_desc->datalen != 0 &&
849 (flags & (ICE_AQ_FLAG_DD | ICE_AQ_FLAG_CMP) ||
850 flags & ICE_AQ_FLAG_RD)) {
851 ice_debug(hw, ICE_DBG_AQ_DESC_BUF, "Buffer:\n");
852 ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, (u8 *)buf,
853 MIN_T(u16, buf_len, datalen));
858 * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ)
859 * @hw: pointer to the HW struct
860 * @cq: pointer to the specific Control queue
862 * Returns true if the firmware has processed all descriptors on the
863 * admin send queue. Returns false if there are still requests pending.
865 static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq)
867 /* AQ designers suggest use of head for better
868 * timing reliability than DD bit
870 return rd32(hw, cq->sq.head) == cq->sq.next_to_use;
874 * ice_sq_send_cmd_nolock - send command to Control Queue (ATQ)
875 * @hw: pointer to the HW struct
876 * @cq: pointer to the specific Control queue
877 * @desc: prefilled descriptor describing the command (non DMA mem)
878 * @buf: buffer to use for indirect commands (or NULL for direct commands)
879 * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
880 * @cd: pointer to command details structure
882 * This is the main send command routine for the ATQ. It runs the queue,
883 * cleans the queue, etc.
885 static enum ice_status
886 ice_sq_send_cmd_nolock(struct ice_hw *hw, struct ice_ctl_q_info *cq,
887 struct ice_aq_desc *desc, void *buf, u16 buf_size,
888 struct ice_sq_cd *cd)
890 struct ice_dma_mem *dma_buf = NULL;
891 struct ice_aq_desc *desc_on_ring;
892 bool cmd_completed = false;
893 enum ice_status status = ICE_SUCCESS;
894 struct ice_sq_cd *details;
899 /* if reset is in progress return a soft error */
900 if (hw->reset_ongoing)
901 return ICE_ERR_RESET_ONGOING;
903 cq->sq_last_status = ICE_AQ_RC_OK;
906 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send queue not initialized.\n");
907 status = ICE_ERR_AQ_EMPTY;
908 goto sq_send_command_error;
911 if ((buf && !buf_size) || (!buf && buf_size)) {
912 status = ICE_ERR_PARAM;
913 goto sq_send_command_error;
917 if (buf_size > cq->sq_buf_size) {
918 ice_debug(hw, ICE_DBG_AQ_MSG, "Invalid buffer size for Control Send queue: %d.\n",
920 status = ICE_ERR_INVAL_SIZE;
921 goto sq_send_command_error;
924 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_BUF);
925 if (buf_size > ICE_AQ_LG_BUF)
926 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
929 val = rd32(hw, cq->sq.head);
930 if (val >= cq->num_sq_entries) {
931 ice_debug(hw, ICE_DBG_AQ_MSG, "head overrun at %d in the Control Send Queue ring\n",
933 status = ICE_ERR_AQ_EMPTY;
934 goto sq_send_command_error;
937 details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use);
941 ice_memset(details, 0, sizeof(*details), ICE_NONDMA_MEM);
943 /* Call clean and check queue available function to reclaim the
944 * descriptors that were processed by FW/MBX; the function returns the
945 * number of desc available. The clean function called here could be
946 * called in a separate thread in case of asynchronous completions.
948 if (ice_clean_sq(hw, cq) == 0) {
949 ice_debug(hw, ICE_DBG_AQ_MSG, "Error: Control Send Queue is full.\n");
950 status = ICE_ERR_AQ_FULL;
951 goto sq_send_command_error;
954 /* initialize the temp desc pointer with the right desc */
955 desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use);
957 /* if the desc is available copy the temp desc to the right place */
958 ice_memcpy(desc_on_ring, desc, sizeof(*desc_on_ring),
961 /* if buf is not NULL assume indirect command */
963 dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use];
964 /* copy the user buf into the respective DMA buf */
965 ice_memcpy(dma_buf->va, buf, buf_size, ICE_NONDMA_TO_DMA);
966 desc_on_ring->datalen = CPU_TO_LE16(buf_size);
968 /* Update the address values in the desc with the pa value
969 * for respective buffer
971 desc_on_ring->params.generic.addr_high =
972 CPU_TO_LE32(ICE_HI_DWORD(dma_buf->pa));
973 desc_on_ring->params.generic.addr_low =
974 CPU_TO_LE32(ICE_LO_DWORD(dma_buf->pa));
977 /* Debug desc and buffer */
978 ice_debug(hw, ICE_DBG_AQ_DESC, "ATQ: Control Send queue desc and buffer:\n");
980 ice_debug_cq(hw, (void *)desc_on_ring, buf, buf_size);
982 (cq->sq.next_to_use)++;
983 if (cq->sq.next_to_use == cq->sq.count)
984 cq->sq.next_to_use = 0;
985 wr32(hw, cq->sq.tail, cq->sq.next_to_use);
988 if (ice_sq_done(hw, cq))
991 ice_usec_delay(ICE_CTL_Q_SQ_CMD_USEC, false);
993 } while (total_delay < cq->sq_cmd_timeout);
995 /* if ready, copy the desc back to temp */
996 if (ice_sq_done(hw, cq)) {
997 ice_memcpy(desc, desc_on_ring, sizeof(*desc),
1000 /* get returned length to copy */
1001 u16 copy_size = LE16_TO_CPU(desc->datalen);
1003 if (copy_size > buf_size) {
1004 ice_debug(hw, ICE_DBG_AQ_MSG, "Return len %d > than buf len %d\n",
1005 copy_size, buf_size);
1006 status = ICE_ERR_AQ_ERROR;
1008 ice_memcpy(buf, dma_buf->va, copy_size,
1012 retval = LE16_TO_CPU(desc->retval);
1014 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue command 0x%04X completed with error 0x%X\n",
1015 LE16_TO_CPU(desc->opcode),
1018 /* strip off FW internal code */
1021 cmd_completed = true;
1022 if (!status && retval != ICE_AQ_RC_OK)
1023 status = ICE_ERR_AQ_ERROR;
1024 cq->sq_last_status = (enum ice_aq_err)retval;
1027 ice_debug(hw, ICE_DBG_AQ_MSG, "ATQ: desc and buffer writeback:\n");
1029 ice_debug_cq(hw, (void *)desc, buf, buf_size);
1031 /* save writeback AQ if requested */
1032 if (details->wb_desc)
1033 ice_memcpy(details->wb_desc, desc_on_ring,
1034 sizeof(*details->wb_desc), ICE_DMA_TO_NONDMA);
1036 /* update the error if time out occurred */
1037 if (!cmd_completed) {
1038 if (rd32(hw, cq->rq.len) & cq->rq.len_crit_mask ||
1039 rd32(hw, cq->sq.len) & cq->sq.len_crit_mask) {
1040 ice_debug(hw, ICE_DBG_AQ_MSG, "Critical FW error.\n");
1041 status = ICE_ERR_AQ_FW_CRITICAL;
1043 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue Writeback timeout.\n");
1044 status = ICE_ERR_AQ_TIMEOUT;
1048 sq_send_command_error:
1053 * ice_sq_send_cmd - send command to Control Queue (ATQ)
1054 * @hw: pointer to the HW struct
1055 * @cq: pointer to the specific Control queue
1056 * @desc: prefilled descriptor describing the command
1057 * @buf: buffer to use for indirect commands (or NULL for direct commands)
1058 * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
1059 * @cd: pointer to command details structure
1061 * This is the main send command routine for the ATQ. It runs the queue,
1062 * cleans the queue, etc.
1065 ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
1066 struct ice_aq_desc *desc, void *buf, u16 buf_size,
1067 struct ice_sq_cd *cd)
1069 enum ice_status status = ICE_SUCCESS;
1071 /* if reset is in progress return a soft error */
1072 if (hw->reset_ongoing)
1073 return ICE_ERR_RESET_ONGOING;
1075 ice_acquire_lock(&cq->sq_lock);
1076 status = ice_sq_send_cmd_nolock(hw, cq, desc, buf, buf_size, cd);
1077 ice_release_lock(&cq->sq_lock);
1083 * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function
1084 * @desc: pointer to the temp descriptor (non DMA mem)
1085 * @opcode: the opcode can be used to decide which flags to turn off or on
1087 * Fill the desc with default values
1089 void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode)
1091 /* zero out the desc */
1092 ice_memset(desc, 0, sizeof(*desc), ICE_NONDMA_MEM);
1093 desc->opcode = CPU_TO_LE16(opcode);
1094 desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_SI);
1099 * @hw: pointer to the HW struct
1100 * @cq: pointer to the specific Control queue
1101 * @e: event info from the receive descriptor, includes any buffers
1102 * @pending: number of events that could be left to process
1104 * This function cleans one Admin Receive Queue element and returns
1105 * the contents through e. It can also return how many events are
1106 * left to process through 'pending'.
1109 ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq,
1110 struct ice_rq_event_info *e, u16 *pending)
1112 u16 ntc = cq->rq.next_to_clean;
1113 enum ice_aq_err rq_last_status;
1114 enum ice_status ret_code = ICE_SUCCESS;
1115 struct ice_aq_desc *desc;
1116 struct ice_dma_mem *bi;
1122 /* pre-clean the event info */
1123 ice_memset(&e->desc, 0, sizeof(e->desc), ICE_NONDMA_MEM);
1125 /* take the lock before we start messing with the ring */
1126 ice_acquire_lock(&cq->rq_lock);
1128 if (!cq->rq.count) {
1129 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive queue not initialized.\n");
1130 ret_code = ICE_ERR_AQ_EMPTY;
1131 goto clean_rq_elem_err;
1134 /* set next_to_use to head */
1135 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1138 /* nothing to do - shouldn't need to update ring's values */
1139 ret_code = ICE_ERR_AQ_NO_WORK;
1140 goto clean_rq_elem_out;
1143 /* now clean the next descriptor */
1144 desc = ICE_CTL_Q_DESC(cq->rq, ntc);
1147 rq_last_status = (enum ice_aq_err)LE16_TO_CPU(desc->retval);
1148 flags = LE16_TO_CPU(desc->flags);
1149 if (flags & ICE_AQ_FLAG_ERR) {
1150 ret_code = ICE_ERR_AQ_ERROR;
1151 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive Queue Event 0x%04X received with error 0x%X\n",
1152 LE16_TO_CPU(desc->opcode), rq_last_status);
1154 ice_memcpy(&e->desc, desc, sizeof(e->desc), ICE_DMA_TO_NONDMA);
1155 datalen = LE16_TO_CPU(desc->datalen);
1156 e->msg_len = MIN_T(u16, datalen, e->buf_len);
1157 if (e->msg_buf && e->msg_len)
1158 ice_memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va,
1159 e->msg_len, ICE_DMA_TO_NONDMA);
1161 ice_debug(hw, ICE_DBG_AQ_DESC, "ARQ: desc and buffer:\n");
1163 ice_debug_cq(hw, (void *)desc, e->msg_buf, cq->rq_buf_size);
1165 /* Restore the original datalen and buffer address in the desc,
1166 * FW updates datalen to indicate the event message size
1168 bi = &cq->rq.r.rq_bi[ntc];
1169 ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
1171 desc->flags = CPU_TO_LE16(ICE_AQ_FLAG_BUF);
1172 if (cq->rq_buf_size > ICE_AQ_LG_BUF)
1173 desc->flags |= CPU_TO_LE16(ICE_AQ_FLAG_LB);
1174 desc->datalen = CPU_TO_LE16(bi->size);
1175 desc->params.generic.addr_high = CPU_TO_LE32(ICE_HI_DWORD(bi->pa));
1176 desc->params.generic.addr_low = CPU_TO_LE32(ICE_LO_DWORD(bi->pa));
1178 /* set tail = the last cleaned desc index. */
1179 wr32(hw, cq->rq.tail, ntc);
1180 /* ntc is updated to tail + 1 */
1182 if (ntc == cq->num_rq_entries)
1184 cq->rq.next_to_clean = ntc;
1185 cq->rq.next_to_use = ntu;
1188 /* Set pending if needed, unlock and return */
1190 /* re-read HW head to calculate actual pending messages */
1191 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1192 *pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc));
1195 ice_release_lock(&cq->rq_lock);