1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Broadcom
8 #include <rte_bitmap.h>
10 #include "bcmfs_device.h"
11 #include "bcmfs_dev_msg.h"
12 #include "bcmfs_hw_defs.h"
13 #include "bcmfs_logs.h"
15 #include "bcmfs_rm_common.h"
17 /* FS4 configuration */
18 #define RING_BD_TOGGLE_INVALID(offset) \
19 (((offset) >> FS_RING_BD_ALIGN_ORDER) & 0x1)
20 #define RING_BD_TOGGLE_VALID(offset) \
21 (!RING_BD_TOGGLE_INVALID(offset))
23 #define RING_VER_MAGIC 0x76303031
25 /* Per-Ring register offsets */
26 #define RING_VER 0x000
27 #define RING_BD_START_ADDR 0x004
28 #define RING_BD_READ_PTR 0x008
29 #define RING_BD_WRITE_PTR 0x00c
30 #define RING_BD_READ_PTR_DDR_LS 0x010
31 #define RING_BD_READ_PTR_DDR_MS 0x014
32 #define RING_CMPL_START_ADDR 0x018
33 #define RING_CMPL_WRITE_PTR 0x01c
34 #define RING_NUM_REQ_RECV_LS 0x020
35 #define RING_NUM_REQ_RECV_MS 0x024
36 #define RING_NUM_REQ_TRANS_LS 0x028
37 #define RING_NUM_REQ_TRANS_MS 0x02c
38 #define RING_NUM_REQ_OUTSTAND 0x030
39 #define RING_CONTROL 0x034
40 #define RING_FLUSH_DONE 0x038
41 #define RING_MSI_ADDR_LS 0x03c
42 #define RING_MSI_ADDR_MS 0x040
43 #define RING_MSI_CONTROL 0x048
44 #define RING_BD_READ_PTR_DDR_CONTROL 0x04c
45 #define RING_MSI_DATA_VALUE 0x064
47 /* Register RING_BD_START_ADDR fields */
48 #define BD_LAST_UPDATE_HW_SHIFT 28
49 #define BD_LAST_UPDATE_HW_MASK 0x1
50 #define BD_START_ADDR_VALUE(pa) \
51 ((uint32_t)((((uint64_t)(pa)) >> FS_RING_BD_ALIGN_ORDER) & 0x0fffffff))
52 #define BD_START_ADDR_DECODE(val) \
53 ((uint64_t)((val) & 0x0fffffff) << FS_RING_BD_ALIGN_ORDER)
55 /* Register RING_CMPL_START_ADDR fields */
56 #define CMPL_START_ADDR_VALUE(pa) \
57 ((uint32_t)((((uint64_t)(pa)) >> FS_RING_CMPL_ALIGN_ORDER) & 0x7ffffff))
59 /* Register RING_CONTROL fields */
60 #define CONTROL_MASK_DISABLE_CONTROL 12
61 #define CONTROL_FLUSH_SHIFT 5
62 #define CONTROL_ACTIVE_SHIFT 4
63 #define CONTROL_RATE_ADAPT_MASK 0xf
64 #define CONTROL_RATE_DYNAMIC 0x0
65 #define CONTROL_RATE_FAST 0x8
66 #define CONTROL_RATE_MEDIUM 0x9
67 #define CONTROL_RATE_SLOW 0xa
68 #define CONTROL_RATE_IDLE 0xb
70 /* Register RING_FLUSH_DONE fields */
71 #define FLUSH_DONE_MASK 0x1
73 /* Register RING_MSI_CONTROL fields */
74 #define MSI_TIMER_VAL_SHIFT 16
75 #define MSI_TIMER_VAL_MASK 0xffff
76 #define MSI_ENABLE_SHIFT 15
77 #define MSI_ENABLE_MASK 0x1
78 #define MSI_COUNT_SHIFT 0
79 #define MSI_COUNT_MASK 0x3ff
81 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
82 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16
83 #define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff
84 #define BD_READ_PTR_DDR_ENABLE_SHIFT 15
85 #define BD_READ_PTR_DDR_ENABLE_MASK 0x1
87 /* ====== Broadcom FS4-RM ring descriptor defines ===== */
90 /* General descriptor format */
91 #define DESC_TYPE_SHIFT 60
92 #define DESC_TYPE_MASK 0xf
93 #define DESC_PAYLOAD_SHIFT 0
94 #define DESC_PAYLOAD_MASK 0x0fffffffffffffff
96 /* Null descriptor format */
98 #define NULL_TOGGLE_SHIFT 58
99 #define NULL_TOGGLE_MASK 0x1
101 /* Header descriptor format */
102 #define HEADER_TYPE 1
103 #define HEADER_TOGGLE_SHIFT 58
104 #define HEADER_TOGGLE_MASK 0x1
105 #define HEADER_ENDPKT_SHIFT 57
106 #define HEADER_ENDPKT_MASK 0x1
107 #define HEADER_STARTPKT_SHIFT 56
108 #define HEADER_STARTPKT_MASK 0x1
109 #define HEADER_BDCOUNT_SHIFT 36
110 #define HEADER_BDCOUNT_MASK 0x1f
111 #define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK
112 #define HEADER_FLAGS_SHIFT 16
113 #define HEADER_FLAGS_MASK 0xffff
114 #define HEADER_OPAQUE_SHIFT 0
115 #define HEADER_OPAQUE_MASK 0xffff
117 /* Source (SRC) descriptor format */
119 #define SRC_LENGTH_SHIFT 44
120 #define SRC_LENGTH_MASK 0xffff
121 #define SRC_ADDR_SHIFT 0
122 #define SRC_ADDR_MASK 0x00000fffffffffff
124 /* Destination (DST) descriptor format */
126 #define DST_LENGTH_SHIFT 44
127 #define DST_LENGTH_MASK 0xffff
128 #define DST_ADDR_SHIFT 0
129 #define DST_ADDR_MASK 0x00000fffffffffff
131 /* Next pointer (NPTR) descriptor format */
133 #define NPTR_TOGGLE_SHIFT 58
134 #define NPTR_TOGGLE_MASK 0x1
135 #define NPTR_ADDR_SHIFT 0
136 #define NPTR_ADDR_MASK 0x00000fffffffffff
138 /* Mega source (MSRC) descriptor format */
140 #define MSRC_LENGTH_SHIFT 44
141 #define MSRC_LENGTH_MASK 0xffff
142 #define MSRC_ADDR_SHIFT 0
143 #define MSRC_ADDR_MASK 0x00000fffffffffff
145 /* Mega destination (MDST) descriptor format */
147 #define MDST_LENGTH_SHIFT 44
148 #define MDST_LENGTH_MASK 0xffff
149 #define MDST_ADDR_SHIFT 0
150 #define MDST_ADDR_MASK 0x00000fffffffffff
153 bcmfs4_is_next_table_desc(void *desc_ptr)
155 uint64_t desc = rm_read_desc(desc_ptr);
156 uint32_t type = FS_DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
158 return (type == NPTR_TYPE) ? true : false;
162 bcmfs4_next_table_desc(uint32_t toggle, uint64_t next_addr)
164 return (rm_build_desc(NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
165 rm_build_desc(toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK) |
166 rm_build_desc(next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK));
170 bcmfs4_null_desc(uint32_t toggle)
172 return (rm_build_desc(NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
173 rm_build_desc(toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK));
177 bcmfs4_flip_header_toggle(void *desc_ptr)
179 uint64_t desc = rm_read_desc(desc_ptr);
181 if (desc & ((uint64_t)0x1 << HEADER_TOGGLE_SHIFT))
182 desc &= ~((uint64_t)0x1 << HEADER_TOGGLE_SHIFT);
184 desc |= ((uint64_t)0x1 << HEADER_TOGGLE_SHIFT);
186 rm_write_desc(desc_ptr, desc);
190 bcmfs4_header_desc(uint32_t toggle, uint32_t startpkt,
191 uint32_t endpkt, uint32_t bdcount,
192 uint32_t flags, uint32_t opaque)
194 return (rm_build_desc(HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
195 rm_build_desc(toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK) |
196 rm_build_desc(startpkt, HEADER_STARTPKT_SHIFT,
197 HEADER_STARTPKT_MASK) |
198 rm_build_desc(endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK) |
199 rm_build_desc(bdcount, HEADER_BDCOUNT_SHIFT,
200 HEADER_BDCOUNT_MASK) |
201 rm_build_desc(flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK) |
202 rm_build_desc(opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK));
206 bcmfs4_enqueue_desc(uint32_t nhpos, uint32_t nhcnt,
207 uint32_t reqid, uint64_t desc,
208 void **desc_ptr, uint32_t *toggle,
209 void *start_desc, void *end_desc)
212 uint32_t nhavail, _toggle, _startpkt, _endpkt, _bdcount;
215 * Each request or packet start with a HEADER descriptor followed
216 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
217 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
218 * following a HEADER descriptor is represented by BDCOUNT field
219 * of HEADER descriptor. The max value of BDCOUNT field is 31 which
220 * means we can only have 31 non-HEADER descriptors following one
223 * In general use, number of non-HEADER descriptors can easily go
224 * beyond 31. To tackle this situation, we have packet (or request)
225 * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.
227 * To use packet extension, the first HEADER descriptor of request
228 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
229 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
230 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
231 * TOGGLE bit of the first HEADER will be set to invalid state to
232 * ensure that FlexDMA engine does not start fetching descriptors
233 * till all descriptors are enqueued. The user of this function
234 * will flip the TOGGLE bit of first HEADER after all descriptors
238 if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
239 /* Prepare the header descriptor */
240 nhavail = (nhcnt - nhpos);
241 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
242 _startpkt = (nhpos == 0) ? 0x1 : 0x0;
243 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
244 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
245 nhavail : HEADER_BDCOUNT_MAX;
246 if (nhavail <= HEADER_BDCOUNT_MAX)
249 _bdcount = HEADER_BDCOUNT_MAX;
250 d = bcmfs4_header_desc(_toggle, _startpkt, _endpkt,
251 _bdcount, 0x0, reqid);
253 /* Write header descriptor */
254 rm_write_desc(*desc_ptr, d);
256 /* Point to next descriptor */
257 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
258 if (*desc_ptr == end_desc)
259 *desc_ptr = start_desc;
261 /* Skip next pointer descriptors */
262 while (bcmfs4_is_next_table_desc(*desc_ptr)) {
263 *toggle = (*toggle) ? 0 : 1;
264 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
265 if (*desc_ptr == end_desc)
266 *desc_ptr = start_desc;
270 /* Write desired descriptor */
271 rm_write_desc(*desc_ptr, desc);
273 /* Point to next descriptor */
274 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
275 if (*desc_ptr == end_desc)
276 *desc_ptr = start_desc;
278 /* Skip next pointer descriptors */
279 while (bcmfs4_is_next_table_desc(*desc_ptr)) {
280 *toggle = (*toggle) ? 0 : 1;
281 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
282 if (*desc_ptr == end_desc)
283 *desc_ptr = start_desc;
288 bcmfs4_src_desc(uint64_t addr, unsigned int length)
290 return (rm_build_desc(SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
291 rm_build_desc(length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK) |
292 rm_build_desc(addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK));
296 bcmfs4_msrc_desc(uint64_t addr, unsigned int length_div_16)
298 return (rm_build_desc(MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
299 rm_build_desc(length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK) |
300 rm_build_desc(addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK));
304 bcmfs4_dst_desc(uint64_t addr, unsigned int length)
306 return (rm_build_desc(DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
307 rm_build_desc(length, DST_LENGTH_SHIFT, DST_LENGTH_MASK) |
308 rm_build_desc(addr, DST_ADDR_SHIFT, DST_ADDR_MASK));
312 bcmfs4_mdst_desc(uint64_t addr, unsigned int length_div_16)
314 return (rm_build_desc(MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
315 rm_build_desc(length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK) |
316 rm_build_desc(addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK));
320 bcmfs4_sanity_check(struct bcmfs_qp_message *msg)
327 for (i = 0; i < msg->srcs_count; i++) {
328 if (msg->srcs_len[i] & 0xf) {
329 if (msg->srcs_len[i] > SRC_LENGTH_MASK)
332 if (msg->srcs_len[i] > (MSRC_LENGTH_MASK * 16))
336 for (i = 0; i < msg->dsts_count; i++) {
337 if (msg->dsts_len[i] & 0xf) {
338 if (msg->dsts_len[i] > DST_LENGTH_MASK)
341 if (msg->dsts_len[i] > (MDST_LENGTH_MASK * 16))
350 estimate_nonheader_desc_count(struct bcmfs_qp_message *msg)
353 unsigned int src = 0;
354 unsigned int dst = 0;
355 unsigned int dst_target = 0;
357 while (src < msg->srcs_count ||
358 dst < msg->dsts_count) {
359 if (src < msg->srcs_count) {
361 dst_target = msg->srcs_len[src];
364 dst_target = UINT_MAX;
366 while (dst_target && dst < msg->dsts_count) {
368 if (msg->dsts_len[dst] < dst_target)
369 dst_target -= msg->dsts_len[dst];
380 bcmfs4_enqueue_msg(struct bcmfs_qp_message *msg,
381 uint32_t nhcnt, uint32_t reqid,
382 void *desc_ptr, uint32_t toggle,
383 void *start_desc, void *end_desc)
387 unsigned int src = 0;
388 unsigned int dst = 0;
389 unsigned int dst_target = 0;
390 void *orig_desc_ptr = desc_ptr;
392 if (!desc_ptr || !start_desc || !end_desc)
395 if (desc_ptr < start_desc || end_desc <= desc_ptr)
398 while (src < msg->srcs_count || dst < msg->dsts_count) {
399 if (src < msg->srcs_count) {
400 if (msg->srcs_len[src] & 0xf) {
401 d = bcmfs4_src_desc(msg->srcs_addr[src],
404 d = bcmfs4_msrc_desc(msg->srcs_addr[src],
405 msg->srcs_len[src] / 16);
407 bcmfs4_enqueue_desc(nhpos, nhcnt, reqid,
408 d, &desc_ptr, &toggle,
409 start_desc, end_desc);
411 dst_target = msg->srcs_len[src];
414 dst_target = UINT_MAX;
417 while (dst_target && (dst < msg->dsts_count)) {
418 if (msg->dsts_len[dst] & 0xf) {
419 d = bcmfs4_dst_desc(msg->dsts_addr[dst],
422 d = bcmfs4_mdst_desc(msg->dsts_addr[dst],
423 msg->dsts_len[dst] / 16);
425 bcmfs4_enqueue_desc(nhpos, nhcnt, reqid,
426 d, &desc_ptr, &toggle,
427 start_desc, end_desc);
429 if (msg->dsts_len[dst] < dst_target)
430 dst_target -= msg->dsts_len[dst];
433 dst++; /* for next buffer */
437 /* Null descriptor with invalid toggle bit */
438 rm_write_desc(desc_ptr, bcmfs4_null_desc(!toggle));
440 /* Ensure that descriptors have been written to memory */
443 bcmfs4_flip_header_toggle(orig_desc_ptr);
449 bcmfs4_enqueue_single_request_qp(struct bcmfs_qp *qp, void *op)
457 uint8_t exit_cleanup = false;
458 struct bcmfs_queue *txq = &qp->tx_q;
459 struct bcmfs_qp_message *msg = (struct bcmfs_qp_message *)op;
461 /* Do sanity check on message */
462 if (!bcmfs4_sanity_check(msg)) {
463 BCMFS_DP_LOG(ERR, "Invalid msg on queue %d", qp->qpair_id);
467 /* Scan from the beginning */
468 __rte_bitmap_scan_init(qp->ctx_bmp);
469 /* Scan bitmap to get the free pool */
470 ret = rte_bitmap_scan(qp->ctx_bmp, &pos, &slab);
472 BCMFS_DP_LOG(ERR, "BD memory exhausted");
476 reqid = pos + __builtin_ctzll(slab);
477 rte_bitmap_clear(qp->ctx_bmp, reqid);
478 qp->ctx_pool[reqid] = (unsigned long)msg;
481 * Number required descriptors = number of non-header descriptors +
482 * number of header descriptors +
485 nhcnt = estimate_nonheader_desc_count(msg);
487 /* Write descriptors to ring */
488 next = bcmfs4_enqueue_msg(msg, nhcnt, reqid,
489 (uint8_t *)txq->base_addr + txq->tx_write_ptr,
490 RING_BD_TOGGLE_VALID(txq->tx_write_ptr),
492 (uint8_t *)txq->base_addr + txq->queue_size);
494 BCMFS_DP_LOG(ERR, "Enqueue for desc failed on queue %d",
501 /* Save ring BD write offset */
502 txq->tx_write_ptr = (uint32_t)((uint8_t *)next -
503 (uint8_t *)txq->base_addr);
505 qp->nb_pending_requests++;
510 /* Cleanup if we failed */
512 rte_bitmap_set(qp->ctx_bmp, reqid);
518 bcmfs4_ring_doorbell_qp(struct bcmfs_qp *qp __rte_unused)
520 /* no door bell method supported */
524 bcmfs4_dequeue_qp(struct bcmfs_qp *qp, void **ops, uint16_t budget)
530 unsigned long context = 0;
531 struct bcmfs_queue *hwq = &qp->cmpl_q;
532 uint32_t cmpl_read_offset, cmpl_write_offset;
535 * Check whether budget is valid, else set the budget to maximum
536 * so that all the available completions will be processed.
538 if (budget > qp->nb_pending_requests)
539 budget = qp->nb_pending_requests;
542 * Get current completion read and write offset
543 * Note: We should read completion write pointer at least once
544 * after we get a MSI interrupt because HW maintains internal
545 * MSI status which will allow next MSI interrupt only after
546 * completion write pointer is read.
548 cmpl_write_offset = FS_MMIO_READ32((uint8_t *)qp->ioreg +
549 RING_CMPL_WRITE_PTR);
550 cmpl_write_offset *= FS_RING_DESC_SIZE;
551 cmpl_read_offset = hwq->cmpl_read_ptr;
553 /* Ensure completion pointer is read before proceeding */
556 /* For each completed request notify mailbox clients */
558 while ((cmpl_read_offset != cmpl_write_offset) && (budget > 0)) {
559 /* Dequeue next completion descriptor */
560 desc = *((uint64_t *)((uint8_t *)hwq->base_addr +
563 /* Next read offset */
564 cmpl_read_offset += FS_RING_DESC_SIZE;
565 if (cmpl_read_offset == FS_RING_CMPL_SIZE)
566 cmpl_read_offset = 0;
568 /* Decode error from completion descriptor */
569 err = rm_cmpl_desc_to_error(desc);
571 BCMFS_DP_LOG(ERR, "error desc rcvd");
573 /* Determine request id from completion descriptor */
574 reqid = rm_cmpl_desc_to_reqid(desc);
576 /* Determine message pointer based on reqid */
577 context = qp->ctx_pool[reqid];
579 BCMFS_DP_LOG(ERR, "HW error detected");
581 /* Release reqid for recycling */
582 qp->ctx_pool[reqid] = 0;
583 rte_bitmap_set(qp->ctx_bmp, reqid);
585 *ops = (void *)context;
587 /* Increment number of completions processed */
593 hwq->cmpl_read_ptr = cmpl_read_offset;
595 qp->nb_pending_requests -= count;
601 bcmfs4_start_qp(struct bcmfs_qp *qp)
605 uint64_t d, next_addr, msi;
606 struct bcmfs_queue *tx_queue = &qp->tx_q;
607 struct bcmfs_queue *cmpl_queue = &qp->cmpl_q;
609 /* Disable/deactivate ring */
610 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
612 /* Configure next table pointer entries in BD memory */
613 for (off = 0; off < tx_queue->queue_size; off += FS_RING_DESC_SIZE) {
614 next_addr = off + FS_RING_DESC_SIZE;
615 if (next_addr == tx_queue->queue_size)
617 next_addr += (uint64_t)tx_queue->base_phys_addr;
618 if (FS_RING_BD_ALIGN_CHECK(next_addr))
619 d = bcmfs4_next_table_desc(RING_BD_TOGGLE_VALID(off),
622 d = bcmfs4_null_desc(RING_BD_TOGGLE_INVALID(off));
623 rm_write_desc((uint8_t *)tx_queue->base_addr + off, d);
627 * If user interrupt the test in between the run(Ctrl+C), then all
628 * subsequent test run will fail because sw cmpl_read_offset and hw
629 * cmpl_write_offset will be pointing at different completion BD. To
630 * handle this we should flush all the rings in the startup instead
631 * of shutdown function.
632 * Ring flush will reset hw cmpl_write_offset.
635 /* Set ring flush state */
637 FS_MMIO_WRITE32(BIT(CONTROL_FLUSH_SHIFT),
638 (uint8_t *)qp->ioreg + RING_CONTROL);
641 * If previous test is stopped in between the run, then
642 * sw has to read cmpl_write_offset else DME/AE will be not
643 * come out of flush state.
645 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_CMPL_WRITE_PTR);
647 if (FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
653 BCMFS_DP_LOG(ERR, "Ring flush timeout hw-queue %d",
657 /* Clear ring flush state */
659 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
661 if (!(FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
667 BCMFS_DP_LOG(ERR, "Ring clear flush timeout hw-queue %d",
671 /* Program BD start address */
672 val = BD_START_ADDR_VALUE(tx_queue->base_phys_addr);
673 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_BD_START_ADDR);
675 /* BD write pointer will be same as HW write pointer */
676 tx_queue->tx_write_ptr = FS_MMIO_READ32((uint8_t *)qp->ioreg +
678 tx_queue->tx_write_ptr *= FS_RING_DESC_SIZE;
681 for (off = 0; off < FS_RING_CMPL_SIZE; off += FS_RING_DESC_SIZE)
682 rm_write_desc((uint8_t *)cmpl_queue->base_addr + off, 0x0);
684 /* Program completion start address */
685 val = CMPL_START_ADDR_VALUE(cmpl_queue->base_phys_addr);
686 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_CMPL_START_ADDR);
688 /* Completion read pointer will be same as HW write pointer */
689 cmpl_queue->cmpl_read_ptr = FS_MMIO_READ32((uint8_t *)qp->ioreg +
690 RING_CMPL_WRITE_PTR);
691 cmpl_queue->cmpl_read_ptr *= FS_RING_DESC_SIZE;
693 /* Read ring Tx, Rx, and Outstanding counts to clear */
694 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_LS);
695 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_MS);
696 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_LS);
697 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_MS);
698 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_OUTSTAND);
700 /* Configure per-Ring MSI registers with dummy location */
701 /* We leave 1k * FS_RING_DESC_SIZE size from base phys for MSI */
702 msi = cmpl_queue->base_phys_addr + (1024 * FS_RING_DESC_SIZE);
703 FS_MMIO_WRITE32((msi & 0xFFFFFFFF),
704 (uint8_t *)qp->ioreg + RING_MSI_ADDR_LS);
705 FS_MMIO_WRITE32(((msi >> 32) & 0xFFFFFFFF),
706 (uint8_t *)qp->ioreg + RING_MSI_ADDR_MS);
707 FS_MMIO_WRITE32(qp->qpair_id,
708 (uint8_t *)qp->ioreg + RING_MSI_DATA_VALUE);
710 /* Configure RING_MSI_CONTROL */
712 val |= (MSI_TIMER_VAL_MASK << MSI_TIMER_VAL_SHIFT);
713 val |= BIT(MSI_ENABLE_SHIFT);
714 val |= (0x1 & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
715 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_MSI_CONTROL);
717 /* Enable/activate ring */
718 val = BIT(CONTROL_ACTIVE_SHIFT);
719 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_CONTROL);
725 bcmfs4_shutdown_qp(struct bcmfs_qp *qp)
727 /* Disable/deactivate ring */
728 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
731 struct bcmfs_hw_queue_pair_ops bcmfs4_qp_ops = {
733 .enq_one_req = bcmfs4_enqueue_single_request_qp,
734 .ring_db = bcmfs4_ring_doorbell_qp,
735 .dequeue = bcmfs4_dequeue_qp,
736 .startq = bcmfs4_start_qp,
737 .stopq = bcmfs4_shutdown_qp,
740 RTE_INIT(bcmfs4_register_qp_ops)
742 bcmfs_hw_queue_pair_register_ops(&bcmfs4_qp_ops);