1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
5 #include "dlb_hw_types.h"
6 #include "../../dlb_user.h"
7 #include "dlb_resource.h"
9 #include "dlb_osdep_bitmap.h"
10 #include "dlb_osdep_types.h"
12 #include "../../dlb_priv.h"
13 #include "../../dlb_inline_fns.h"
15 #define DLB_DOM_LIST_HEAD(head, type) \
16 DLB_LIST_HEAD((head), type, domain_list)
18 #define DLB_FUNC_LIST_HEAD(head, type) \
19 DLB_LIST_HEAD((head), type, func_list)
21 #define DLB_DOM_LIST_FOR(head, ptr, iter) \
22 DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
24 #define DLB_FUNC_LIST_FOR(head, ptr, iter) \
25 DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
27 #define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
28 DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
30 #define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
31 DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
33 static inline void dlb_flush_csr(struct dlb_hw *hw)
35 DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
38 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
40 dlb_list_init_head(&rsrc->avail_domains);
41 dlb_list_init_head(&rsrc->used_domains);
42 dlb_list_init_head(&rsrc->avail_ldb_queues);
43 dlb_list_init_head(&rsrc->avail_ldb_ports);
44 dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
45 dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
46 dlb_list_init_head(&rsrc->avail_dir_credit_pools);
49 static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
51 dlb_list_init_head(&domain->used_ldb_queues);
52 dlb_list_init_head(&domain->used_ldb_ports);
53 dlb_list_init_head(&domain->used_dir_pq_pairs);
54 dlb_list_init_head(&domain->used_ldb_credit_pools);
55 dlb_list_init_head(&domain->used_dir_credit_pools);
56 dlb_list_init_head(&domain->avail_ldb_queues);
57 dlb_list_init_head(&domain->avail_ldb_ports);
58 dlb_list_init_head(&domain->avail_dir_pq_pairs);
59 dlb_list_init_head(&domain->avail_ldb_credit_pools);
60 dlb_list_init_head(&domain->avail_dir_credit_pools);
63 int dlb_resource_init(struct dlb_hw *hw)
65 struct dlb_list_entry *list;
68 /* For optimal load-balancing, ports that map to one or more QIDs in
69 * common should not be in numerical sequence. This is application
70 * dependent, but the driver interleaves port IDs as much as possible
71 * to reduce the likelihood of this. This initial allocation maximizes
72 * the average distance between an ID and its immediate neighbors (i.e.
73 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
76 u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
77 0, 31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
78 48, 15, 46, 13, 44, 11, 42, 9, 40, 7, 38, 5, 36, 3, 34, 1,
79 32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
80 16, 47, 14, 45, 12, 43, 10, 41, 8, 39, 6, 37, 4, 35, 2, 33
83 /* Zero-out resource tracking data structures */
84 memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
85 memset(&hw->pf, 0, sizeof(hw->pf));
87 dlb_init_fn_rsrc_lists(&hw->pf);
89 for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
90 memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
91 dlb_init_domain_rsrc_lists(&hw->domains[i]);
92 hw->domains[i].parent_func = &hw->pf;
95 /* Give all resources to the PF driver */
96 hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
97 for (i = 0; i < hw->pf.num_avail_domains; i++) {
98 list = &hw->domains[i].func_list;
100 dlb_list_add(&hw->pf.avail_domains, list);
103 hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
104 for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
105 list = &hw->rsrcs.ldb_queues[i].func_list;
107 dlb_list_add(&hw->pf.avail_ldb_queues, list);
110 hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
111 for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
112 struct dlb_ldb_port *port;
114 port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
116 dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
119 hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
120 for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
121 list = &hw->rsrcs.dir_pq_pairs[i].func_list;
123 dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
126 hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
127 for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
128 list = &hw->rsrcs.ldb_credit_pools[i].func_list;
130 dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
133 hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
134 for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
135 list = &hw->rsrcs.dir_credit_pools[i].func_list;
137 dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
140 /* There are 5120 history list entries, which allows us to overprovision
141 * the inflight limit (4096) by 1k.
143 if (dlb_bitmap_alloc(hw,
144 &hw->pf.avail_hist_list_entries,
145 DLB_MAX_NUM_HIST_LIST_ENTRIES))
148 if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
151 if (dlb_bitmap_alloc(hw,
152 &hw->pf.avail_qed_freelist_entries,
153 DLB_MAX_NUM_LDB_CREDITS))
156 if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
159 if (dlb_bitmap_alloc(hw,
160 &hw->pf.avail_dqed_freelist_entries,
161 DLB_MAX_NUM_DIR_CREDITS))
164 if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
167 if (dlb_bitmap_alloc(hw,
168 &hw->pf.avail_aqed_freelist_entries,
169 DLB_MAX_NUM_AQOS_ENTRIES))
172 if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
175 /* Initialize the hardware resource IDs */
176 for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
177 hw->domains[i].id = i;
179 for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
180 hw->rsrcs.ldb_queues[i].id = i;
182 for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
183 hw->rsrcs.ldb_ports[i].id = i;
185 for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
186 hw->rsrcs.dir_pq_pairs[i].id = i;
188 for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
189 hw->rsrcs.ldb_credit_pools[i].id = i;
191 for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
192 hw->rsrcs.dir_credit_pools[i].id = i;
194 for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
195 hw->rsrcs.sn_groups[i].id = i;
196 /* Default mode (0) is 32 sequence numbers per queue */
197 hw->rsrcs.sn_groups[i].mode = 0;
198 hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
199 hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
205 void dlb_resource_free(struct dlb_hw *hw)
207 dlb_bitmap_free(hw->pf.avail_hist_list_entries);
209 dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
211 dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
213 dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
216 static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
218 if (id >= DLB_MAX_NUM_DOMAINS)
221 return &hw->domains[id];
224 static int dlb_attach_ldb_queues(struct dlb_hw *hw,
225 struct dlb_function_resources *rsrcs,
226 struct dlb_domain *domain,
228 struct dlb_cmd_response *resp)
232 if (rsrcs->num_avail_ldb_queues < num_queues) {
233 resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
237 for (i = 0; i < num_queues; i++) {
238 struct dlb_ldb_queue *queue;
240 queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
244 "[%s()] Internal error: domain validation failed\n",
249 dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
251 queue->domain_id = domain->id;
254 dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
257 rsrcs->num_avail_ldb_queues -= num_queues;
263 /* Return the assigned queues */
264 for (j = 0; j < i; j++) {
265 struct dlb_ldb_queue *queue;
267 queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
269 /* Unrecoverable internal error */
273 queue->owned = false;
275 dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
277 dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
283 static struct dlb_ldb_port *
284 dlb_get_next_ldb_port(struct dlb_hw *hw,
285 struct dlb_function_resources *rsrcs,
288 struct dlb_list_entry *iter;
290 struct dlb_ldb_port *port;
292 /* To reduce the odds of consecutive load-balanced ports mapping to the
293 * same queue(s), the driver attempts to allocate ports whose neighbors
294 * are owned by a different domain.
296 DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
304 if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
307 prev = DLB_MAX_NUM_LDB_PORTS - 1;
309 if (!hw->rsrcs.ldb_ports[next].owned ||
310 hw->rsrcs.ldb_ports[next].domain_id == domain_id)
313 if (!hw->rsrcs.ldb_ports[prev].owned ||
314 hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
320 /* Failing that, the driver looks for a port with one neighbor owned by
321 * a different domain and the other unallocated.
323 DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
331 if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
334 prev = DLB_MAX_NUM_LDB_PORTS - 1;
336 if (!hw->rsrcs.ldb_ports[prev].owned &&
337 hw->rsrcs.ldb_ports[next].owned &&
338 hw->rsrcs.ldb_ports[next].domain_id != domain_id)
341 if (!hw->rsrcs.ldb_ports[next].owned &&
342 hw->rsrcs.ldb_ports[prev].owned &&
343 hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
347 /* Failing that, the driver looks for a port with both neighbors
350 DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
358 if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
361 prev = DLB_MAX_NUM_LDB_PORTS - 1;
363 if (!hw->rsrcs.ldb_ports[prev].owned &&
364 !hw->rsrcs.ldb_ports[next].owned)
368 /* If all else fails, the driver returns the next available port. */
369 return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
372 static int dlb_attach_ldb_ports(struct dlb_hw *hw,
373 struct dlb_function_resources *rsrcs,
374 struct dlb_domain *domain,
376 struct dlb_cmd_response *resp)
380 if (rsrcs->num_avail_ldb_ports < num_ports) {
381 resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
385 for (i = 0; i < num_ports; i++) {
386 struct dlb_ldb_port *port;
388 port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
392 "[%s()] Internal error: domain validation failed\n",
397 dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
399 port->domain_id = domain->id;
402 dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
405 rsrcs->num_avail_ldb_ports -= num_ports;
411 /* Return the assigned ports */
412 for (j = 0; j < i; j++) {
413 struct dlb_ldb_port *port;
415 port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
417 /* Unrecoverable internal error */
423 dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
425 dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
431 static int dlb_attach_dir_ports(struct dlb_hw *hw,
432 struct dlb_function_resources *rsrcs,
433 struct dlb_domain *domain,
435 struct dlb_cmd_response *resp)
439 if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
440 resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
444 for (i = 0; i < num_ports; i++) {
445 struct dlb_dir_pq_pair *port;
447 port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
451 "[%s()] Internal error: domain validation failed\n",
456 dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
458 port->domain_id = domain->id;
461 dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
464 rsrcs->num_avail_dir_pq_pairs -= num_ports;
470 /* Return the assigned ports */
471 for (j = 0; j < i; j++) {
472 struct dlb_dir_pq_pair *port;
474 port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
476 /* Unrecoverable internal error */
482 dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
484 dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
490 static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
491 struct dlb_domain *domain,
493 struct dlb_cmd_response *resp)
495 struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
497 if (dlb_bitmap_count(bitmap) < (int)num_credits) {
498 resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
505 base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
509 domain->qed_freelist.base = base;
510 domain->qed_freelist.bound = base + num_credits;
511 domain->qed_freelist.offset = 0;
513 dlb_bitmap_clear_range(bitmap, base, num_credits);
519 resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
523 static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
524 struct dlb_domain *domain,
526 struct dlb_cmd_response *resp)
528 struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
530 if (dlb_bitmap_count(bitmap) < (int)num_credits) {
531 resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
538 base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
542 domain->dqed_freelist.base = base;
543 domain->dqed_freelist.bound = base + num_credits;
544 domain->dqed_freelist.offset = 0;
546 dlb_bitmap_clear_range(bitmap, base, num_credits);
552 resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
556 static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
557 struct dlb_function_resources *rsrcs,
558 struct dlb_domain *domain,
559 u32 num_credit_pools,
560 struct dlb_cmd_response *resp)
564 if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
565 resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
569 for (i = 0; i < num_credit_pools; i++) {
570 struct dlb_credit_pool *pool;
572 pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
576 "[%s()] Internal error: domain validation failed\n",
581 dlb_list_del(&rsrcs->avail_ldb_credit_pools,
584 pool->domain_id = domain->id;
587 dlb_list_add(&domain->avail_ldb_credit_pools,
591 rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
597 /* Return the assigned credit pools */
598 for (j = 0; j < i; j++) {
599 struct dlb_credit_pool *pool;
601 pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
603 /* Unrecoverable internal error */
609 dlb_list_del(&domain->avail_ldb_credit_pools,
612 dlb_list_add(&rsrcs->avail_ldb_credit_pools,
619 static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
620 struct dlb_function_resources *rsrcs,
621 struct dlb_domain *domain,
622 u32 num_credit_pools,
623 struct dlb_cmd_response *resp)
627 if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
628 resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
632 for (i = 0; i < num_credit_pools; i++) {
633 struct dlb_credit_pool *pool;
635 pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
639 "[%s()] Internal error: domain validation failed\n",
644 dlb_list_del(&rsrcs->avail_dir_credit_pools,
647 pool->domain_id = domain->id;
650 dlb_list_add(&domain->avail_dir_credit_pools,
654 rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
660 /* Return the assigned credit pools */
661 for (j = 0; j < i; j++) {
662 struct dlb_credit_pool *pool;
664 pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
666 /* Unrecoverable internal error */
672 dlb_list_del(&domain->avail_dir_credit_pools,
675 dlb_list_add(&rsrcs->avail_dir_credit_pools,
683 dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
684 struct dlb_domain *domain,
685 u32 num_hist_list_entries,
686 struct dlb_cmd_response *resp)
688 struct dlb_bitmap *bitmap;
691 if (num_hist_list_entries) {
692 bitmap = rsrcs->avail_hist_list_entries;
694 base = dlb_bitmap_find_set_bit_range(bitmap,
695 num_hist_list_entries);
699 domain->total_hist_list_entries = num_hist_list_entries;
700 domain->avail_hist_list_entries = num_hist_list_entries;
701 domain->hist_list_entry_base = base;
702 domain->hist_list_entry_offset = 0;
704 dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
709 resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
713 static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
714 struct dlb_domain *domain,
715 u32 num_atomic_inflights,
716 struct dlb_cmd_response *resp)
718 if (num_atomic_inflights) {
719 struct dlb_bitmap *bitmap =
720 rsrcs->avail_aqed_freelist_entries;
723 base = dlb_bitmap_find_set_bit_range(bitmap,
724 num_atomic_inflights);
728 domain->aqed_freelist.base = base;
729 domain->aqed_freelist.bound = base + num_atomic_inflights;
730 domain->aqed_freelist.offset = 0;
732 dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
738 resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
744 dlb_domain_attach_resources(struct dlb_hw *hw,
745 struct dlb_function_resources *rsrcs,
746 struct dlb_domain *domain,
747 struct dlb_create_sched_domain_args *args,
748 struct dlb_cmd_response *resp)
752 ret = dlb_attach_ldb_queues(hw,
755 args->num_ldb_queues,
760 ret = dlb_attach_ldb_ports(hw,
768 ret = dlb_attach_dir_ports(hw,
776 ret = dlb_attach_ldb_credits(rsrcs,
778 args->num_ldb_credits,
783 ret = dlb_attach_dir_credits(rsrcs,
785 args->num_dir_credits,
790 ret = dlb_attach_ldb_credit_pools(hw,
793 args->num_ldb_credit_pools,
798 ret = dlb_attach_dir_credit_pools(hw,
801 args->num_dir_credit_pools,
806 ret = dlb_attach_domain_hist_list_entries(rsrcs,
808 args->num_hist_list_entries,
813 ret = dlb_attach_atomic_inflights(rsrcs,
815 args->num_atomic_inflights,
820 domain->configured = true;
822 domain->started = false;
824 rsrcs->num_avail_domains--;
829 static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
830 struct dlb_ldb_port *port)
832 union dlb_lsp_cq_ldb_dsbl reg;
834 /* Don't re-enable the port if a removal is pending. The caller should
835 * mark this port as enabled (if it isn't already), and when the
836 * removal completes the port will be enabled.
838 if (port->num_pending_removals)
841 reg.field.disabled = 0;
843 DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
848 static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
849 struct dlb_dir_pq_pair *port)
851 union dlb_lsp_cq_dir_dsbl reg;
853 reg.field.disabled = 0;
855 DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
861 static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
862 struct dlb_ldb_port *port)
864 union dlb_lsp_cq_ldb_dsbl reg;
866 reg.field.disabled = 1;
868 DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
873 static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
874 struct dlb_dir_pq_pair *port)
876 union dlb_lsp_cq_dir_dsbl reg;
878 reg.field.disabled = 1;
880 DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
887 void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
889 union dlb_dp_dir_csr_ctrl r0;
891 r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
893 r0.field.cfg_vasr_dis = 1;
895 DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
898 void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
900 union dlb_chp_cfg_chp_csr_ctrl r0;
902 r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
904 r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
906 DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
909 void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
911 union dlb_sys_cq_mode r0;
913 r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
915 r0.field.ldb_cq64 = 1;
917 DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
920 void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
922 union dlb_sys_cq_mode r0;
924 r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
926 r0.field.dir_cq64 = 1;
928 DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
931 void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
933 union dlb_sys_sys_alarm_int_enable r0;
935 r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
937 r0.field.pf_to_vf_isr_pend_error = 0;
939 DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
943 dlb_get_num_ports_in_use(struct dlb_hw *hw)
945 unsigned int i, n = 0;
947 for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
948 if (hw->rsrcs.ldb_ports[i].owned)
951 for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
952 if (hw->rsrcs.dir_pq_pairs[i].owned)
958 static bool dlb_port_find_slot(struct dlb_ldb_port *port,
959 enum dlb_qid_map_state state,
964 for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
965 if (port->qid_map[i].state == state)
971 return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
974 static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
975 enum dlb_qid_map_state state,
976 struct dlb_ldb_queue *queue,
981 for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
982 if (port->qid_map[i].state == state &&
983 port->qid_map[i].qid == queue->id)
989 return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
992 static int dlb_port_slot_state_transition(struct dlb_hw *hw,
993 struct dlb_ldb_port *port,
994 struct dlb_ldb_queue *queue,
996 enum dlb_qid_map_state new_state)
998 enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
999 struct dlb_domain *domain;
1001 domain = dlb_get_domain_from_id(hw, port->domain_id);
1002 if (domain == NULL) {
1004 "[%s()] Internal error: unable to find domain %d\n",
1005 __func__, port->domain_id);
1009 switch (curr_state) {
1010 case DLB_QUEUE_UNMAPPED:
1011 switch (new_state) {
1012 case DLB_QUEUE_MAPPED:
1013 queue->num_mappings++;
1014 port->num_mappings++;
1016 case DLB_QUEUE_MAP_IN_PROGRESS:
1017 queue->num_pending_additions++;
1018 domain->num_pending_additions++;
1024 case DLB_QUEUE_MAPPED:
1025 switch (new_state) {
1026 case DLB_QUEUE_UNMAPPED:
1027 queue->num_mappings--;
1028 port->num_mappings--;
1030 case DLB_QUEUE_UNMAP_IN_PROGRESS:
1031 port->num_pending_removals++;
1032 domain->num_pending_removals++;
1034 case DLB_QUEUE_MAPPED:
1035 /* Priority change, nothing to update */
1041 case DLB_QUEUE_MAP_IN_PROGRESS:
1042 switch (new_state) {
1043 case DLB_QUEUE_UNMAPPED:
1044 queue->num_pending_additions--;
1045 domain->num_pending_additions--;
1047 case DLB_QUEUE_MAPPED:
1048 queue->num_mappings++;
1049 port->num_mappings++;
1050 queue->num_pending_additions--;
1051 domain->num_pending_additions--;
1057 case DLB_QUEUE_UNMAP_IN_PROGRESS:
1058 switch (new_state) {
1059 case DLB_QUEUE_UNMAPPED:
1060 port->num_pending_removals--;
1061 domain->num_pending_removals--;
1062 queue->num_mappings--;
1063 port->num_mappings--;
1065 case DLB_QUEUE_MAPPED:
1066 port->num_pending_removals--;
1067 domain->num_pending_removals--;
1069 case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
1070 /* Nothing to update */
1076 case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
1077 switch (new_state) {
1078 case DLB_QUEUE_UNMAP_IN_PROGRESS:
1079 /* Nothing to update */
1081 case DLB_QUEUE_UNMAPPED:
1082 /* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
1083 * becomes UNMAPPED before it transitions to
1086 queue->num_mappings--;
1087 port->num_mappings--;
1088 port->num_pending_removals--;
1089 domain->num_pending_removals--;
1099 port->qid_map[slot].state = new_state;
1102 "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
1103 __func__, queue->id, port->id, curr_state,
1109 "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
1110 __func__, queue->id, port->id, curr_state,
1115 /* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
1116 * function names imply, and should only be called by the dynamic CQ mapping
1119 static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
1120 struct dlb_domain *domain,
1121 struct dlb_ldb_queue *queue)
1123 struct dlb_list_entry *iter;
1125 struct dlb_ldb_port *port;
1128 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
1129 enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
1131 if (!dlb_port_find_slot_queue(port, state, queue, &slot))
1135 dlb_ldb_port_cq_disable(hw, port);
1139 static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
1140 struct dlb_domain *domain,
1141 struct dlb_ldb_queue *queue)
1143 struct dlb_list_entry *iter;
1145 struct dlb_ldb_port *port;
1148 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
1149 enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
1151 if (!dlb_port_find_slot_queue(port, state, queue, &slot))
1155 dlb_ldb_port_cq_enable(hw, port);
1159 static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
1160 struct dlb_ldb_port *p,
1161 struct dlb_ldb_queue *q,
1164 union dlb_lsp_cq2priov r0;
1165 union dlb_lsp_cq2qid r1;
1166 union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
1167 union dlb_lsp_qid_ldb_qid2cqidx r3;
1168 union dlb_lsp_qid_ldb_qid2cqidx2 r4;
1169 enum dlb_qid_map_state state;
1172 /* Look for a pending or already mapped slot, else an unused slot */
1173 if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
1174 !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
1175 !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
1177 "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
1178 __func__, __LINE__);
1182 if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
1184 "[%s():%d] Internal error: port slot tracking failed\n",
1185 __func__, __LINE__);
1189 /* Read-modify-write the priority and valid bit register */
1190 r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
1192 r0.field.v |= 1 << i;
1193 r0.field.prio |= (priority & 0x7) << i * 3;
1195 DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
1197 /* Read-modify-write the QID map register */
1198 r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
1200 if (i == 0 || i == 4)
1201 r1.field.qid_p0 = q->id;
1202 if (i == 1 || i == 5)
1203 r1.field.qid_p1 = q->id;
1204 if (i == 2 || i == 6)
1205 r1.field.qid_p2 = q->id;
1206 if (i == 3 || i == 7)
1207 r1.field.qid_p3 = q->id;
1209 DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
1211 r2.val = DLB_CSR_RD(hw,
1212 DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
1215 r3.val = DLB_CSR_RD(hw,
1216 DLB_LSP_QID_LDB_QID2CQIDX(q->id,
1219 r4.val = DLB_CSR_RD(hw,
1220 DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
1223 switch (p->id % 4) {
1225 r2.field.cq_p0 |= 1 << i;
1226 r3.field.cq_p0 |= 1 << i;
1227 r4.field.cq_p0 |= 1 << i;
1231 r2.field.cq_p1 |= 1 << i;
1232 r3.field.cq_p1 |= 1 << i;
1233 r4.field.cq_p1 |= 1 << i;
1237 r2.field.cq_p2 |= 1 << i;
1238 r3.field.cq_p2 |= 1 << i;
1239 r4.field.cq_p2 |= 1 << i;
1243 r2.field.cq_p3 |= 1 << i;
1244 r3.field.cq_p3 |= 1 << i;
1245 r4.field.cq_p3 |= 1 << i;
1250 DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
1255 DLB_LSP_QID_LDB_QID2CQIDX(q->id,
1260 DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
1266 p->qid_map[i].qid = q->id;
1267 p->qid_map[i].priority = priority;
1269 state = DLB_QUEUE_MAPPED;
1271 return dlb_port_slot_state_transition(hw, p, q, i, state);
1274 static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
1275 struct dlb_ldb_port *port,
1276 struct dlb_ldb_queue *queue,
1279 union dlb_lsp_qid_aqed_active_cnt r0;
1280 union dlb_lsp_qid_ldb_enqueue_cnt r1;
1281 union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
1283 /* Set the atomic scheduling haswork bit */
1284 r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
1286 r2.field.cq = port->id;
1287 r2.field.qidix = slot;
1289 r2.field.rlist_haswork_v = r0.field.count > 0;
1291 /* Set the non-atomic scheduling haswork bit */
1292 DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
1294 r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
1296 memset(&r2, 0, sizeof(r2));
1298 r2.field.cq = port->id;
1299 r2.field.qidix = slot;
1301 r2.field.nalb_haswork_v = (r1.field.count > 0);
1303 DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
1310 static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
1311 struct dlb_ldb_port *port,
1314 union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
1316 r0.field.cq = port->id;
1317 r0.field.qidix = slot;
1319 r0.field.inflight_ok_v = 1;
1321 DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
1326 static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
1327 struct dlb_ldb_port *port,
1330 union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
1332 r0.field.cq = port->id;
1333 r0.field.qidix = slot;
1335 r0.field.inflight_ok_v = 1;
1337 DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
1342 static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
1343 struct dlb_ldb_queue *queue)
1345 union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
1347 r0.field.limit = queue->num_qid_inflights;
1349 DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
1352 static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
1353 struct dlb_ldb_queue *queue)
1356 DLB_LSP_QID_LDB_INFL_LIM(queue->id),
1357 DLB_LSP_QID_LDB_INFL_LIM_RST);
1360 static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
1361 struct dlb_domain *domain,
1362 struct dlb_ldb_port *port,
1363 struct dlb_ldb_queue *queue)
1365 struct dlb_list_entry *iter;
1367 union dlb_lsp_qid_ldb_infl_cnt r0;
1368 enum dlb_qid_map_state state;
1372 r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
1374 if (r0.field.count) {
1376 "[%s()] Internal error: non-zero QID inflight count\n",
1381 /* For each port with a pending mapping to this queue, perform the
1382 * static mapping and set the corresponding has_work bits.
1384 state = DLB_QUEUE_MAP_IN_PROGRESS;
1385 if (!dlb_port_find_slot_queue(port, state, queue, &slot))
1388 if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
1390 "[%s():%d] Internal error: port slot tracking failed\n",
1391 __func__, __LINE__);
1395 prio = port->qid_map[slot].priority;
1397 /* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
1398 * the port's qid_map state.
1400 ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
1404 ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
1408 /* Ensure IF_status(cq,qid) is 0 before enabling the port to
1409 * prevent spurious schedules to cause the queue's inflight
1410 * count to increase.
1412 dlb_ldb_port_clear_queue_if_status(hw, port, slot);
1414 /* Reset the queue's inflight status */
1415 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
1416 state = DLB_QUEUE_MAPPED;
1417 if (!dlb_port_find_slot_queue(port, state, queue, &slot))
1420 dlb_ldb_port_set_queue_if_status(hw, port, slot);
1423 dlb_ldb_queue_set_inflight_limit(hw, queue);
1425 /* Re-enable CQs mapped to this queue */
1426 dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
1428 /* If this queue has other mappings pending, clear its inflight limit */
1429 if (queue->num_pending_additions > 0)
1430 dlb_ldb_queue_clear_inflight_limit(hw, queue);
1436 * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
1437 * @hw: dlb_hw handle for a particular device.
1438 * @port: load-balanced port
1439 * @queue: load-balanced queue
1440 * @priority: queue servicing priority
1442 * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
1443 * at a later point, and <0 if an error occurred.
1445 static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
1446 struct dlb_ldb_port *port,
1447 struct dlb_ldb_queue *queue,
1450 union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
1451 enum dlb_qid_map_state state;
1452 struct dlb_domain *domain;
1455 domain = dlb_get_domain_from_id(hw, port->domain_id);
1456 if (domain == NULL) {
1458 "[%s()] Internal error: unable to find domain %d\n",
1459 __func__, port->domain_id);
1463 /* Set the QID inflight limit to 0 to prevent further scheduling of the
1466 DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
1468 if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
1470 "Internal error: No available unmapped slots\n");
1474 if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
1476 "[%s():%d] Internal error: port slot tracking failed\n",
1477 __func__, __LINE__);
1481 port->qid_map[slot].qid = queue->id;
1482 port->qid_map[slot].priority = priority;
1484 state = DLB_QUEUE_MAP_IN_PROGRESS;
1485 ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
1489 r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
1491 if (r0.field.count) {
1492 /* The queue is owed completions so it's not safe to map it
1493 * yet. Schedule a kernel thread to complete the mapping later,
1494 * once software has completed all the queue's inflight events.
1496 if (!os_worker_active(hw))
1497 os_schedule_work(hw);
1502 /* Disable the affected CQ, and the CQs already mapped to the QID,
1503 * before reading the QID's inflight count a second time. There is an
1504 * unlikely race in which the QID may schedule one more QE after we
1505 * read an inflight count of 0, and disabling the CQs guarantees that
1506 * the race will not occur after a re-read of the inflight count
1510 dlb_ldb_port_cq_disable(hw, port);
1512 dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
1514 r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
1516 if (r0.field.count) {
1518 dlb_ldb_port_cq_enable(hw, port);
1520 dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
1522 /* The queue is owed completions so it's not safe to map it
1523 * yet. Schedule a kernel thread to complete the mapping later,
1524 * once software has completed all the queue's inflight events.
1526 if (!os_worker_active(hw))
1527 os_schedule_work(hw);
1532 return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
1536 static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
1537 struct dlb_domain *domain,
1538 struct dlb_ldb_port *port,
1539 struct dlb_ldb_queue *queue,
1542 if (domain->started)
1543 return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
1545 return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
1548 static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
1549 struct dlb_ldb_port *port,
1550 struct dlb_ldb_queue *queue)
1552 enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
1553 union dlb_lsp_cq2priov r0;
1554 union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
1555 union dlb_lsp_qid_ldb_qid2cqidx r2;
1556 union dlb_lsp_qid_ldb_qid2cqidx2 r3;
1561 /* Find the queue's slot */
1562 mapped = DLB_QUEUE_MAPPED;
1563 in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
1564 pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
1566 if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
1567 !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
1568 !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
1570 "[%s():%d] Internal error: QID %d isn't mapped\n",
1571 __func__, __LINE__, queue->id);
1575 if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
1577 "[%s():%d] Internal error: port slot tracking failed\n",
1578 __func__, __LINE__);
1583 queue_id = queue->id;
1585 /* Read-modify-write the priority and valid bit register */
1586 r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
1588 r0.field.v &= ~(1 << i);
1590 DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
1592 r1.val = DLB_CSR_RD(hw,
1593 DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
1596 r2.val = DLB_CSR_RD(hw,
1597 DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
1600 r3.val = DLB_CSR_RD(hw,
1601 DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
1604 switch (port_id % 4) {
1606 r1.field.cq_p0 &= ~(1 << i);
1607 r2.field.cq_p0 &= ~(1 << i);
1608 r3.field.cq_p0 &= ~(1 << i);
1612 r1.field.cq_p1 &= ~(1 << i);
1613 r2.field.cq_p1 &= ~(1 << i);
1614 r3.field.cq_p1 &= ~(1 << i);
1618 r1.field.cq_p2 &= ~(1 << i);
1619 r2.field.cq_p2 &= ~(1 << i);
1620 r3.field.cq_p2 &= ~(1 << i);
1624 r1.field.cq_p3 &= ~(1 << i);
1625 r2.field.cq_p3 &= ~(1 << i);
1626 r3.field.cq_p3 &= ~(1 << i);
1631 DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
1635 DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
1639 DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
1644 unmapped = DLB_QUEUE_UNMAPPED;
1646 return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
1650 dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
1651 struct dlb_function_resources *rsrcs,
1652 struct dlb_create_sched_domain_args *args,
1653 struct dlb_cmd_response *resp)
1655 struct dlb_list_entry *iter;
1657 struct dlb_bitmap *ldb_credit_freelist;
1658 struct dlb_bitmap *dir_credit_freelist;
1659 unsigned int ldb_credit_freelist_count;
1660 unsigned int dir_credit_freelist_count;
1661 unsigned int max_contig_aqed_entries;
1662 unsigned int max_contig_dqed_entries;
1663 unsigned int max_contig_qed_entries;
1664 unsigned int max_contig_hl_entries;
1665 struct dlb_bitmap *aqed_freelist;
1666 enum dlb_dev_revision revision;
1668 ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
1669 dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
1670 aqed_freelist = rsrcs->avail_aqed_freelist_entries;
1672 ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
1673 dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
1675 max_contig_hl_entries =
1676 dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
1677 max_contig_aqed_entries =
1678 dlb_bitmap_longest_set_range(aqed_freelist);
1679 max_contig_qed_entries =
1680 dlb_bitmap_longest_set_range(ldb_credit_freelist);
1681 max_contig_dqed_entries =
1682 dlb_bitmap_longest_set_range(dir_credit_freelist);
1684 if (rsrcs->num_avail_domains < 1)
1685 resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
1686 else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
1687 resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
1688 else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
1689 resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
1690 else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
1691 resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
1692 else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
1693 resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
1694 else if (ldb_credit_freelist_count < args->num_ldb_credits)
1695 resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
1696 else if (dir_credit_freelist_count < args->num_dir_credits)
1697 resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
1698 else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
1699 resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
1700 else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
1701 resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
1702 else if (max_contig_hl_entries < args->num_hist_list_entries)
1703 resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
1704 else if (max_contig_aqed_entries < args->num_atomic_inflights)
1705 resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
1706 else if (max_contig_qed_entries < args->num_ldb_credits)
1707 resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
1708 else if (max_contig_dqed_entries < args->num_dir_credits)
1709 resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
1711 /* DLB A-stepping workaround for hardware write buffer lock up issue:
1712 * limit the maximum configured ports to less than 128 and disable CQ
1713 * occupancy interrupts.
1715 revision = os_get_dev_revision(hw);
1717 if (revision < DLB_B0) {
1718 u32 n = dlb_get_num_ports_in_use(hw);
1720 n += args->num_ldb_ports + args->num_dir_ports;
1722 if (n >= DLB_A_STEP_MAX_PORTS)
1723 resp->status = args->num_ldb_ports ?
1724 DLB_ST_LDB_PORTS_UNAVAILABLE :
1725 DLB_ST_DIR_PORTS_UNAVAILABLE;
1736 dlb_log_create_sched_domain_args(struct dlb_hw *hw,
1737 struct dlb_create_sched_domain_args *args)
1739 DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
1740 DLB_HW_INFO(hw, "\tNumber of LDB queues: %d\n",
1741 args->num_ldb_queues);
1742 DLB_HW_INFO(hw, "\tNumber of LDB ports: %d\n",
1743 args->num_ldb_ports);
1744 DLB_HW_INFO(hw, "\tNumber of DIR ports: %d\n",
1745 args->num_dir_ports);
1746 DLB_HW_INFO(hw, "\tNumber of ATM inflights: %d\n",
1747 args->num_atomic_inflights);
1748 DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
1749 args->num_hist_list_entries);
1750 DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
1751 args->num_ldb_credits);
1752 DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
1753 args->num_dir_credits);
1754 DLB_HW_INFO(hw, "\tNumber of LDB credit pools: %d\n",
1755 args->num_ldb_credit_pools);
1756 DLB_HW_INFO(hw, "\tNumber of DIR credit pools: %d\n",
1757 args->num_dir_credit_pools);
1761 * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
1762 * domain and its resources.
1763 * @hw: Contains the current state of the DLB hardware.
1764 * @args: User-provided arguments.
1765 * @resp: Response to user.
1767 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
1768 * satisfy a request, resp->status will be set accordingly.
1770 int dlb_hw_create_sched_domain(struct dlb_hw *hw,
1771 struct dlb_create_sched_domain_args *args,
1772 struct dlb_cmd_response *resp)
1774 struct dlb_domain *domain;
1775 struct dlb_function_resources *rsrcs;
1780 dlb_log_create_sched_domain_args(hw, args);
1782 /* Verify that hardware resources are available before attempting to
1783 * satisfy the request. This simplifies the error unwinding code.
1785 if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
1788 domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
1790 /* Verification should catch this. */
1791 if (domain == NULL) {
1793 "[%s():%d] Internal error: no available domains\n",
1794 __func__, __LINE__);
1798 if (domain->configured) {
1800 "[%s()] Internal error: avail_domains contains configured domains.\n",
1805 dlb_init_domain_rsrc_lists(domain);
1807 /* Verification should catch this too. */
1808 ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
1811 "[%s()] Internal error: failed to verify args.\n",
1817 dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
1819 dlb_list_add(&rsrcs->used_domains, &domain->func_list);
1821 resp->id = domain->id;
1828 dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
1829 struct dlb_domain *domain,
1830 struct dlb_create_ldb_pool_args *args,
1831 struct dlb_credit_pool *pool)
1833 union dlb_sys_ldb_pool_enbld r0 = { {0} };
1834 union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
1835 union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
1836 union dlb_chp_qed_fl_base r3 = { {0} };
1837 union dlb_chp_qed_fl_lim r4 = { {0} };
1838 union dlb_chp_qed_fl_push_ptr r5 = { {0} };
1839 union dlb_chp_qed_fl_pop_ptr r6 = { {0} };
1841 r1.field.limit = args->num_ldb_credits;
1843 DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
1845 r2.field.count = args->num_ldb_credits;
1847 DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
1849 r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
1851 DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
1853 r4.field.freelist_disable = 0;
1854 r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
1856 DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
1858 r5.field.push_ptr = r3.field.base;
1859 r5.field.generation = 1;
1861 DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
1863 r6.field.pop_ptr = r3.field.base;
1864 r6.field.generation = 0;
1866 DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
1868 r0.field.pool_enabled = 1;
1870 DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
1872 pool->avail_credits = args->num_ldb_credits;
1873 pool->total_credits = args->num_ldb_credits;
1874 domain->qed_freelist.offset += args->num_ldb_credits;
1876 pool->configured = true;
1880 dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
1882 struct dlb_create_ldb_pool_args *args,
1883 struct dlb_cmd_response *resp)
1885 struct dlb_freelist *qed_freelist;
1886 struct dlb_domain *domain;
1888 domain = dlb_get_domain_from_id(hw, domain_id);
1890 if (domain == NULL) {
1891 resp->status = DLB_ST_INVALID_DOMAIN_ID;
1895 if (!domain->configured) {
1896 resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
1900 qed_freelist = &domain->qed_freelist;
1902 if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
1903 resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
1907 if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
1908 resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
1912 if (domain->started) {
1913 resp->status = DLB_ST_DOMAIN_STARTED;
1921 dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
1923 struct dlb_create_ldb_pool_args *args)
1925 DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
1926 DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
1927 DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
1928 args->num_ldb_credits);
1932 * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
1933 * @hw: Contains the current state of the DLB hardware.
1934 * @args: User-provided arguments.
1935 * @resp: Response to user.
1937 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
1938 * satisfy a request, resp->status will be set accordingly.
1940 int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
1942 struct dlb_create_ldb_pool_args *args,
1943 struct dlb_cmd_response *resp)
1945 struct dlb_credit_pool *pool;
1946 struct dlb_domain *domain;
1948 dlb_log_create_ldb_pool_args(hw, domain_id, args);
1950 /* Verify that hardware resources are available before attempting to
1951 * satisfy the request. This simplifies the error unwinding code.
1953 if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
1956 domain = dlb_get_domain_from_id(hw, domain_id);
1957 if (domain == NULL) {
1959 "[%s():%d] Internal error: domain not found\n",
1960 __func__, __LINE__);
1964 pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
1966 /* Verification should catch this. */
1969 "[%s():%d] Internal error: no available ldb credit pools\n",
1970 __func__, __LINE__);
1974 dlb_configure_ldb_credit_pool(hw, domain, args, pool);
1976 /* Configuration succeeded, so move the resource from the 'avail' to
1979 dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
1981 dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
1984 resp->id = pool->id;
1990 dlb_configure_dir_credit_pool(struct dlb_hw *hw,
1991 struct dlb_domain *domain,
1992 struct dlb_create_dir_pool_args *args,
1993 struct dlb_credit_pool *pool)
1995 union dlb_sys_dir_pool_enbld r0 = { {0} };
1996 union dlb_chp_dir_pool_crd_lim r1 = { {0} };
1997 union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
1998 union dlb_chp_dqed_fl_base r3 = { {0} };
1999 union dlb_chp_dqed_fl_lim r4 = { {0} };
2000 union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
2001 union dlb_chp_dqed_fl_pop_ptr r6 = { {0} };
2003 r1.field.limit = args->num_dir_credits;
2005 DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
2007 r2.field.count = args->num_dir_credits;
2009 DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
2011 r3.field.base = domain->dqed_freelist.base +
2012 domain->dqed_freelist.offset;
2014 DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
2016 r4.field.freelist_disable = 0;
2017 r4.field.limit = r3.field.base + args->num_dir_credits - 1;
2019 DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
2021 r5.field.push_ptr = r3.field.base;
2022 r5.field.generation = 1;
2024 DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
2026 r6.field.pop_ptr = r3.field.base;
2027 r6.field.generation = 0;
2029 DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
2031 r0.field.pool_enabled = 1;
2033 DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
2035 pool->avail_credits = args->num_dir_credits;
2036 pool->total_credits = args->num_dir_credits;
2037 domain->dqed_freelist.offset += args->num_dir_credits;
2039 pool->configured = true;
2043 dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
2045 struct dlb_create_dir_pool_args *args,
2046 struct dlb_cmd_response *resp)
2048 struct dlb_freelist *dqed_freelist;
2049 struct dlb_domain *domain;
2051 domain = dlb_get_domain_from_id(hw, domain_id);
2053 if (domain == NULL) {
2054 resp->status = DLB_ST_INVALID_DOMAIN_ID;
2058 if (!domain->configured) {
2059 resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
2063 dqed_freelist = &domain->dqed_freelist;
2065 if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
2066 resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
2070 if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
2071 resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
2075 if (domain->started) {
2076 resp->status = DLB_ST_DOMAIN_STARTED;
2084 dlb_log_create_dir_pool_args(struct dlb_hw *hw,
2086 struct dlb_create_dir_pool_args *args)
2088 DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
2089 DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
2090 DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
2091 args->num_dir_credits);
2095 * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
2096 * @hw: Contains the current state of the DLB hardware.
2097 * @args: User-provided arguments.
2098 * @resp: Response to user.
2100 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
2101 * satisfy a request, resp->status will be set accordingly.
2103 int dlb_hw_create_dir_pool(struct dlb_hw *hw,
2105 struct dlb_create_dir_pool_args *args,
2106 struct dlb_cmd_response *resp)
2108 struct dlb_credit_pool *pool;
2109 struct dlb_domain *domain;
2111 dlb_log_create_dir_pool_args(hw, domain_id, args);
2113 /* Verify that hardware resources are available before attempting to
2114 * satisfy the request. This simplifies the error unwinding code.
2116 /* At least one available pool */
2117 if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
2120 domain = dlb_get_domain_from_id(hw, domain_id);
2121 if (domain == NULL) {
2123 "[%s():%d] Internal error: domain not found\n",
2124 __func__, __LINE__);
2128 pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
2130 /* Verification should catch this. */
2133 "[%s():%d] Internal error: no available dir credit pools\n",
2134 __func__, __LINE__);
2138 dlb_configure_dir_credit_pool(hw, domain, args, pool);
2140 /* Configuration succeeded, so move the resource from the 'avail' to
2143 dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
2145 dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
2148 resp->id = pool->id;
2153 static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
2154 struct dlb_ldb_port *port)
2156 union dlb_lsp_cq_ldb_infl_cnt r0;
2158 r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
2160 return r0.field.count;
2163 static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
2164 struct dlb_ldb_port *port)
2166 union dlb_lsp_cq_ldb_tkn_cnt r0;
2168 r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
2170 return r0.field.token_count;
2173 static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
2175 u32 infl_cnt, tkn_cnt;
2178 infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
2180 /* Account for the initial token count, which is used in order to
2181 * provide a CQ with depth less than 8.
2183 tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
2185 if (infl_cnt || tkn_cnt) {
2186 struct dlb_hcw hcw_mem[8], *hcw;
2189 pp_addr = os_map_producer_port(hw, port->id, true);
2191 /* Point hcw to a 64B-aligned location */
2192 hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
2194 /* Program the first HCW for a completion and token return and
2195 * the other HCWs as NOOPS
2198 memset(hcw, 0, 4 * sizeof(*hcw));
2199 hcw->qe_comp = (infl_cnt > 0);
2200 hcw->cq_token = (tkn_cnt > 0);
2201 hcw->lock_id = tkn_cnt - 1;
2203 /* Return tokens in the first HCW */
2204 dlb_movdir64b(pp_addr, hcw);
2208 /* Issue remaining completions (if any) */
2209 for (i = 1; i < infl_cnt; i++)
2210 dlb_movdir64b(pp_addr, hcw);
2212 os_fence_hcw(hw, pp_addr);
2214 os_unmap_producer_port(hw, pp_addr);
2220 static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
2221 struct dlb_domain *domain,
2224 struct dlb_list_entry *iter;
2226 struct dlb_ldb_port *port;
2229 /* If the domain hasn't been started, there's no traffic to drain */
2230 if (!domain->started)
2233 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
2235 dlb_ldb_port_cq_disable(hw, port);
2237 ret = dlb_drain_ldb_cq(hw, port);
2242 dlb_ldb_port_cq_enable(hw, port);
2248 static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
2249 struct dlb_domain *domain)
2251 int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
2252 struct dlb_list_entry *iter;
2254 union dlb_sys_ldb_vasqid_v r0;
2255 struct dlb_ldb_queue *queue;
2257 r0.field.vasqid_v = 0;
2259 DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
2260 int idx = domain_offset + queue->id;
2262 DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
2266 static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
2267 struct dlb_domain *domain)
2269 struct dlb_list_entry *iter;
2271 union dlb_chp_sn_chk_enbl r1;
2272 struct dlb_ldb_port *port;
2276 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
2278 DLB_CHP_SN_CHK_ENBL(port->id),
2282 static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
2283 struct dlb_domain *domain)
2285 struct dlb_list_entry *iter;
2287 union dlb_chp_ldb_pp_crd_req_state r0;
2288 struct dlb_ldb_port *port;
2290 r0.field.no_pp_credit_update = 1;
2292 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
2294 DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
2298 static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
2299 struct dlb_domain *domain)
2301 struct dlb_list_entry *iter;
2303 union dlb_chp_ldb_cq_int_enb r0 = { {0} };
2304 union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
2305 struct dlb_ldb_port *port;
2307 r0.field.en_tim = 0;
2308 r0.field.en_depth = 0;
2310 r1.field.wd_enable = 0;
2312 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
2314 DLB_CHP_LDB_CQ_INT_ENB(port->id),
2318 DLB_CHP_LDB_CQ_WD_ENB(port->id),
2323 static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
2324 struct dlb_domain *domain)
2326 int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
2327 struct dlb_list_entry *iter;
2329 union dlb_sys_dir_vasqid_v r0;
2330 struct dlb_dir_pq_pair *port;
2332 r0.field.vasqid_v = 0;
2334 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
2335 int idx = domain_offset + port->id;
2337 DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
2341 static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
2342 struct dlb_domain *domain)
2344 struct dlb_list_entry *iter;
2346 union dlb_chp_dir_cq_int_enb r0 = { {0} };
2347 union dlb_chp_dir_cq_wd_enb r1 = { {0} };
2348 struct dlb_dir_pq_pair *port;
2350 r0.field.en_tim = 0;
2351 r0.field.en_depth = 0;
2353 r1.field.wd_enable = 0;
2355 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
2357 DLB_CHP_DIR_CQ_INT_ENB(port->id),
2361 DLB_CHP_DIR_CQ_WD_ENB(port->id),
2366 static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
2367 struct dlb_domain *domain)
2369 struct dlb_list_entry *iter;
2371 union dlb_chp_dir_pp_crd_req_state r0;
2372 struct dlb_dir_pq_pair *port;
2374 r0.field.no_pp_credit_update = 1;
2376 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
2378 DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
2382 static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
2383 struct dlb_domain *domain)
2385 struct dlb_list_entry *iter;
2387 struct dlb_dir_pq_pair *port;
2389 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
2390 port->enabled = false;
2392 dlb_dir_port_cq_disable(hw, port);
2396 static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
2397 struct dlb_domain *domain)
2399 struct dlb_list_entry *iter;
2401 struct dlb_ldb_port *port;
2403 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
2404 port->enabled = false;
2406 dlb_ldb_port_cq_disable(hw, port);
2410 static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
2411 struct dlb_domain *domain)
2413 struct dlb_list_entry *iter;
2415 struct dlb_ldb_port *port;
2417 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
2418 port->enabled = true;
2420 dlb_ldb_port_cq_enable(hw, port);
2424 static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
2427 if (id >= DLB_MAX_NUM_LDB_QUEUES)
2430 return &hw->rsrcs.ldb_queues[id];
2433 static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
2434 struct dlb_ldb_port *port,
2437 union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
2439 r2.field.cq = port->id;
2440 r2.field.qidix = slot;
2442 r2.field.rlist_haswork_v = 1;
2444 DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
2446 memset(&r2, 0, sizeof(r2));
2448 r2.field.cq = port->id;
2449 r2.field.qidix = slot;
2451 r2.field.nalb_haswork_v = 1;
2453 DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
2458 static void dlb_domain_finish_map_port(struct dlb_hw *hw,
2459 struct dlb_domain *domain,
2460 struct dlb_ldb_port *port)
2464 for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
2465 union dlb_lsp_qid_ldb_infl_cnt r0;
2466 struct dlb_ldb_queue *queue;
2469 if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
2472 qid = port->qid_map[i].qid;
2474 queue = dlb_get_ldb_queue_from_id(hw, qid);
2476 if (queue == NULL) {
2478 "[%s()] Internal error: unable to find queue %d\n",
2483 r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
2488 /* Disable the affected CQ, and the CQs already mapped to the
2489 * QID, before reading the QID's inflight count a second time.
2490 * There is an unlikely race in which the QID may schedule one
2491 * more QE after we read an inflight count of 0, and disabling
2492 * the CQs guarantees that the race will not occur after a
2493 * re-read of the inflight count register.
2496 dlb_ldb_port_cq_disable(hw, port);
2498 dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
2500 r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
2502 if (r0.field.count) {
2504 dlb_ldb_port_cq_enable(hw, port);
2506 dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
2511 dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
2516 dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
2517 struct dlb_domain *domain)
2519 struct dlb_list_entry *iter;
2521 struct dlb_ldb_port *port;
2523 if (!domain->configured || domain->num_pending_additions == 0)
2526 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
2527 dlb_domain_finish_map_port(hw, domain, port);
2529 return domain->num_pending_additions;
2532 unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
2536 /* Finish queue map jobs for any domain that needs it */
2537 for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
2538 struct dlb_domain *domain = &hw->domains[i];
2540 num += dlb_domain_finish_map_qid_procedures(hw, domain);
2547 static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
2548 struct dlb_domain *domain)
2550 struct dlb_list_entry *iter;
2552 struct dlb_ldb_port *port;
2554 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
2557 for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
2558 if (dlb_ldb_cq_inflight_count(hw, port) == 0)
2562 if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
2564 "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
2565 __func__, port->id);
2574 static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
2575 struct dlb_domain *domain,
2576 struct dlb_ldb_port *port,
2579 enum dlb_qid_map_state state;
2580 struct dlb_ldb_queue *queue;
2582 queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
2584 state = port->qid_map[slot].state;
2586 /* Update the QID2CQIDX and CQ2QID vectors */
2587 dlb_ldb_port_unmap_qid(hw, port, queue);
2589 /* Ensure the QID will not be serviced by this {CQ, slot} by clearing
2592 dlb_ldb_port_clear_has_work_bits(hw, port, slot);
2594 /* Reset the {CQ, slot} to its default state */
2595 dlb_ldb_port_set_queue_if_status(hw, port, slot);
2597 /* Re-enable the CQ if it was not manually disabled by the user */
2599 dlb_ldb_port_cq_enable(hw, port);
2601 /* If there is a mapping that is pending this slot's removal, perform
2604 if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
2605 struct dlb_ldb_port_qid_map *map;
2606 struct dlb_ldb_queue *map_queue;
2609 map = &port->qid_map[slot];
2611 map->qid = map->pending_qid;
2612 map->priority = map->pending_priority;
2614 map_queue = &hw->rsrcs.ldb_queues[map->qid];
2615 prio = map->priority;
2617 dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
2621 static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
2622 struct dlb_domain *domain,
2623 struct dlb_ldb_port *port)
2625 union dlb_lsp_cq_ldb_infl_cnt r0;
2628 if (port->num_pending_removals == 0)
2631 /* The unmap requires all the CQ's outstanding inflights to be
2634 r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
2635 if (r0.field.count > 0)
2638 for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
2639 struct dlb_ldb_port_qid_map *map;
2641 map = &port->qid_map[i];
2643 if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
2644 map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
2647 dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
2654 dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
2655 struct dlb_domain *domain)
2657 struct dlb_list_entry *iter;
2659 struct dlb_ldb_port *port;
2661 if (!domain->configured || domain->num_pending_removals == 0)
2664 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
2665 dlb_domain_finish_unmap_port(hw, domain, port);
2667 return domain->num_pending_removals;
2670 unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
2674 /* Finish queue unmap jobs for any domain that needs it */
2675 for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
2676 struct dlb_domain *domain = &hw->domains[i];
2678 num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
2684 /* Returns whether the queue is empty, including its inflight and replay
2687 static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
2688 struct dlb_ldb_queue *queue)
2690 union dlb_lsp_qid_ldb_replay_cnt r0;
2691 union dlb_lsp_qid_aqed_active_cnt r1;
2692 union dlb_lsp_qid_atq_enqueue_cnt r2;
2693 union dlb_lsp_qid_ldb_enqueue_cnt r3;
2694 union dlb_lsp_qid_ldb_infl_cnt r4;
2696 r0.val = DLB_CSR_RD(hw,
2697 DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
2701 r1.val = DLB_CSR_RD(hw,
2702 DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
2706 r2.val = DLB_CSR_RD(hw,
2707 DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
2711 r3.val = DLB_CSR_RD(hw,
2712 DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
2716 r4.val = DLB_CSR_RD(hw,
2717 DLB_LSP_QID_LDB_INFL_CNT(queue->id));
2724 static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
2725 struct dlb_domain *domain)
2727 struct dlb_list_entry *iter;
2729 struct dlb_ldb_queue *queue;
2731 DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
2732 if (queue->num_mappings == 0)
2735 if (!dlb_ldb_queue_is_empty(hw, queue))
2742 static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
2743 struct dlb_domain *domain)
2747 /* If the domain hasn't been started, there's no traffic to drain */
2748 if (!domain->started)
2751 if (domain->num_pending_removals > 0) {
2753 "[%s()] Internal error: failed to unmap domain queues\n",
2758 for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
2759 ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
2763 if (dlb_domain_mapped_queues_empty(hw, domain))
2767 if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
2769 "[%s()] Internal error: failed to empty queues\n",
2774 /* Drain the CQs one more time. For the queues to go empty, they would
2775 * have scheduled one or more QEs.
2777 ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
2784 static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
2785 struct dlb_domain *domain,
2786 struct dlb_ldb_queue *queue)
2788 struct dlb_ldb_port *port;
2791 /* If a domain has LDB queues, it must have LDB ports */
2792 if (dlb_list_empty(&domain->used_ldb_ports)) {
2794 "[%s()] Internal error: No configured LDB ports\n",
2799 port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
2801 /* If necessary, free up a QID slot in this CQ */
2802 if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
2803 struct dlb_ldb_queue *mapped_queue;
2805 mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
2807 ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
2812 ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
2816 return dlb_domain_drain_mapped_queues(hw, domain);
2819 static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
2820 struct dlb_domain *domain)
2822 struct dlb_list_entry *iter;
2824 struct dlb_ldb_queue *queue;
2827 /* If the domain hasn't been started, there's no traffic to drain */
2828 if (!domain->started)
2831 DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
2832 if (queue->num_mappings != 0 ||
2833 dlb_ldb_queue_is_empty(hw, queue))
2836 ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
2844 static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
2845 struct dlb_domain *domain)
2847 struct dlb_list_entry *iter;
2849 struct dlb_credit_pool *pool;
2851 /* Confirm that all credits are returned to the domain's credit pools */
2852 DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
2853 union dlb_chp_qed_fl_push_ptr r0;
2854 union dlb_chp_qed_fl_pop_ptr r1;
2855 unsigned long pop_offs, push_offs;
2858 push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
2859 pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
2861 for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
2862 r0.val = DLB_CSR_RD(hw, push_offs);
2864 r1.val = DLB_CSR_RD(hw, pop_offs);
2866 /* Break early if the freelist is replenished */
2867 if (r1.field.pop_ptr == r0.field.push_ptr &&
2868 r1.field.generation != r0.field.generation) {
2873 /* Error if the freelist is not full */
2874 if (r1.field.pop_ptr != r0.field.push_ptr ||
2875 r1.field.generation == r0.field.generation) {
2883 static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
2884 struct dlb_domain *domain)
2886 struct dlb_list_entry *iter;
2888 struct dlb_credit_pool *pool;
2890 /* Confirm that all credits are returned to the domain's credit pools */
2891 DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
2892 union dlb_chp_dqed_fl_push_ptr r0;
2893 union dlb_chp_dqed_fl_pop_ptr r1;
2894 unsigned long pop_offs, push_offs;
2897 push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
2898 pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
2900 for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
2901 r0.val = DLB_CSR_RD(hw, push_offs);
2903 r1.val = DLB_CSR_RD(hw, pop_offs);
2905 /* Break early if the freelist is replenished */
2906 if (r1.field.pop_ptr == r0.field.push_ptr &&
2907 r1.field.generation != r0.field.generation) {
2912 /* Error if the freelist is not full */
2913 if (r1.field.pop_ptr != r0.field.push_ptr ||
2914 r1.field.generation == r0.field.generation) {
2922 static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
2923 struct dlb_dir_pq_pair *queue)
2925 union dlb_lsp_qid_dir_enqueue_cnt r0;
2927 r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
2929 return r0.field.count;
2932 static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
2933 struct dlb_dir_pq_pair *queue)
2935 return dlb_dir_queue_depth(hw, queue) == 0;
2938 static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
2939 struct dlb_domain *domain)
2941 struct dlb_list_entry *iter;
2943 struct dlb_dir_pq_pair *queue;
2945 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
2946 if (!dlb_dir_queue_is_empty(hw, queue))
2953 static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
2954 struct dlb_dir_pq_pair *port)
2956 union dlb_lsp_cq_dir_tkn_cnt r0;
2958 r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
2960 return r0.field.count;
2963 static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
2965 unsigned int port_id = port->id;
2968 /* Return any outstanding tokens */
2969 cnt = dlb_dir_cq_token_count(hw, port);
2972 struct dlb_hcw hcw_mem[8], *hcw;
2975 pp_addr = os_map_producer_port(hw, port_id, false);
2977 /* Point hcw to a 64B-aligned location */
2978 hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
2980 /* Program the first HCW for a batch token return and
2983 memset(hcw, 0, 4 * sizeof(*hcw));
2985 hcw->lock_id = cnt - 1;
2987 dlb_movdir64b(pp_addr, hcw);
2989 os_fence_hcw(hw, pp_addr);
2991 os_unmap_producer_port(hw, pp_addr);
2995 static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
2996 struct dlb_domain *domain,
2999 struct dlb_list_entry *iter;
3001 struct dlb_dir_pq_pair *port;
3003 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
3004 /* Can't drain a port if it's not configured, and there's
3005 * nothing to drain if its queue is unconfigured.
3007 if (!port->port_configured || !port->queue_configured)
3011 dlb_dir_port_cq_disable(hw, port);
3013 dlb_drain_dir_cq(hw, port);
3016 dlb_dir_port_cq_enable(hw, port);
3022 static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
3023 struct dlb_domain *domain)
3027 /* If the domain hasn't been started, there's no traffic to drain */
3028 if (!domain->started)
3031 for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
3032 dlb_domain_drain_dir_cqs(hw, domain, true);
3034 if (dlb_domain_dir_queues_empty(hw, domain))
3038 if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
3040 "[%s()] Internal error: failed to empty queues\n",
3045 /* Drain the CQs one more time. For the queues to go empty, they would
3046 * have scheduled one or more QEs.
3048 dlb_domain_drain_dir_cqs(hw, domain, true);
3053 static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
3054 struct dlb_domain *domain)
3056 struct dlb_list_entry *iter;
3058 struct dlb_dir_pq_pair *port;
3059 union dlb_sys_dir_pp_v r1;
3063 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
3065 DLB_SYS_DIR_PP_V(port->id),
3069 static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
3070 struct dlb_domain *domain)
3072 struct dlb_list_entry *iter;
3074 union dlb_sys_ldb_pp_v r1;
3075 struct dlb_ldb_port *port;
3079 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
3081 DLB_SYS_LDB_PP_V(port->id),
3084 hw->pf.num_enabled_ldb_ports--;
3088 static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
3089 struct dlb_domain *domain)
3091 struct dlb_list_entry *iter;
3093 union dlb_sys_dir_pool_enbld r0 = { {0} };
3094 struct dlb_credit_pool *pool;
3096 DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
3098 DLB_SYS_DIR_POOL_ENBLD(pool->id),
3102 static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
3103 struct dlb_domain *domain)
3105 struct dlb_list_entry *iter;
3107 union dlb_sys_ldb_pool_enbld r0 = { {0} };
3108 struct dlb_credit_pool *pool;
3110 DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
3112 DLB_SYS_LDB_POOL_ENBLD(pool->id),
3116 static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
3118 union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
3119 union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
3122 r1.field.vf_reset_start = 1;
3124 r1.field.vf_reset_type = type;
3125 r1.field.vf_reset_id = id;
3127 DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
3129 /* Wait for hardware to complete. This is a finite time operation,
3130 * but wait set a loop bound just in case.
3132 for (i = 0; i < 1024 * 1024; i++) {
3133 r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
3135 if (r0.field.chp_vf_reset_done &&
3136 r0.field.rop_vf_reset_done &&
3137 r0.field.lsp_vf_reset_done &&
3138 r0.field.nalb_vf_reset_done &&
3139 r0.field.ap_vf_reset_done &&
3140 r0.field.dp_vf_reset_done &&
3141 r0.field.qed_vf_reset_done &&
3142 r0.field.dqed_vf_reset_done &&
3143 r0.field.aqed_vf_reset_done)
3152 static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
3153 struct dlb_domain *domain)
3155 struct dlb_list_entry *iter;
3157 struct dlb_dir_pq_pair *dir_port;
3158 struct dlb_ldb_queue *ldb_queue;
3159 struct dlb_ldb_port *ldb_port;
3160 struct dlb_credit_pool *pool;
3163 DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
3164 ret = dlb_reset_hw_resource(hw,
3165 VF_RST_TYPE_POOL_LDB,
3171 DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
3172 ret = dlb_reset_hw_resource(hw,
3173 VF_RST_TYPE_POOL_DIR,
3179 DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
3180 ret = dlb_reset_hw_resource(hw,
3181 VF_RST_TYPE_QID_LDB,
3187 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
3188 ret = dlb_reset_hw_resource(hw,
3189 VF_RST_TYPE_QID_DIR,
3195 DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
3196 ret = dlb_reset_hw_resource(hw,
3203 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
3204 ret = dlb_reset_hw_resource(hw,
3214 static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
3215 struct dlb_domain *domain)
3217 struct dlb_list_entry *iter;
3219 struct dlb_dir_pq_pair *dir_port;
3220 struct dlb_ldb_port *ldb_port;
3221 struct dlb_credit_pool *pool;
3222 struct dlb_ldb_queue *queue;
3224 /* Confirm that all credits are returned to the domain's credit pools */
3225 DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
3226 union dlb_chp_dqed_fl_pop_ptr r0;
3227 union dlb_chp_dqed_fl_push_ptr r1;
3229 r0.val = DLB_CSR_RD(hw,
3230 DLB_CHP_DQED_FL_POP_PTR(pool->id));
3232 r1.val = DLB_CSR_RD(hw,
3233 DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
3235 if (r0.field.pop_ptr != r1.field.push_ptr ||
3236 r0.field.generation == r1.field.generation) {
3238 "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
3239 __func__, pool->id);
3244 /* Confirm that all the domain's queue's inflight counts and AQED
3245 * active counts are 0.
3247 DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
3248 if (!dlb_ldb_queue_is_empty(hw, queue)) {
3250 "[%s()] Internal error: failed to empty ldb queue %d\n",
3251 __func__, queue->id);
3256 /* Confirm that all the domain's CQs inflight and token counts are 0. */
3257 DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
3258 if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
3259 dlb_ldb_cq_token_count(hw, ldb_port)) {
3261 "[%s()] Internal error: failed to empty ldb port %d\n",
3262 __func__, ldb_port->id);
3267 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
3268 if (!dlb_dir_queue_is_empty(hw, dir_port)) {
3270 "[%s()] Internal error: failed to empty dir queue %d\n",
3271 __func__, dir_port->id);
3275 if (dlb_dir_cq_token_count(hw, dir_port)) {
3277 "[%s()] Internal error: failed to empty dir port %d\n",
3278 __func__, dir_port->id);
3286 static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
3287 struct dlb_ldb_port *port)
3289 union dlb_chp_ldb_pp_state_reset r0 = { {0} };
3292 DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
3293 DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
3295 /* Reset the port's load-balanced and directed credit state */
3296 r0.field.dir_type = 0;
3297 r0.field.reset_pp_state = 1;
3300 DLB_CHP_LDB_PP_STATE_RESET(port->id),
3303 r0.field.dir_type = 1;
3304 r0.field.reset_pp_state = 1;
3307 DLB_CHP_LDB_PP_STATE_RESET(port->id),
3311 DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
3312 DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
3315 DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
3316 DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
3319 DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
3320 DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
3323 DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
3324 DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
3327 DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
3328 DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
3331 DLB_CHP_LDB_LDB_PP2POOL(port->id),
3332 DLB_CHP_LDB_LDB_PP2POOL_RST);
3335 DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
3336 DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
3339 DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
3340 DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
3343 DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
3344 DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
3347 DLB_CHP_LDB_DIR_PP2POOL(port->id),
3348 DLB_CHP_LDB_DIR_PP2POOL_RST);
3351 DLB_SYS_LDB_PP2LDBPOOL(port->id),
3352 DLB_SYS_LDB_PP2LDBPOOL_RST);
3355 DLB_SYS_LDB_PP2DIRPOOL(port->id),
3356 DLB_SYS_LDB_PP2DIRPOOL_RST);
3359 DLB_CHP_HIST_LIST_LIM(port->id),
3360 DLB_CHP_HIST_LIST_LIM_RST);
3363 DLB_CHP_HIST_LIST_BASE(port->id),
3364 DLB_CHP_HIST_LIST_BASE_RST);
3367 DLB_CHP_HIST_LIST_POP_PTR(port->id),
3368 DLB_CHP_HIST_LIST_POP_PTR_RST);
3371 DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
3372 DLB_CHP_HIST_LIST_PUSH_PTR_RST);
3375 DLB_CHP_LDB_CQ_WPTR(port->id),
3376 DLB_CHP_LDB_CQ_WPTR_RST);
3379 DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
3380 DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
3383 DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
3384 DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
3387 DLB_CHP_LDB_CQ_INT_ENB(port->id),
3388 DLB_CHP_LDB_CQ_INT_ENB_RST);
3391 DLB_LSP_CQ_LDB_INFL_LIM(port->id),
3392 DLB_LSP_CQ_LDB_INFL_LIM_RST);
3395 DLB_LSP_CQ2PRIOV(port->id),
3396 DLB_LSP_CQ2PRIOV_RST);
3399 DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
3400 DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
3403 DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
3404 DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
3407 DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
3408 DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
3411 DLB_LSP_CQ_LDB_DSBL(port->id),
3412 DLB_LSP_CQ_LDB_DSBL_RST);
3415 DLB_SYS_LDB_CQ2VF_PF(port->id),
3416 DLB_SYS_LDB_CQ2VF_PF_RST);
3419 DLB_SYS_LDB_PP2VF_PF(port->id),
3420 DLB_SYS_LDB_PP2VF_PF_RST);
3423 DLB_SYS_LDB_CQ_ADDR_L(port->id),
3424 DLB_SYS_LDB_CQ_ADDR_L_RST);
3427 DLB_SYS_LDB_CQ_ADDR_U(port->id),
3428 DLB_SYS_LDB_CQ_ADDR_U_RST);
3431 DLB_SYS_LDB_PP_ADDR_L(port->id),
3432 DLB_SYS_LDB_PP_ADDR_L_RST);
3435 DLB_SYS_LDB_PP_ADDR_U(port->id),
3436 DLB_SYS_LDB_PP_ADDR_U_RST);
3439 DLB_SYS_LDB_PP_V(port->id),
3440 DLB_SYS_LDB_PP_V_RST);
3443 DLB_SYS_LDB_PP2VAS(port->id),
3444 DLB_SYS_LDB_PP2VAS_RST);
3447 DLB_SYS_LDB_CQ_ISR(port->id),
3448 DLB_SYS_LDB_CQ_ISR_RST);
3451 DLB_SYS_WBUF_LDB_FLAGS(port->id),
3452 DLB_SYS_WBUF_LDB_FLAGS_RST);
3455 static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
3456 struct dlb_dir_pq_pair *port)
3458 union dlb_chp_dir_pp_state_reset r0 = { {0} };
3461 DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
3462 DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
3464 /* Reset the port's load-balanced and directed credit state */
3465 r0.field.dir_type = 0;
3466 r0.field.reset_pp_state = 1;
3469 DLB_CHP_DIR_PP_STATE_RESET(port->id),
3472 r0.field.dir_type = 1;
3473 r0.field.reset_pp_state = 1;
3476 DLB_CHP_DIR_PP_STATE_RESET(port->id),
3480 DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
3481 DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
3484 DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
3485 DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
3488 DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
3489 DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
3492 DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
3493 DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
3496 DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
3497 DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
3500 DLB_CHP_DIR_LDB_PP2POOL(port->id),
3501 DLB_CHP_DIR_LDB_PP2POOL_RST);
3504 DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
3505 DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
3508 DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
3509 DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
3512 DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
3513 DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
3516 DLB_CHP_DIR_DIR_PP2POOL(port->id),
3517 DLB_CHP_DIR_DIR_PP2POOL_RST);
3520 DLB_SYS_DIR_PP2LDBPOOL(port->id),
3521 DLB_SYS_DIR_PP2LDBPOOL_RST);
3524 DLB_SYS_DIR_PP2DIRPOOL(port->id),
3525 DLB_SYS_DIR_PP2DIRPOOL_RST);
3528 DLB_CHP_DIR_CQ_WPTR(port->id),
3529 DLB_CHP_DIR_CQ_WPTR_RST);
3532 DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
3533 DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
3536 DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
3537 DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
3540 DLB_LSP_CQ_DIR_DSBL(port->id),
3541 DLB_LSP_CQ_DIR_DSBL_RST);
3544 DLB_CHP_DIR_CQ_WPTR(port->id),
3545 DLB_CHP_DIR_CQ_WPTR_RST);
3548 DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
3549 DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
3552 DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
3553 DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
3556 DLB_CHP_DIR_CQ_INT_ENB(port->id),
3557 DLB_CHP_DIR_CQ_INT_ENB_RST);
3560 DLB_SYS_DIR_CQ2VF_PF(port->id),
3561 DLB_SYS_DIR_CQ2VF_PF_RST);
3564 DLB_SYS_DIR_PP2VF_PF(port->id),
3565 DLB_SYS_DIR_PP2VF_PF_RST);
3568 DLB_SYS_DIR_CQ_ADDR_L(port->id),
3569 DLB_SYS_DIR_CQ_ADDR_L_RST);
3572 DLB_SYS_DIR_CQ_ADDR_U(port->id),
3573 DLB_SYS_DIR_CQ_ADDR_U_RST);
3576 DLB_SYS_DIR_PP_ADDR_L(port->id),
3577 DLB_SYS_DIR_PP_ADDR_L_RST);
3580 DLB_SYS_DIR_PP_ADDR_U(port->id),
3581 DLB_SYS_DIR_PP_ADDR_U_RST);
3584 DLB_SYS_DIR_PP_V(port->id),
3585 DLB_SYS_DIR_PP_V_RST);
3588 DLB_SYS_DIR_PP2VAS(port->id),
3589 DLB_SYS_DIR_PP2VAS_RST);
3592 DLB_SYS_DIR_CQ_ISR(port->id),
3593 DLB_SYS_DIR_CQ_ISR_RST);
3596 DLB_SYS_WBUF_DIR_FLAGS(port->id),
3597 DLB_SYS_WBUF_DIR_FLAGS_RST);
3600 static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
3601 struct dlb_domain *domain)
3603 struct dlb_list_entry *iter;
3605 struct dlb_dir_pq_pair *port;
3607 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
3608 __dlb_domain_reset_dir_port_registers(hw, port);
3611 static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
3612 struct dlb_domain *domain)
3614 struct dlb_list_entry *iter;
3616 struct dlb_ldb_queue *queue;
3618 DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
3620 DLB_AQED_PIPE_FL_LIM(queue->id),
3621 DLB_AQED_PIPE_FL_LIM_RST);
3624 DLB_AQED_PIPE_FL_BASE(queue->id),
3625 DLB_AQED_PIPE_FL_BASE_RST);
3628 DLB_AQED_PIPE_FL_POP_PTR(queue->id),
3629 DLB_AQED_PIPE_FL_POP_PTR_RST);
3632 DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
3633 DLB_AQED_PIPE_FL_PUSH_PTR_RST);
3636 DLB_AQED_PIPE_QID_FID_LIM(queue->id),
3637 DLB_AQED_PIPE_QID_FID_LIM_RST);
3640 DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
3641 DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
3644 DLB_LSP_QID_LDB_INFL_LIM(queue->id),
3645 DLB_LSP_QID_LDB_INFL_LIM_RST);
3648 DLB_SYS_LDB_QID_V(queue->id),
3649 DLB_SYS_LDB_QID_V_RST);
3652 DLB_SYS_LDB_QID_V(queue->id),
3653 DLB_SYS_LDB_QID_V_RST);
3656 DLB_CHP_ORD_QID_SN(queue->id),
3657 DLB_CHP_ORD_QID_SN_RST);
3660 DLB_CHP_ORD_QID_SN_MAP(queue->id),
3661 DLB_CHP_ORD_QID_SN_MAP_RST);
3664 DLB_RO_PIPE_QID2GRPSLT(queue->id),
3665 DLB_RO_PIPE_QID2GRPSLT_RST);
3669 static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
3670 struct dlb_domain *domain)
3672 struct dlb_list_entry *iter;
3674 struct dlb_dir_pq_pair *queue;
3676 DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
3678 DLB_SYS_DIR_QID_V(queue->id),
3679 DLB_SYS_DIR_QID_V_RST);
3683 static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
3684 struct dlb_domain *domain)
3686 struct dlb_list_entry *iter;
3688 struct dlb_credit_pool *pool;
3690 DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
3692 DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
3693 DLB_CHP_LDB_POOL_CRD_LIM_RST);
3696 DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
3697 DLB_CHP_LDB_POOL_CRD_CNT_RST);
3700 DLB_CHP_QED_FL_BASE(pool->id),
3701 DLB_CHP_QED_FL_BASE_RST);
3704 DLB_CHP_QED_FL_LIM(pool->id),
3705 DLB_CHP_QED_FL_LIM_RST);
3708 DLB_CHP_QED_FL_PUSH_PTR(pool->id),
3709 DLB_CHP_QED_FL_PUSH_PTR_RST);
3712 DLB_CHP_QED_FL_POP_PTR(pool->id),
3713 DLB_CHP_QED_FL_POP_PTR_RST);
3717 static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
3718 struct dlb_domain *domain)
3720 struct dlb_list_entry *iter;
3722 struct dlb_credit_pool *pool;
3724 DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
3726 DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
3727 DLB_CHP_DIR_POOL_CRD_LIM_RST);
3730 DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
3731 DLB_CHP_DIR_POOL_CRD_CNT_RST);
3734 DLB_CHP_DQED_FL_BASE(pool->id),
3735 DLB_CHP_DQED_FL_BASE_RST);
3738 DLB_CHP_DQED_FL_LIM(pool->id),
3739 DLB_CHP_DQED_FL_LIM_RST);
3742 DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
3743 DLB_CHP_DQED_FL_PUSH_PTR_RST);
3746 DLB_CHP_DQED_FL_POP_PTR(pool->id),
3747 DLB_CHP_DQED_FL_POP_PTR_RST);
3751 static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
3752 struct dlb_domain *domain)
3754 struct dlb_list_entry *iter;
3756 struct dlb_ldb_port *port;
3758 DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
3759 __dlb_domain_reset_ldb_port_registers(hw, port);
3762 static void dlb_domain_reset_registers(struct dlb_hw *hw,
3763 struct dlb_domain *domain)
3765 dlb_domain_reset_ldb_port_registers(hw, domain);
3767 dlb_domain_reset_dir_port_registers(hw, domain);
3769 dlb_domain_reset_ldb_queue_registers(hw, domain);
3771 dlb_domain_reset_dir_queue_registers(hw, domain);
3773 dlb_domain_reset_ldb_pool_registers(hw, domain);
3775 dlb_domain_reset_dir_pool_registers(hw, domain);
3778 static int dlb_domain_reset_software_state(struct dlb_hw *hw,
3779 struct dlb_domain *domain)
3781 struct dlb_ldb_queue *tmp_ldb_queue;
3782 RTE_SET_USED(tmp_ldb_queue);
3783 struct dlb_dir_pq_pair *tmp_dir_port;
3784 RTE_SET_USED(tmp_dir_port);
3785 struct dlb_ldb_port *tmp_ldb_port;
3786 RTE_SET_USED(tmp_ldb_port);
3787 struct dlb_credit_pool *tmp_pool;
3788 RTE_SET_USED(tmp_pool);
3789 struct dlb_list_entry *iter1;
3790 RTE_SET_USED(iter1);
3791 struct dlb_list_entry *iter2;
3792 RTE_SET_USED(iter2);
3793 struct dlb_ldb_queue *ldb_queue;
3794 struct dlb_dir_pq_pair *dir_port;
3795 struct dlb_ldb_port *ldb_port;
3796 struct dlb_credit_pool *pool;
3798 struct dlb_function_resources *rsrcs;
3799 struct dlb_list_head *list;
3802 rsrcs = domain->parent_func;
3804 /* Move the domain's ldb queues to the function's avail list */
3805 list = &domain->used_ldb_queues;
3806 DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
3807 if (ldb_queue->sn_cfg_valid) {
3808 struct dlb_sn_group *grp;
3810 grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
3812 dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
3813 ldb_queue->sn_cfg_valid = false;
3816 ldb_queue->owned = false;
3817 ldb_queue->num_mappings = 0;
3818 ldb_queue->num_pending_additions = 0;
3820 dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
3821 dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
3822 rsrcs->num_avail_ldb_queues++;
3825 list = &domain->avail_ldb_queues;
3826 DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
3827 ldb_queue->owned = false;
3829 dlb_list_del(&domain->avail_ldb_queues,
3830 &ldb_queue->domain_list);
3831 dlb_list_add(&rsrcs->avail_ldb_queues,
3832 &ldb_queue->func_list);
3833 rsrcs->num_avail_ldb_queues++;
3836 /* Move the domain's ldb ports to the function's avail list */
3837 list = &domain->used_ldb_ports;
3838 DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
3841 ldb_port->owned = false;
3842 ldb_port->configured = false;
3843 ldb_port->num_pending_removals = 0;
3844 ldb_port->num_mappings = 0;
3845 for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
3846 ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
3848 dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
3849 dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
3850 rsrcs->num_avail_ldb_ports++;
3853 list = &domain->avail_ldb_ports;
3854 DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
3855 ldb_port->owned = false;
3857 dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
3858 dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
3859 rsrcs->num_avail_ldb_ports++;
3862 /* Move the domain's dir ports to the function's avail list */
3863 list = &domain->used_dir_pq_pairs;
3864 DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
3865 dir_port->owned = false;
3866 dir_port->port_configured = false;
3868 dlb_list_del(&domain->used_dir_pq_pairs,
3869 &dir_port->domain_list);
3871 dlb_list_add(&rsrcs->avail_dir_pq_pairs,
3872 &dir_port->func_list);
3873 rsrcs->num_avail_dir_pq_pairs++;
3876 list = &domain->avail_dir_pq_pairs;
3877 DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
3878 dir_port->owned = false;
3880 dlb_list_del(&domain->avail_dir_pq_pairs,
3881 &dir_port->domain_list);
3883 dlb_list_add(&rsrcs->avail_dir_pq_pairs,
3884 &dir_port->func_list);
3885 rsrcs->num_avail_dir_pq_pairs++;
3888 /* Return hist list entries to the function */
3889 ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
3890 domain->hist_list_entry_base,
3891 domain->total_hist_list_entries);
3894 "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
3899 domain->total_hist_list_entries = 0;
3900 domain->avail_hist_list_entries = 0;
3901 domain->hist_list_entry_base = 0;
3902 domain->hist_list_entry_offset = 0;
3904 /* Return QED entries to the function */
3905 ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
3906 domain->qed_freelist.base,
3907 (domain->qed_freelist.bound -
3908 domain->qed_freelist.base));
3911 "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
3916 domain->qed_freelist.base = 0;
3917 domain->qed_freelist.bound = 0;
3918 domain->qed_freelist.offset = 0;
3920 /* Return DQED entries back to the function */
3921 ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
3922 domain->dqed_freelist.base,
3923 (domain->dqed_freelist.bound -
3924 domain->dqed_freelist.base));
3927 "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
3932 domain->dqed_freelist.base = 0;
3933 domain->dqed_freelist.bound = 0;
3934 domain->dqed_freelist.offset = 0;
3936 /* Return AQED entries back to the function */
3937 ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
3938 domain->aqed_freelist.base,
3939 (domain->aqed_freelist.bound -
3940 domain->aqed_freelist.base));
3943 "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
3948 domain->aqed_freelist.base = 0;
3949 domain->aqed_freelist.bound = 0;
3950 domain->aqed_freelist.offset = 0;
3952 /* Return ldb credit pools back to the function's avail list */
3953 list = &domain->used_ldb_credit_pools;
3954 DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
3955 pool->owned = false;
3956 pool->configured = false;
3958 dlb_list_del(&domain->used_ldb_credit_pools,
3959 &pool->domain_list);
3960 dlb_list_add(&rsrcs->avail_ldb_credit_pools,
3962 rsrcs->num_avail_ldb_credit_pools++;
3965 list = &domain->avail_ldb_credit_pools;
3966 DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
3967 pool->owned = false;
3969 dlb_list_del(&domain->avail_ldb_credit_pools,
3970 &pool->domain_list);
3971 dlb_list_add(&rsrcs->avail_ldb_credit_pools,
3973 rsrcs->num_avail_ldb_credit_pools++;
3976 /* Move dir credit pools back to the function */
3977 list = &domain->used_dir_credit_pools;
3978 DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
3979 pool->owned = false;
3980 pool->configured = false;
3982 dlb_list_del(&domain->used_dir_credit_pools,
3983 &pool->domain_list);
3984 dlb_list_add(&rsrcs->avail_dir_credit_pools,
3986 rsrcs->num_avail_dir_credit_pools++;
3989 list = &domain->avail_dir_credit_pools;
3990 DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
3991 pool->owned = false;
3993 dlb_list_del(&domain->avail_dir_credit_pools,
3994 &pool->domain_list);
3995 dlb_list_add(&rsrcs->avail_dir_credit_pools,
3997 rsrcs->num_avail_dir_credit_pools++;
4000 domain->num_pending_removals = 0;
4001 domain->num_pending_additions = 0;
4002 domain->configured = false;
4003 domain->started = false;
4005 /* Move the domain out of the used_domains list and back to the
4006 * function's avail_domains list.
4008 dlb_list_del(&rsrcs->used_domains, &domain->func_list);
4009 dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
4010 rsrcs->num_avail_domains++;
4015 static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
4017 DLB_HW_INFO(hw, "DLB reset domain:\n");
4018 DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
4022 * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
4023 * hardware resources.
4024 * @hw: Contains the current state of the DLB hardware.
4025 * @args: User-provided arguments.
4026 * @resp: Response to user.
4028 * Note: User software *must* stop sending to this domain's producer ports
4029 * before invoking this function, otherwise undefined behavior will result.
4031 * Return: returns < 0 on error, 0 otherwise.
4033 int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
4035 struct dlb_domain *domain;
4038 dlb_log_reset_domain(hw, domain_id);
4040 domain = dlb_get_domain_from_id(hw, domain_id);
4042 if (domain == NULL || !domain->configured)
4045 /* For each queue owned by this domain, disable its write permissions to
4046 * cause any traffic sent to it to be dropped. Well-behaved software
4047 * should not be sending QEs at this point.
4049 dlb_domain_disable_dir_queue_write_perms(hw, domain);
4051 dlb_domain_disable_ldb_queue_write_perms(hw, domain);
4053 /* Disable credit updates and turn off completion tracking on all the
4056 dlb_domain_disable_dir_port_crd_updates(hw, domain);
4058 dlb_domain_disable_ldb_port_crd_updates(hw, domain);
4060 dlb_domain_disable_dir_port_interrupts(hw, domain);
4062 dlb_domain_disable_ldb_port_interrupts(hw, domain);
4064 dlb_domain_disable_ldb_seq_checks(hw, domain);
4066 /* Disable the LDB CQs and drain them in order to complete the map and
4067 * unmap procedures, which require zero CQ inflights and zero QID
4068 * inflights respectively.
4070 dlb_domain_disable_ldb_cqs(hw, domain);
4072 ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
4076 ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
4080 ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
4084 ret = dlb_domain_finish_map_qid_procedures(hw, domain);
4088 /* Re-enable the CQs in order to drain the mapped queues. */
4089 dlb_domain_enable_ldb_cqs(hw, domain);
4091 ret = dlb_domain_drain_mapped_queues(hw, domain);
4095 ret = dlb_domain_drain_unmapped_queues(hw, domain);
4099 ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
4102 "[%s()] Internal error: LDB credits failed to refill\n",
4107 /* Done draining LDB QEs, so disable the CQs. */
4108 dlb_domain_disable_ldb_cqs(hw, domain);
4110 /* Directed queues are reset in dlb_domain_reset_hw_resources(), but
4111 * that process does not decrement the directed queue size counters used
4112 * by SMON for its average DQED depth measurement. So, we manually drain
4113 * the directed queues here.
4115 dlb_domain_drain_dir_queues(hw, domain);
4117 ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
4120 "[%s()] Internal error: DIR credits failed to refill\n",
4125 /* Done draining DIR QEs, so disable the CQs. */
4126 dlb_domain_disable_dir_cqs(hw, domain);
4128 dlb_domain_disable_dir_producer_ports(hw, domain);
4130 dlb_domain_disable_ldb_producer_ports(hw, domain);
4132 dlb_domain_disable_dir_pools(hw, domain);
4134 dlb_domain_disable_ldb_pools(hw, domain);
4136 /* Reset the QID, credit pool, and CQ hardware.
4138 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
4140 * A spurious interrupt can occur on subsequent use of a reset CQ.
4142 ret = dlb_domain_reset_hw_resources(hw, domain);
4146 ret = dlb_domain_verify_reset_success(hw, domain);
4150 dlb_domain_reset_registers(hw, domain);
4152 /* Hardware reset complete. Reset the domain's software state */
4153 ret = dlb_domain_reset_software_state(hw, domain);
4160 void dlb_hw_get_num_resources(struct dlb_hw *hw,
4161 struct dlb_get_num_resources_args *arg)
4163 struct dlb_function_resources *rsrcs;
4164 struct dlb_bitmap *map;
4168 arg->num_sched_domains = rsrcs->num_avail_domains;
4170 arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
4172 arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
4174 arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
4176 map = rsrcs->avail_aqed_freelist_entries;
4178 arg->num_atomic_inflights = dlb_bitmap_count(map);
4180 arg->max_contiguous_atomic_inflights =
4181 dlb_bitmap_longest_set_range(map);
4183 map = rsrcs->avail_hist_list_entries;
4185 arg->num_hist_list_entries = dlb_bitmap_count(map);
4187 arg->max_contiguous_hist_list_entries =
4188 dlb_bitmap_longest_set_range(map);
4190 map = rsrcs->avail_qed_freelist_entries;
4192 arg->num_ldb_credits = dlb_bitmap_count(map);
4194 arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
4196 map = rsrcs->avail_dqed_freelist_entries;
4198 arg->num_dir_credits = dlb_bitmap_count(map);
4200 arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
4202 arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
4204 arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
4207 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
4209 union dlb_sys_sys_alarm_int_enable r0;
4211 r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
4213 r0.field.vf_to_pf_isr_pend_error = 0;
4215 DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
4218 static void dlb_configure_ldb_queue(struct dlb_hw *hw,
4219 struct dlb_domain *domain,
4220 struct dlb_ldb_queue *queue,
4221 struct dlb_create_ldb_queue_args *args)
4223 union dlb_sys_ldb_vasqid_v r0 = { {0} };
4224 union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
4225 union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
4226 union dlb_aqed_pipe_fl_lim r3 = { {0} };
4227 union dlb_aqed_pipe_fl_base r4 = { {0} };
4228 union dlb_chp_ord_qid_sn_map r7 = { {0} };
4229 union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
4230 union dlb_sys_ldb_qid_v r11 = { {0} };
4231 union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
4232 union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
4233 union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
4234 union dlb_ro_pipe_qid2grpslt r9 = { {0} };
4235 struct dlb_sn_group *sn_group;
4238 /* QID write permissions are turned on when the domain is started */
4239 r0.field.vasqid_v = 0;
4241 offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
4243 DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
4246 * Unordered QIDs get 4K inflights, ordered get as many as the number
4247 * of sequence numbers.
4249 r1.field.limit = args->num_qid_inflights;
4251 DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
4253 r2.field.limit = queue->aqed_freelist.bound -
4254 queue->aqed_freelist.base;
4256 if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
4257 r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
4260 DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
4262 r3.field.freelist_disable = 0;
4263 r3.field.limit = queue->aqed_freelist.bound - 1;
4265 DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
4267 r4.field.base = queue->aqed_freelist.base;
4269 DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
4271 r5.field.push_ptr = r4.field.base;
4272 r5.field.generation = 1;
4274 DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
4276 r6.field.pop_ptr = r4.field.base;
4277 r6.field.generation = 0;
4279 DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
4282 sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
4283 r7.field.mode = sn_group->mode;
4284 r7.field.slot = queue->sn_slot;
4285 r7.field.grp = sn_group->id;
4287 DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
4290 * This register limits the number of inflight flows a queue can have
4291 * at one time. It has an upper bound of 2048, but can be
4292 * over-subscribed. 512 is chosen so that a single queue doesn't use
4293 * the entire atomic storage, but can use a substantial portion if
4296 r8.field.qid_fid_limit = 512;
4298 DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
4300 r9.field.group = sn_group->id;
4301 r9.field.slot = queue->sn_slot;
4303 DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
4305 r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
4306 r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
4308 DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
4310 r11.field.qid_v = 1;
4312 DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
4315 int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
4317 if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
4320 return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
4323 int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
4324 unsigned int group_id)
4326 if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
4329 return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
4332 static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
4333 unsigned int group_id,
4336 DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
4337 DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
4338 DLB_HW_INFO(hw, "\tValue: %lu\n", val);
4341 int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
4342 unsigned int group_id,
4345 u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
4346 union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
4347 struct dlb_sn_group *group;
4350 if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
4353 group = &hw->rsrcs.sn_groups[group_id];
4355 /* Once the first load-balanced queue using an SN group is configured,
4356 * the group cannot be changed.
4358 if (group->slot_use_bitmap != 0)
4361 for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
4362 if (val == valid_allocations[mode])
4365 if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
4369 group->sequence_numbers_per_queue = val;
4371 r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
4372 r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
4373 r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
4374 r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
4376 DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
4378 dlb_log_set_group_sequence_numbers(hw, group_id, val);
4384 dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
4385 struct dlb_ldb_queue *queue,
4386 struct dlb_create_ldb_queue_args *args)
4391 queue->sn_cfg_valid = false;
4393 if (args->num_sequence_numbers == 0)
4396 for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
4397 struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
4399 if (group->sequence_numbers_per_queue ==
4400 args->num_sequence_numbers &&
4401 !dlb_sn_group_full(group)) {
4402 slot = dlb_sn_group_alloc_slot(group);
4410 "[%s():%d] Internal error: no sequence number slots available\n",
4411 __func__, __LINE__);
4415 queue->sn_cfg_valid = true;
4416 queue->sn_group = i;
4417 queue->sn_slot = slot;
4422 dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
4423 struct dlb_domain *domain,
4424 struct dlb_ldb_queue *queue,
4425 struct dlb_create_ldb_queue_args *args)
4429 ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
4433 /* Attach QID inflights */
4434 queue->num_qid_inflights = args->num_qid_inflights;
4436 /* Attach atomic inflights */
4437 queue->aqed_freelist.base = domain->aqed_freelist.base +
4438 domain->aqed_freelist.offset;
4439 queue->aqed_freelist.bound = queue->aqed_freelist.base +
4440 args->num_atomic_inflights;
4441 domain->aqed_freelist.offset += args->num_atomic_inflights;
4447 dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
4449 struct dlb_create_ldb_queue_args *args,
4450 struct dlb_cmd_response *resp)
4452 struct dlb_freelist *aqed_freelist;
4453 struct dlb_domain *domain;
4456 domain = dlb_get_domain_from_id(hw, domain_id);
4459 resp->status = DLB_ST_INVALID_DOMAIN_ID;
4463 if (!domain->configured) {
4464 resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
4468 if (domain->started) {
4469 resp->status = DLB_ST_DOMAIN_STARTED;
4473 if (dlb_list_empty(&domain->avail_ldb_queues)) {
4474 resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
4478 if (args->num_sequence_numbers) {
4479 for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
4480 struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
4482 if (group->sequence_numbers_per_queue ==
4483 args->num_sequence_numbers &&
4484 !dlb_sn_group_full(group))
4488 if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
4489 resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
4494 if (args->num_qid_inflights > 4096) {
4495 resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
4499 /* Inflights must be <= number of sequence numbers if ordered */
4500 if (args->num_sequence_numbers != 0 &&
4501 args->num_qid_inflights > args->num_sequence_numbers) {
4502 resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
4506 aqed_freelist = &domain->aqed_freelist;
4508 if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
4509 resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
4517 dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
4519 struct dlb_create_ldb_queue_args *args)
4521 DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
4522 DLB_HW_INFO(hw, "\tDomain ID: %d\n",
4524 DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
4525 args->num_sequence_numbers);
4526 DLB_HW_INFO(hw, "\tNumber of QID inflights: %d\n",
4527 args->num_qid_inflights);
4528 DLB_HW_INFO(hw, "\tNumber of ATM inflights: %d\n",
4529 args->num_atomic_inflights);
4533 * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
4534 * @hw: Contains the current state of the DLB hardware.
4535 * @args: User-provided arguments.
4536 * @resp: Response to user.
4538 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
4539 * satisfy a request, resp->status will be set accordingly.
4541 int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
4543 struct dlb_create_ldb_queue_args *args,
4544 struct dlb_cmd_response *resp)
4546 struct dlb_ldb_queue *queue;
4547 struct dlb_domain *domain;
4550 dlb_log_create_ldb_queue_args(hw, domain_id, args);
4552 /* Verify that hardware resources are available before attempting to
4553 * satisfy the request. This simplifies the error unwinding code.
4555 /* At least one available queue */
4556 if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
4559 domain = dlb_get_domain_from_id(hw, domain_id);
4562 "[%s():%d] Internal error: domain not found\n",
4563 __func__, __LINE__);
4567 queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
4569 /* Verification should catch this. */
4572 "[%s():%d] Internal error: no available ldb queues\n",
4573 __func__, __LINE__);
4577 ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
4580 "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
4581 __func__, __LINE__);
4585 dlb_configure_ldb_queue(hw, domain, queue, args);
4587 queue->num_mappings = 0;
4589 queue->configured = true;
4591 /* Configuration succeeded, so move the resource from the 'avail' to
4594 dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
4596 dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
4599 resp->id = queue->id;