1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
7 #include "dlb2_hw_types.h"
8 #include "dlb2_osdep.h"
9 #include "dlb2_osdep_bitmap.h"
10 #include "dlb2_osdep_types.h"
11 #include "dlb2_regs.h"
12 #include "dlb2_resource.h"
14 #include "../../dlb2_priv.h"
15 #include "../../dlb2_inline_fns.h"
17 #define DLB2_DOM_LIST_HEAD(head, type) \
18 DLB2_LIST_HEAD((head), type, domain_list)
20 #define DLB2_FUNC_LIST_HEAD(head, type) \
21 DLB2_LIST_HEAD((head), type, func_list)
23 #define DLB2_DOM_LIST_FOR(head, ptr, iter) \
24 DLB2_LIST_FOR_EACH(head, ptr, domain_list, iter)
26 #define DLB2_FUNC_LIST_FOR(head, ptr, iter) \
27 DLB2_LIST_FOR_EACH(head, ptr, func_list, iter)
29 #define DLB2_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
30 DLB2_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
32 #define DLB2_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
33 DLB2_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
35 void dlb2_hw_enable_sparse_dir_cq_mode(struct dlb2_hw *hw)
37 union dlb2_chp_cfg_chp_csr_ctrl r0;
39 r0.val = DLB2_CSR_RD(hw, DLB2_CHP_CFG_CHP_CSR_CTRL);
41 r0.field.cfg_64bytes_qe_dir_cq_mode = 1;
43 DLB2_CSR_WR(hw, DLB2_CHP_CFG_CHP_CSR_CTRL, r0.val);
46 void dlb2_hw_enable_sparse_ldb_cq_mode(struct dlb2_hw *hw)
48 union dlb2_chp_cfg_chp_csr_ctrl r0;
50 r0.val = DLB2_CSR_RD(hw, DLB2_CHP_CFG_CHP_CSR_CTRL);
52 r0.field.cfg_64bytes_qe_ldb_cq_mode = 1;
54 DLB2_CSR_WR(hw, DLB2_CHP_CFG_CHP_CSR_CTRL, r0.val);
58 * The PF driver cannot assume that a register write will affect subsequent HCW
59 * writes. To ensure a write completes, the driver must read back a CSR. This
60 * function only need be called for configuration that can occur after the
61 * domain has started; prior to starting, applications can't send HCWs.
63 static inline void dlb2_flush_csr(struct dlb2_hw *hw)
65 DLB2_CSR_RD(hw, DLB2_SYS_TOTAL_VAS);
68 static void dlb2_dir_port_cq_enable(struct dlb2_hw *hw,
69 struct dlb2_dir_pq_pair *port)
71 union dlb2_lsp_cq_dir_dsbl reg;
73 reg.field.disabled = 0;
75 DLB2_CSR_WR(hw, DLB2_LSP_CQ_DIR_DSBL(port->id.phys_id), reg.val);
80 static u32 dlb2_dir_queue_depth(struct dlb2_hw *hw,
81 struct dlb2_dir_pq_pair *queue)
83 union dlb2_lsp_qid_dir_enqueue_cnt r0;
85 r0.val = DLB2_CSR_RD(hw,
86 DLB2_LSP_QID_DIR_ENQUEUE_CNT(queue->id.phys_id));
88 return r0.field.count;
91 static void dlb2_ldb_port_cq_enable(struct dlb2_hw *hw,
92 struct dlb2_ldb_port *port)
94 union dlb2_lsp_cq_ldb_dsbl reg;
97 * Don't re-enable the port if a removal is pending. The caller should
98 * mark this port as enabled (if it isn't already), and when the
99 * removal completes the port will be enabled.
101 if (port->num_pending_removals)
104 reg.field.disabled = 0;
106 DLB2_CSR_WR(hw, DLB2_LSP_CQ_LDB_DSBL(port->id.phys_id), reg.val);
111 static void dlb2_ldb_port_cq_disable(struct dlb2_hw *hw,
112 struct dlb2_ldb_port *port)
114 union dlb2_lsp_cq_ldb_dsbl reg;
116 reg.field.disabled = 1;
118 DLB2_CSR_WR(hw, DLB2_LSP_CQ_LDB_DSBL(port->id.phys_id), reg.val);
123 static u32 dlb2_ldb_queue_depth(struct dlb2_hw *hw,
124 struct dlb2_ldb_queue *queue)
126 union dlb2_lsp_qid_aqed_active_cnt r0;
127 union dlb2_lsp_qid_atm_active r1;
128 union dlb2_lsp_qid_ldb_enqueue_cnt r2;
130 r0.val = DLB2_CSR_RD(hw,
131 DLB2_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
132 r1.val = DLB2_CSR_RD(hw,
133 DLB2_LSP_QID_ATM_ACTIVE(queue->id.phys_id));
135 r2.val = DLB2_CSR_RD(hw,
136 DLB2_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
138 return r0.field.count + r1.field.count + r2.field.count;
141 static struct dlb2_ldb_queue *
142 dlb2_get_ldb_queue_from_id(struct dlb2_hw *hw,
145 unsigned int vdev_id)
147 struct dlb2_list_entry *iter1;
148 struct dlb2_list_entry *iter2;
149 struct dlb2_function_resources *rsrcs;
150 struct dlb2_hw_domain *domain;
151 struct dlb2_ldb_queue *queue;
155 if (id >= DLB2_MAX_NUM_LDB_QUEUES)
158 rsrcs = (vdev_req) ? &hw->vdev[vdev_id] : &hw->pf;
161 return &hw->rsrcs.ldb_queues[id];
163 DLB2_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
164 DLB2_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter2)
165 if (queue->id.virt_id == id)
169 DLB2_FUNC_LIST_FOR(rsrcs->avail_ldb_queues, queue, iter1)
170 if (queue->id.virt_id == id)
176 static struct dlb2_hw_domain *dlb2_get_domain_from_id(struct dlb2_hw *hw,
179 unsigned int vdev_id)
181 struct dlb2_list_entry *iteration;
182 struct dlb2_function_resources *rsrcs;
183 struct dlb2_hw_domain *domain;
184 RTE_SET_USED(iteration);
186 if (id >= DLB2_MAX_NUM_DOMAINS)
190 return &hw->domains[id];
192 rsrcs = &hw->vdev[vdev_id];
194 DLB2_FUNC_LIST_FOR(rsrcs->used_domains, domain, iteration)
195 if (domain->id.virt_id == id)
201 static int dlb2_port_slot_state_transition(struct dlb2_hw *hw,
202 struct dlb2_ldb_port *port,
203 struct dlb2_ldb_queue *queue,
205 enum dlb2_qid_map_state new_state)
207 enum dlb2_qid_map_state curr_state = port->qid_map[slot].state;
208 struct dlb2_hw_domain *domain;
211 domain_id = port->domain_id.phys_id;
213 domain = dlb2_get_domain_from_id(hw, domain_id, false, 0);
214 if (domain == NULL) {
216 "[%s()] Internal error: unable to find domain %d\n",
217 __func__, domain_id);
221 switch (curr_state) {
222 case DLB2_QUEUE_UNMAPPED:
224 case DLB2_QUEUE_MAPPED:
225 queue->num_mappings++;
226 port->num_mappings++;
228 case DLB2_QUEUE_MAP_IN_PROG:
229 queue->num_pending_additions++;
230 domain->num_pending_additions++;
236 case DLB2_QUEUE_MAPPED:
238 case DLB2_QUEUE_UNMAPPED:
239 queue->num_mappings--;
240 port->num_mappings--;
242 case DLB2_QUEUE_UNMAP_IN_PROG:
243 port->num_pending_removals++;
244 domain->num_pending_removals++;
246 case DLB2_QUEUE_MAPPED:
247 /* Priority change, nothing to update */
253 case DLB2_QUEUE_MAP_IN_PROG:
255 case DLB2_QUEUE_UNMAPPED:
256 queue->num_pending_additions--;
257 domain->num_pending_additions--;
259 case DLB2_QUEUE_MAPPED:
260 queue->num_mappings++;
261 port->num_mappings++;
262 queue->num_pending_additions--;
263 domain->num_pending_additions--;
269 case DLB2_QUEUE_UNMAP_IN_PROG:
271 case DLB2_QUEUE_UNMAPPED:
272 port->num_pending_removals--;
273 domain->num_pending_removals--;
274 queue->num_mappings--;
275 port->num_mappings--;
277 case DLB2_QUEUE_MAPPED:
278 port->num_pending_removals--;
279 domain->num_pending_removals--;
281 case DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP:
282 /* Nothing to update */
288 case DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP:
290 case DLB2_QUEUE_UNMAP_IN_PROG:
291 /* Nothing to update */
293 case DLB2_QUEUE_UNMAPPED:
295 * An UNMAP_IN_PROG_PENDING_MAP slot briefly
296 * becomes UNMAPPED before it transitions to
299 queue->num_mappings--;
300 port->num_mappings--;
301 port->num_pending_removals--;
302 domain->num_pending_removals--;
312 port->qid_map[slot].state = new_state;
315 "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
316 __func__, queue->id.phys_id, port->id.phys_id,
317 curr_state, new_state);
322 "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
323 __func__, queue->id.phys_id, port->id.phys_id,
324 curr_state, new_state);
328 static bool dlb2_port_find_slot(struct dlb2_ldb_port *port,
329 enum dlb2_qid_map_state state,
334 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
335 if (port->qid_map[i].state == state)
341 return (i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ);
344 static bool dlb2_port_find_slot_queue(struct dlb2_ldb_port *port,
345 enum dlb2_qid_map_state state,
346 struct dlb2_ldb_queue *queue,
351 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
352 if (port->qid_map[i].state == state &&
353 port->qid_map[i].qid == queue->id.phys_id)
359 return (i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ);
363 * dlb2_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as
364 * their function names imply, and should only be called by the dynamic CQ
367 static void dlb2_ldb_queue_disable_mapped_cqs(struct dlb2_hw *hw,
368 struct dlb2_hw_domain *domain,
369 struct dlb2_ldb_queue *queue)
371 struct dlb2_list_entry *iter;
372 struct dlb2_ldb_port *port;
376 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
377 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter) {
378 enum dlb2_qid_map_state state = DLB2_QUEUE_MAPPED;
380 if (!dlb2_port_find_slot_queue(port, state,
385 dlb2_ldb_port_cq_disable(hw, port);
390 static void dlb2_ldb_queue_enable_mapped_cqs(struct dlb2_hw *hw,
391 struct dlb2_hw_domain *domain,
392 struct dlb2_ldb_queue *queue)
394 struct dlb2_list_entry *iter;
395 struct dlb2_ldb_port *port;
399 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
400 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter) {
401 enum dlb2_qid_map_state state = DLB2_QUEUE_MAPPED;
403 if (!dlb2_port_find_slot_queue(port, state,
408 dlb2_ldb_port_cq_enable(hw, port);
413 static void dlb2_ldb_port_clear_queue_if_status(struct dlb2_hw *hw,
414 struct dlb2_ldb_port *port,
417 union dlb2_lsp_ldb_sched_ctrl r0 = { {0} };
419 r0.field.cq = port->id.phys_id;
420 r0.field.qidix = slot;
422 r0.field.inflight_ok_v = 1;
424 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r0.val);
429 static void dlb2_ldb_port_set_queue_if_status(struct dlb2_hw *hw,
430 struct dlb2_ldb_port *port,
433 union dlb2_lsp_ldb_sched_ctrl r0 = { {0} };
435 r0.field.cq = port->id.phys_id;
436 r0.field.qidix = slot;
438 r0.field.inflight_ok_v = 1;
440 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r0.val);
445 static int dlb2_ldb_port_map_qid_static(struct dlb2_hw *hw,
446 struct dlb2_ldb_port *p,
447 struct dlb2_ldb_queue *q,
450 union dlb2_lsp_cq2priov r0;
451 union dlb2_lsp_cq2qid0 r1;
452 union dlb2_atm_qid2cqidix_00 r2;
453 union dlb2_lsp_qid2cqidix_00 r3;
454 union dlb2_lsp_qid2cqidix2_00 r4;
455 enum dlb2_qid_map_state state;
458 /* Look for a pending or already mapped slot, else an unused slot */
459 if (!dlb2_port_find_slot_queue(p, DLB2_QUEUE_MAP_IN_PROG, q, &i) &&
460 !dlb2_port_find_slot_queue(p, DLB2_QUEUE_MAPPED, q, &i) &&
461 !dlb2_port_find_slot(p, DLB2_QUEUE_UNMAPPED, &i)) {
463 "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
468 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
470 "[%s():%d] Internal error: port slot tracking failed\n",
475 /* Read-modify-write the priority and valid bit register */
476 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2PRIOV(p->id.phys_id));
478 r0.field.v |= 1 << i;
479 r0.field.prio |= (priority & 0x7) << i * 3;
481 DLB2_CSR_WR(hw, DLB2_LSP_CQ2PRIOV(p->id.phys_id), r0.val);
483 /* Read-modify-write the QID map register */
485 r1.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2QID0(p->id.phys_id));
487 r1.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2QID1(p->id.phys_id));
489 if (i == 0 || i == 4)
490 r1.field.qid_p0 = q->id.phys_id;
491 if (i == 1 || i == 5)
492 r1.field.qid_p1 = q->id.phys_id;
493 if (i == 2 || i == 6)
494 r1.field.qid_p2 = q->id.phys_id;
495 if (i == 3 || i == 7)
496 r1.field.qid_p3 = q->id.phys_id;
499 DLB2_CSR_WR(hw, DLB2_LSP_CQ2QID0(p->id.phys_id), r1.val);
501 DLB2_CSR_WR(hw, DLB2_LSP_CQ2QID1(p->id.phys_id), r1.val);
503 r2.val = DLB2_CSR_RD(hw,
504 DLB2_ATM_QID2CQIDIX(q->id.phys_id,
507 r3.val = DLB2_CSR_RD(hw,
508 DLB2_LSP_QID2CQIDIX(q->id.phys_id,
511 r4.val = DLB2_CSR_RD(hw,
512 DLB2_LSP_QID2CQIDIX2(q->id.phys_id,
515 switch (p->id.phys_id % 4) {
517 r2.field.cq_p0 |= 1 << i;
518 r3.field.cq_p0 |= 1 << i;
519 r4.field.cq_p0 |= 1 << i;
523 r2.field.cq_p1 |= 1 << i;
524 r3.field.cq_p1 |= 1 << i;
525 r4.field.cq_p1 |= 1 << i;
529 r2.field.cq_p2 |= 1 << i;
530 r3.field.cq_p2 |= 1 << i;
531 r4.field.cq_p2 |= 1 << i;
535 r2.field.cq_p3 |= 1 << i;
536 r3.field.cq_p3 |= 1 << i;
537 r4.field.cq_p3 |= 1 << i;
542 DLB2_ATM_QID2CQIDIX(q->id.phys_id, p->id.phys_id / 4),
546 DLB2_LSP_QID2CQIDIX(q->id.phys_id, p->id.phys_id / 4),
550 DLB2_LSP_QID2CQIDIX2(q->id.phys_id, p->id.phys_id / 4),
555 p->qid_map[i].qid = q->id.phys_id;
556 p->qid_map[i].priority = priority;
558 state = DLB2_QUEUE_MAPPED;
560 return dlb2_port_slot_state_transition(hw, p, q, i, state);
563 static int dlb2_ldb_port_set_has_work_bits(struct dlb2_hw *hw,
564 struct dlb2_ldb_port *port,
565 struct dlb2_ldb_queue *queue,
568 union dlb2_lsp_qid_aqed_active_cnt r0;
569 union dlb2_lsp_qid_ldb_enqueue_cnt r1;
570 union dlb2_lsp_ldb_sched_ctrl r2 = { {0} };
572 /* Set the atomic scheduling haswork bit */
573 r0.val = DLB2_CSR_RD(hw,
574 DLB2_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
576 r2.field.cq = port->id.phys_id;
577 r2.field.qidix = slot;
579 r2.field.rlist_haswork_v = r0.field.count > 0;
581 /* Set the non-atomic scheduling haswork bit */
582 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r2.val);
584 r1.val = DLB2_CSR_RD(hw,
585 DLB2_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
587 memset(&r2, 0, sizeof(r2));
589 r2.field.cq = port->id.phys_id;
590 r2.field.qidix = slot;
592 r2.field.nalb_haswork_v = (r1.field.count > 0);
594 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r2.val);
601 static void dlb2_ldb_port_clear_has_work_bits(struct dlb2_hw *hw,
602 struct dlb2_ldb_port *port,
605 union dlb2_lsp_ldb_sched_ctrl r2 = { {0} };
607 r2.field.cq = port->id.phys_id;
608 r2.field.qidix = slot;
610 r2.field.rlist_haswork_v = 1;
612 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r2.val);
614 memset(&r2, 0, sizeof(r2));
616 r2.field.cq = port->id.phys_id;
617 r2.field.qidix = slot;
619 r2.field.nalb_haswork_v = 1;
621 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r2.val);
626 static void dlb2_ldb_queue_set_inflight_limit(struct dlb2_hw *hw,
627 struct dlb2_ldb_queue *queue)
629 union dlb2_lsp_qid_ldb_infl_lim r0 = { {0} };
631 r0.field.limit = queue->num_qid_inflights;
633 DLB2_CSR_WR(hw, DLB2_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), r0.val);
636 static void dlb2_ldb_queue_clear_inflight_limit(struct dlb2_hw *hw,
637 struct dlb2_ldb_queue *queue)
640 DLB2_LSP_QID_LDB_INFL_LIM(queue->id.phys_id),
641 DLB2_LSP_QID_LDB_INFL_LIM_RST);
644 static int dlb2_ldb_port_finish_map_qid_dynamic(struct dlb2_hw *hw,
645 struct dlb2_hw_domain *domain,
646 struct dlb2_ldb_port *port,
647 struct dlb2_ldb_queue *queue)
649 struct dlb2_list_entry *iter;
650 union dlb2_lsp_qid_ldb_infl_cnt r0;
651 enum dlb2_qid_map_state state;
656 r0.val = DLB2_CSR_RD(hw,
657 DLB2_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
659 if (r0.field.count) {
661 "[%s()] Internal error: non-zero QID inflight count\n",
667 * Static map the port and set its corresponding has_work bits.
669 state = DLB2_QUEUE_MAP_IN_PROG;
670 if (!dlb2_port_find_slot_queue(port, state, queue, &slot))
673 if (slot >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
675 "[%s():%d] Internal error: port slot tracking failed\n",
680 prio = port->qid_map[slot].priority;
683 * Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
684 * the port's qid_map state.
686 ret = dlb2_ldb_port_map_qid_static(hw, port, queue, prio);
690 ret = dlb2_ldb_port_set_has_work_bits(hw, port, queue, slot);
695 * Ensure IF_status(cq,qid) is 0 before enabling the port to
696 * prevent spurious schedules to cause the queue's inflight
699 dlb2_ldb_port_clear_queue_if_status(hw, port, slot);
701 /* Reset the queue's inflight status */
702 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
703 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter) {
704 state = DLB2_QUEUE_MAPPED;
705 if (!dlb2_port_find_slot_queue(port, state,
709 dlb2_ldb_port_set_queue_if_status(hw, port, slot);
713 dlb2_ldb_queue_set_inflight_limit(hw, queue);
715 /* Re-enable CQs mapped to this queue */
716 dlb2_ldb_queue_enable_mapped_cqs(hw, domain, queue);
718 /* If this queue has other mappings pending, clear its inflight limit */
719 if (queue->num_pending_additions > 0)
720 dlb2_ldb_queue_clear_inflight_limit(hw, queue);
726 * dlb2_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
727 * @hw: dlb2_hw handle for a particular device.
728 * @port: load-balanced port
729 * @queue: load-balanced queue
730 * @priority: queue servicing priority
732 * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
733 * at a later point, and <0 if an error occurred.
735 static int dlb2_ldb_port_map_qid_dynamic(struct dlb2_hw *hw,
736 struct dlb2_ldb_port *port,
737 struct dlb2_ldb_queue *queue,
740 union dlb2_lsp_qid_ldb_infl_cnt r0 = { {0} };
741 enum dlb2_qid_map_state state;
742 struct dlb2_hw_domain *domain;
743 int domain_id, slot, ret;
745 domain_id = port->domain_id.phys_id;
747 domain = dlb2_get_domain_from_id(hw, domain_id, false, 0);
748 if (domain == NULL) {
750 "[%s()] Internal error: unable to find domain %d\n",
751 __func__, port->domain_id.phys_id);
756 * Set the QID inflight limit to 0 to prevent further scheduling of the
759 DLB2_CSR_WR(hw, DLB2_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), 0);
761 if (!dlb2_port_find_slot(port, DLB2_QUEUE_UNMAPPED, &slot)) {
763 "Internal error: No available unmapped slots\n");
767 if (slot >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
769 "[%s():%d] Internal error: port slot tracking failed\n",
774 port->qid_map[slot].qid = queue->id.phys_id;
775 port->qid_map[slot].priority = priority;
777 state = DLB2_QUEUE_MAP_IN_PROG;
778 ret = dlb2_port_slot_state_transition(hw, port, queue, slot, state);
782 r0.val = DLB2_CSR_RD(hw,
783 DLB2_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
785 if (r0.field.count) {
787 * The queue is owed completions so it's not safe to map it
788 * yet. Schedule a kernel thread to complete the mapping later,
789 * once software has completed all the queue's inflight events.
791 if (!os_worker_active(hw))
792 os_schedule_work(hw);
798 * Disable the affected CQ, and the CQs already mapped to the QID,
799 * before reading the QID's inflight count a second time. There is an
800 * unlikely race in which the QID may schedule one more QE after we
801 * read an inflight count of 0, and disabling the CQs guarantees that
802 * the race will not occur after a re-read of the inflight count
806 dlb2_ldb_port_cq_disable(hw, port);
808 dlb2_ldb_queue_disable_mapped_cqs(hw, domain, queue);
810 r0.val = DLB2_CSR_RD(hw,
811 DLB2_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
813 if (r0.field.count) {
815 dlb2_ldb_port_cq_enable(hw, port);
817 dlb2_ldb_queue_enable_mapped_cqs(hw, domain, queue);
820 * The queue is owed completions so it's not safe to map it
821 * yet. Schedule a kernel thread to complete the mapping later,
822 * once software has completed all the queue's inflight events.
824 if (!os_worker_active(hw))
825 os_schedule_work(hw);
830 return dlb2_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
833 static void dlb2_domain_finish_map_port(struct dlb2_hw *hw,
834 struct dlb2_hw_domain *domain,
835 struct dlb2_ldb_port *port)
839 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
840 union dlb2_lsp_qid_ldb_infl_cnt r0;
841 struct dlb2_ldb_queue *queue;
844 if (port->qid_map[i].state != DLB2_QUEUE_MAP_IN_PROG)
847 qid = port->qid_map[i].qid;
849 queue = dlb2_get_ldb_queue_from_id(hw, qid, false, 0);
853 "[%s()] Internal error: unable to find queue %d\n",
858 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_QID_LDB_INFL_CNT(qid));
864 * Disable the affected CQ, and the CQs already mapped to the
865 * QID, before reading the QID's inflight count a second time.
866 * There is an unlikely race in which the QID may schedule one
867 * more QE after we read an inflight count of 0, and disabling
868 * the CQs guarantees that the race will not occur after a
869 * re-read of the inflight count register.
872 dlb2_ldb_port_cq_disable(hw, port);
874 dlb2_ldb_queue_disable_mapped_cqs(hw, domain, queue);
876 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_QID_LDB_INFL_CNT(qid));
878 if (r0.field.count) {
880 dlb2_ldb_port_cq_enable(hw, port);
882 dlb2_ldb_queue_enable_mapped_cqs(hw, domain, queue);
887 dlb2_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
892 dlb2_domain_finish_map_qid_procedures(struct dlb2_hw *hw,
893 struct dlb2_hw_domain *domain)
895 struct dlb2_list_entry *iter;
896 struct dlb2_ldb_port *port;
900 if (!domain->configured || domain->num_pending_additions == 0)
903 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
904 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter)
905 dlb2_domain_finish_map_port(hw, domain, port);
908 return domain->num_pending_additions;
911 static int dlb2_ldb_port_unmap_qid(struct dlb2_hw *hw,
912 struct dlb2_ldb_port *port,
913 struct dlb2_ldb_queue *queue)
915 enum dlb2_qid_map_state mapped, in_progress, pending_map, unmapped;
916 union dlb2_lsp_cq2priov r0;
917 union dlb2_atm_qid2cqidix_00 r1;
918 union dlb2_lsp_qid2cqidix_00 r2;
919 union dlb2_lsp_qid2cqidix2_00 r3;
924 /* Find the queue's slot */
925 mapped = DLB2_QUEUE_MAPPED;
926 in_progress = DLB2_QUEUE_UNMAP_IN_PROG;
927 pending_map = DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP;
929 if (!dlb2_port_find_slot_queue(port, mapped, queue, &i) &&
930 !dlb2_port_find_slot_queue(port, in_progress, queue, &i) &&
931 !dlb2_port_find_slot_queue(port, pending_map, queue, &i)) {
933 "[%s():%d] Internal error: QID %d isn't mapped\n",
934 __func__, __LINE__, queue->id.phys_id);
938 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
940 "[%s():%d] Internal error: port slot tracking failed\n",
945 port_id = port->id.phys_id;
946 queue_id = queue->id.phys_id;
948 /* Read-modify-write the priority and valid bit register */
949 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2PRIOV(port_id));
951 r0.field.v &= ~(1 << i);
953 DLB2_CSR_WR(hw, DLB2_LSP_CQ2PRIOV(port_id), r0.val);
955 r1.val = DLB2_CSR_RD(hw,
956 DLB2_ATM_QID2CQIDIX(queue_id, port_id / 4));
958 r2.val = DLB2_CSR_RD(hw,
959 DLB2_LSP_QID2CQIDIX(queue_id, port_id / 4));
961 r3.val = DLB2_CSR_RD(hw,
962 DLB2_LSP_QID2CQIDIX2(queue_id, port_id / 4));
964 switch (port_id % 4) {
966 r1.field.cq_p0 &= ~(1 << i);
967 r2.field.cq_p0 &= ~(1 << i);
968 r3.field.cq_p0 &= ~(1 << i);
972 r1.field.cq_p1 &= ~(1 << i);
973 r2.field.cq_p1 &= ~(1 << i);
974 r3.field.cq_p1 &= ~(1 << i);
978 r1.field.cq_p2 &= ~(1 << i);
979 r2.field.cq_p2 &= ~(1 << i);
980 r3.field.cq_p2 &= ~(1 << i);
984 r1.field.cq_p3 &= ~(1 << i);
985 r2.field.cq_p3 &= ~(1 << i);
986 r3.field.cq_p3 &= ~(1 << i);
991 DLB2_ATM_QID2CQIDIX(queue_id, port_id / 4),
995 DLB2_LSP_QID2CQIDIX(queue_id, port_id / 4),
999 DLB2_LSP_QID2CQIDIX2(queue_id, port_id / 4),
1004 unmapped = DLB2_QUEUE_UNMAPPED;
1006 return dlb2_port_slot_state_transition(hw, port, queue, i, unmapped);
1009 static int dlb2_ldb_port_map_qid(struct dlb2_hw *hw,
1010 struct dlb2_hw_domain *domain,
1011 struct dlb2_ldb_port *port,
1012 struct dlb2_ldb_queue *queue,
1015 if (domain->started)
1016 return dlb2_ldb_port_map_qid_dynamic(hw, port, queue, prio);
1018 return dlb2_ldb_port_map_qid_static(hw, port, queue, prio);
1022 dlb2_domain_finish_unmap_port_slot(struct dlb2_hw *hw,
1023 struct dlb2_hw_domain *domain,
1024 struct dlb2_ldb_port *port,
1027 enum dlb2_qid_map_state state;
1028 struct dlb2_ldb_queue *queue;
1030 queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
1032 state = port->qid_map[slot].state;
1034 /* Update the QID2CQIDX and CQ2QID vectors */
1035 dlb2_ldb_port_unmap_qid(hw, port, queue);
1038 * Ensure the QID will not be serviced by this {CQ, slot} by clearing
1041 dlb2_ldb_port_clear_has_work_bits(hw, port, slot);
1043 /* Reset the {CQ, slot} to its default state */
1044 dlb2_ldb_port_set_queue_if_status(hw, port, slot);
1046 /* Re-enable the CQ if it wasn't manually disabled by the user */
1048 dlb2_ldb_port_cq_enable(hw, port);
1051 * If there is a mapping that is pending this slot's removal, perform
1054 if (state == DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP) {
1055 struct dlb2_ldb_port_qid_map *map;
1056 struct dlb2_ldb_queue *map_queue;
1059 map = &port->qid_map[slot];
1061 map->qid = map->pending_qid;
1062 map->priority = map->pending_priority;
1064 map_queue = &hw->rsrcs.ldb_queues[map->qid];
1065 prio = map->priority;
1067 dlb2_ldb_port_map_qid(hw, domain, port, map_queue, prio);
1071 static bool dlb2_domain_finish_unmap_port(struct dlb2_hw *hw,
1072 struct dlb2_hw_domain *domain,
1073 struct dlb2_ldb_port *port)
1075 union dlb2_lsp_cq_ldb_infl_cnt r0;
1078 if (port->num_pending_removals == 0)
1082 * The unmap requires all the CQ's outstanding inflights to be
1085 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ_LDB_INFL_CNT(port->id.phys_id));
1086 if (r0.field.count > 0)
1089 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
1090 struct dlb2_ldb_port_qid_map *map;
1092 map = &port->qid_map[i];
1094 if (map->state != DLB2_QUEUE_UNMAP_IN_PROG &&
1095 map->state != DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP)
1098 dlb2_domain_finish_unmap_port_slot(hw, domain, port, i);
1105 dlb2_domain_finish_unmap_qid_procedures(struct dlb2_hw *hw,
1106 struct dlb2_hw_domain *domain)
1108 struct dlb2_list_entry *iter;
1109 struct dlb2_ldb_port *port;
1113 if (!domain->configured || domain->num_pending_removals == 0)
1116 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
1117 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter)
1118 dlb2_domain_finish_unmap_port(hw, domain, port);
1121 return domain->num_pending_removals;
1124 unsigned int dlb2_finish_unmap_qid_procedures(struct dlb2_hw *hw)
1128 /* Finish queue unmap jobs for any domain that needs it */
1129 for (i = 0; i < DLB2_MAX_NUM_DOMAINS; i++) {
1130 struct dlb2_hw_domain *domain = &hw->domains[i];
1132 num += dlb2_domain_finish_unmap_qid_procedures(hw, domain);
1138 unsigned int dlb2_finish_map_qid_procedures(struct dlb2_hw *hw)
1142 /* Finish queue map jobs for any domain that needs it */
1143 for (i = 0; i < DLB2_MAX_NUM_DOMAINS; i++) {
1144 struct dlb2_hw_domain *domain = &hw->domains[i];
1146 num += dlb2_domain_finish_map_qid_procedures(hw, domain);
1153 static void dlb2_configure_ldb_queue(struct dlb2_hw *hw,
1154 struct dlb2_hw_domain *domain,
1155 struct dlb2_ldb_queue *queue,
1156 struct dlb2_create_ldb_queue_args *args,
1158 unsigned int vdev_id)
1160 union dlb2_sys_vf_ldb_vqid_v r0 = { {0} };
1161 union dlb2_sys_vf_ldb_vqid2qid r1 = { {0} };
1162 union dlb2_sys_ldb_qid2vqid r2 = { {0} };
1163 union dlb2_sys_ldb_vasqid_v r3 = { {0} };
1164 union dlb2_lsp_qid_ldb_infl_lim r4 = { {0} };
1165 union dlb2_lsp_qid_aqed_active_lim r5 = { {0} };
1166 union dlb2_aqed_pipe_qid_hid_width r6 = { {0} };
1167 union dlb2_sys_ldb_qid_its r7 = { {0} };
1168 union dlb2_lsp_qid_atm_depth_thrsh r8 = { {0} };
1169 union dlb2_lsp_qid_naldb_depth_thrsh r9 = { {0} };
1170 union dlb2_aqed_pipe_qid_fid_lim r10 = { {0} };
1171 union dlb2_chp_ord_qid_sn_map r11 = { {0} };
1172 union dlb2_sys_ldb_qid_cfg_v r12 = { {0} };
1173 union dlb2_sys_ldb_qid_v r13 = { {0} };
1175 struct dlb2_sn_group *sn_group;
1178 /* QID write permissions are turned on when the domain is started */
1179 r3.field.vasqid_v = 0;
1181 offs = domain->id.phys_id * DLB2_MAX_NUM_LDB_QUEUES +
1184 DLB2_CSR_WR(hw, DLB2_SYS_LDB_VASQID_V(offs), r3.val);
1187 * Unordered QIDs get 4K inflights, ordered get as many as the number
1188 * of sequence numbers.
1190 r4.field.limit = args->num_qid_inflights;
1192 DLB2_CSR_WR(hw, DLB2_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), r4.val);
1194 r5.field.limit = queue->aqed_limit;
1196 if (r5.field.limit > DLB2_MAX_NUM_AQED_ENTRIES)
1197 r5.field.limit = DLB2_MAX_NUM_AQED_ENTRIES;
1200 DLB2_LSP_QID_AQED_ACTIVE_LIM(queue->id.phys_id),
1203 switch (args->lock_id_comp_level) {
1205 r6.field.compress_code = 1;
1208 r6.field.compress_code = 2;
1211 r6.field.compress_code = 3;
1214 r6.field.compress_code = 4;
1217 r6.field.compress_code = 5;
1220 r6.field.compress_code = 6;
1223 r6.field.compress_code = 7;
1227 r6.field.compress_code = 0;
1231 DLB2_AQED_PIPE_QID_HID_WIDTH(queue->id.phys_id),
1234 /* Don't timestamp QEs that pass through this queue */
1235 r7.field.qid_its = 0;
1238 DLB2_SYS_LDB_QID_ITS(queue->id.phys_id),
1241 r8.field.thresh = args->depth_threshold;
1244 DLB2_LSP_QID_ATM_DEPTH_THRSH(queue->id.phys_id),
1247 r9.field.thresh = args->depth_threshold;
1250 DLB2_LSP_QID_NALDB_DEPTH_THRSH(queue->id.phys_id),
1254 * This register limits the number of inflight flows a queue can have
1255 * at one time. It has an upper bound of 2048, but can be
1256 * over-subscribed. 512 is chosen so that a single queue doesn't use
1257 * the entire atomic storage, but can use a substantial portion if
1260 r10.field.qid_fid_limit = 512;
1263 DLB2_AQED_PIPE_QID_FID_LIM(queue->id.phys_id),
1267 sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
1268 r11.field.mode = sn_group->mode;
1269 r11.field.slot = queue->sn_slot;
1270 r11.field.grp = sn_group->id;
1272 DLB2_CSR_WR(hw, DLB2_CHP_ORD_QID_SN_MAP(queue->id.phys_id), r11.val);
1274 r12.field.sn_cfg_v = (args->num_sequence_numbers != 0);
1275 r12.field.fid_cfg_v = (args->num_atomic_inflights != 0);
1277 DLB2_CSR_WR(hw, DLB2_SYS_LDB_QID_CFG_V(queue->id.phys_id), r12.val);
1280 offs = vdev_id * DLB2_MAX_NUM_LDB_QUEUES + queue->id.virt_id;
1282 r0.field.vqid_v = 1;
1284 DLB2_CSR_WR(hw, DLB2_SYS_VF_LDB_VQID_V(offs), r0.val);
1286 r1.field.qid = queue->id.phys_id;
1288 DLB2_CSR_WR(hw, DLB2_SYS_VF_LDB_VQID2QID(offs), r1.val);
1290 r2.field.vqid = queue->id.virt_id;
1293 DLB2_SYS_LDB_QID2VQID(queue->id.phys_id),
1297 r13.field.qid_v = 1;
1299 DLB2_CSR_WR(hw, DLB2_SYS_LDB_QID_V(queue->id.phys_id), r13.val);
1303 dlb2_ldb_queue_attach_to_sn_group(struct dlb2_hw *hw,
1304 struct dlb2_ldb_queue *queue,
1305 struct dlb2_create_ldb_queue_args *args)
1310 queue->sn_cfg_valid = false;
1312 if (args->num_sequence_numbers == 0)
1315 for (i = 0; i < DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
1316 struct dlb2_sn_group *group = &hw->rsrcs.sn_groups[i];
1318 if (group->sequence_numbers_per_queue ==
1319 args->num_sequence_numbers &&
1320 !dlb2_sn_group_full(group)) {
1321 slot = dlb2_sn_group_alloc_slot(group);
1329 "[%s():%d] Internal error: no sequence number slots available\n",
1330 __func__, __LINE__);
1334 queue->sn_cfg_valid = true;
1335 queue->sn_group = i;
1336 queue->sn_slot = slot;
1341 dlb2_ldb_queue_attach_resources(struct dlb2_hw *hw,
1342 struct dlb2_hw_domain *domain,
1343 struct dlb2_ldb_queue *queue,
1344 struct dlb2_create_ldb_queue_args *args)
1348 ret = dlb2_ldb_queue_attach_to_sn_group(hw, queue, args);
1352 /* Attach QID inflights */
1353 queue->num_qid_inflights = args->num_qid_inflights;
1355 /* Attach atomic inflights */
1356 queue->aqed_limit = args->num_atomic_inflights;
1358 domain->num_avail_aqed_entries -= args->num_atomic_inflights;
1359 domain->num_used_aqed_entries += args->num_atomic_inflights;
1365 dlb2_verify_create_ldb_queue_args(struct dlb2_hw *hw,
1367 struct dlb2_create_ldb_queue_args *args,
1368 struct dlb2_cmd_response *resp,
1370 unsigned int vdev_id)
1372 struct dlb2_hw_domain *domain;
1375 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
1377 if (domain == NULL) {
1378 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
1382 if (!domain->configured) {
1383 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
1387 if (domain->started) {
1388 resp->status = DLB2_ST_DOMAIN_STARTED;
1392 if (dlb2_list_empty(&domain->avail_ldb_queues)) {
1393 resp->status = DLB2_ST_LDB_QUEUES_UNAVAILABLE;
1397 if (args->num_sequence_numbers) {
1398 for (i = 0; i < DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
1399 struct dlb2_sn_group *group = &hw->rsrcs.sn_groups[i];
1401 if (group->sequence_numbers_per_queue ==
1402 args->num_sequence_numbers &&
1403 !dlb2_sn_group_full(group))
1407 if (i == DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
1408 resp->status = DLB2_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
1413 if (args->num_qid_inflights > 4096) {
1414 resp->status = DLB2_ST_INVALID_QID_INFLIGHT_ALLOCATION;
1418 /* Inflights must be <= number of sequence numbers if ordered */
1419 if (args->num_sequence_numbers != 0 &&
1420 args->num_qid_inflights > args->num_sequence_numbers) {
1421 resp->status = DLB2_ST_INVALID_QID_INFLIGHT_ALLOCATION;
1425 if (domain->num_avail_aqed_entries < args->num_atomic_inflights) {
1426 resp->status = DLB2_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
1430 if (args->num_atomic_inflights &&
1431 args->lock_id_comp_level != 0 &&
1432 args->lock_id_comp_level != 64 &&
1433 args->lock_id_comp_level != 128 &&
1434 args->lock_id_comp_level != 256 &&
1435 args->lock_id_comp_level != 512 &&
1436 args->lock_id_comp_level != 1024 &&
1437 args->lock_id_comp_level != 2048 &&
1438 args->lock_id_comp_level != 4096 &&
1439 args->lock_id_comp_level != 65536) {
1440 resp->status = DLB2_ST_INVALID_LOCK_ID_COMP_LEVEL;
1448 dlb2_log_create_ldb_queue_args(struct dlb2_hw *hw,
1450 struct dlb2_create_ldb_queue_args *args,
1452 unsigned int vdev_id)
1454 DLB2_HW_DBG(hw, "DLB2 create load-balanced queue arguments:\n");
1456 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
1457 DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
1459 DLB2_HW_DBG(hw, "\tNumber of sequence numbers: %d\n",
1460 args->num_sequence_numbers);
1461 DLB2_HW_DBG(hw, "\tNumber of QID inflights: %d\n",
1462 args->num_qid_inflights);
1463 DLB2_HW_DBG(hw, "\tNumber of ATM inflights: %d\n",
1464 args->num_atomic_inflights);
1468 * dlb2_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
1469 * @hw: Contains the current state of the DLB2 hardware.
1470 * @domain_id: Domain ID
1471 * @args: User-provided arguments.
1472 * @resp: Response to user.
1473 * @vdev_req: Request came from a virtual device.
1474 * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
1476 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
1477 * satisfy a request, resp->status will be set accordingly.
1479 int dlb2_hw_create_ldb_queue(struct dlb2_hw *hw,
1481 struct dlb2_create_ldb_queue_args *args,
1482 struct dlb2_cmd_response *resp,
1484 unsigned int vdev_id)
1486 struct dlb2_hw_domain *domain;
1487 struct dlb2_ldb_queue *queue;
1490 dlb2_log_create_ldb_queue_args(hw, domain_id, args, vdev_req, vdev_id);
1493 * Verify that hardware resources are available before attempting to
1494 * satisfy the request. This simplifies the error unwinding code.
1496 ret = dlb2_verify_create_ldb_queue_args(hw,
1505 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
1506 if (domain == NULL) {
1508 "[%s():%d] Internal error: domain not found\n",
1509 __func__, __LINE__);
1513 queue = DLB2_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
1514 if (queue == NULL) {
1516 "[%s():%d] Internal error: no available ldb queues\n",
1517 __func__, __LINE__);
1521 ret = dlb2_ldb_queue_attach_resources(hw, domain, queue, args);
1524 "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
1525 __func__, __LINE__);
1529 dlb2_configure_ldb_queue(hw, domain, queue, args, vdev_req, vdev_id);
1531 queue->num_mappings = 0;
1533 queue->configured = true;
1536 * Configuration succeeded, so move the resource from the 'avail' to
1539 dlb2_list_del(&domain->avail_ldb_queues, &queue->domain_list);
1541 dlb2_list_add(&domain->used_ldb_queues, &queue->domain_list);
1544 resp->id = (vdev_req) ? queue->id.virt_id : queue->id.phys_id;
1549 int dlb2_get_group_sequence_numbers(struct dlb2_hw *hw, unsigned int group_id)
1551 if (group_id >= DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
1554 return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
1557 int dlb2_get_group_sequence_number_occupancy(struct dlb2_hw *hw,
1558 unsigned int group_id)
1560 if (group_id >= DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
1563 return dlb2_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
1566 static void dlb2_log_set_group_sequence_numbers(struct dlb2_hw *hw,
1567 unsigned int group_id,
1570 DLB2_HW_DBG(hw, "DLB2 set group sequence numbers:\n");
1571 DLB2_HW_DBG(hw, "\tGroup ID: %u\n", group_id);
1572 DLB2_HW_DBG(hw, "\tValue: %lu\n", val);
1575 int dlb2_set_group_sequence_numbers(struct dlb2_hw *hw,
1576 unsigned int group_id,
1579 u32 valid_allocations[] = {64, 128, 256, 512, 1024};
1580 union dlb2_ro_pipe_grp_sn_mode r0 = { {0} };
1581 struct dlb2_sn_group *group;
1584 if (group_id >= DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
1587 group = &hw->rsrcs.sn_groups[group_id];
1590 * Once the first load-balanced queue using an SN group is configured,
1591 * the group cannot be changed.
1593 if (group->slot_use_bitmap != 0)
1596 for (mode = 0; mode < DLB2_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
1597 if (val == valid_allocations[mode])
1600 if (mode == DLB2_MAX_NUM_SEQUENCE_NUMBER_MODES)
1604 group->sequence_numbers_per_queue = val;
1606 r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
1607 r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
1609 DLB2_CSR_WR(hw, DLB2_RO_PIPE_GRP_SN_MODE, r0.val);
1611 dlb2_log_set_group_sequence_numbers(hw, group_id, val);
1616 static void dlb2_ldb_port_configure_pp(struct dlb2_hw *hw,
1617 struct dlb2_hw_domain *domain,
1618 struct dlb2_ldb_port *port,
1620 unsigned int vdev_id)
1622 union dlb2_sys_ldb_pp2vas r0 = { {0} };
1623 union dlb2_sys_ldb_pp_v r4 = { {0} };
1625 r0.field.vas = domain->id.phys_id;
1627 DLB2_CSR_WR(hw, DLB2_SYS_LDB_PP2VAS(port->id.phys_id), r0.val);
1630 union dlb2_sys_vf_ldb_vpp2pp r1 = { {0} };
1631 union dlb2_sys_ldb_pp2vdev r2 = { {0} };
1632 union dlb2_sys_vf_ldb_vpp_v r3 = { {0} };
1637 * DLB uses producer port address bits 17:12 to determine the
1638 * producer port ID. In Scalable IOV mode, PP accesses come
1639 * through the PF MMIO window for the physical producer port,
1640 * so for translation purposes the virtual and physical port
1643 if (hw->virt_mode == DLB2_VIRT_SRIOV)
1644 virt_id = port->id.virt_id;
1646 virt_id = port->id.phys_id;
1648 r1.field.pp = port->id.phys_id;
1650 offs = vdev_id * DLB2_MAX_NUM_LDB_PORTS + virt_id;
1652 DLB2_CSR_WR(hw, DLB2_SYS_VF_LDB_VPP2PP(offs), r1.val);
1654 r2.field.vdev = vdev_id;
1657 DLB2_SYS_LDB_PP2VDEV(port->id.phys_id),
1662 DLB2_CSR_WR(hw, DLB2_SYS_VF_LDB_VPP_V(offs), r3.val);
1668 DLB2_SYS_LDB_PP_V(port->id.phys_id),
1672 static int dlb2_ldb_port_configure_cq(struct dlb2_hw *hw,
1673 struct dlb2_hw_domain *domain,
1674 struct dlb2_ldb_port *port,
1675 uintptr_t cq_dma_base,
1676 struct dlb2_create_ldb_port_args *args,
1678 unsigned int vdev_id)
1680 union dlb2_sys_ldb_cq_addr_l r0 = { {0} };
1681 union dlb2_sys_ldb_cq_addr_u r1 = { {0} };
1682 union dlb2_sys_ldb_cq2vf_pf_ro r2 = { {0} };
1683 union dlb2_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
1684 union dlb2_lsp_cq_ldb_tkn_depth_sel r4 = { {0} };
1685 union dlb2_chp_hist_list_lim r5 = { {0} };
1686 union dlb2_chp_hist_list_base r6 = { {0} };
1687 union dlb2_lsp_cq_ldb_infl_lim r7 = { {0} };
1688 union dlb2_chp_hist_list_push_ptr r8 = { {0} };
1689 union dlb2_chp_hist_list_pop_ptr r9 = { {0} };
1690 union dlb2_sys_ldb_cq_at r10 = { {0} };
1691 union dlb2_sys_ldb_cq_pasid r11 = { {0} };
1692 union dlb2_chp_ldb_cq2vas r12 = { {0} };
1693 union dlb2_lsp_cq2priov r13 = { {0} };
1695 /* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
1696 r0.field.addr_l = cq_dma_base >> 6;
1698 DLB2_CSR_WR(hw, DLB2_SYS_LDB_CQ_ADDR_L(port->id.phys_id), r0.val);
1700 r1.field.addr_u = cq_dma_base >> 32;
1702 DLB2_CSR_WR(hw, DLB2_SYS_LDB_CQ_ADDR_U(port->id.phys_id), r1.val);
1705 * 'ro' == relaxed ordering. This setting allows DLB2 to write
1706 * cache lines out-of-order (but QEs within a cache line are always
1707 * updated in-order).
1709 r2.field.vf = vdev_id;
1710 r2.field.is_pf = !vdev_req && (hw->virt_mode != DLB2_VIRT_SIOV);
1713 DLB2_CSR_WR(hw, DLB2_SYS_LDB_CQ2VF_PF_RO(port->id.phys_id), r2.val);
1715 if (args->cq_depth <= 8) {
1716 r3.field.token_depth_select = 1;
1717 } else if (args->cq_depth == 16) {
1718 r3.field.token_depth_select = 2;
1719 } else if (args->cq_depth == 32) {
1720 r3.field.token_depth_select = 3;
1721 } else if (args->cq_depth == 64) {
1722 r3.field.token_depth_select = 4;
1723 } else if (args->cq_depth == 128) {
1724 r3.field.token_depth_select = 5;
1725 } else if (args->cq_depth == 256) {
1726 r3.field.token_depth_select = 6;
1727 } else if (args->cq_depth == 512) {
1728 r3.field.token_depth_select = 7;
1729 } else if (args->cq_depth == 1024) {
1730 r3.field.token_depth_select = 8;
1733 "[%s():%d] Internal error: invalid CQ depth\n",
1734 __func__, __LINE__);
1739 DLB2_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id.phys_id),
1743 * To support CQs with depth less than 8, program the token count
1744 * register with a non-zero initial value. Operations such as domain
1745 * reset must take this initial value into account when quiescing the
1748 port->init_tkn_cnt = 0;
1750 if (args->cq_depth < 8) {
1751 union dlb2_lsp_cq_ldb_tkn_cnt r14 = { {0} };
1753 port->init_tkn_cnt = 8 - args->cq_depth;
1755 r14.field.token_count = port->init_tkn_cnt;
1758 DLB2_LSP_CQ_LDB_TKN_CNT(port->id.phys_id),
1762 DLB2_LSP_CQ_LDB_TKN_CNT(port->id.phys_id),
1763 DLB2_LSP_CQ_LDB_TKN_CNT_RST);
1766 r4.field.token_depth_select = r3.field.token_depth_select;
1767 r4.field.ignore_depth = 0;
1770 DLB2_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id.phys_id),
1773 /* Reset the CQ write pointer */
1775 DLB2_CHP_LDB_CQ_WPTR(port->id.phys_id),
1776 DLB2_CHP_LDB_CQ_WPTR_RST);
1778 r5.field.limit = port->hist_list_entry_limit - 1;
1780 DLB2_CSR_WR(hw, DLB2_CHP_HIST_LIST_LIM(port->id.phys_id), r5.val);
1782 r6.field.base = port->hist_list_entry_base;
1784 DLB2_CSR_WR(hw, DLB2_CHP_HIST_LIST_BASE(port->id.phys_id), r6.val);
1787 * The inflight limit sets a cap on the number of QEs for which this CQ
1788 * can owe completions at one time.
1790 r7.field.limit = args->cq_history_list_size;
1792 DLB2_CSR_WR(hw, DLB2_LSP_CQ_LDB_INFL_LIM(port->id.phys_id), r7.val);
1794 r8.field.push_ptr = r6.field.base;
1795 r8.field.generation = 0;
1798 DLB2_CHP_HIST_LIST_PUSH_PTR(port->id.phys_id),
1801 r9.field.pop_ptr = r6.field.base;
1802 r9.field.generation = 0;
1804 DLB2_CSR_WR(hw, DLB2_CHP_HIST_LIST_POP_PTR(port->id.phys_id), r9.val);
1807 * Address translation (AT) settings: 0: untranslated, 2: translated
1808 * (see ATS spec regarding Address Type field for more details)
1810 r10.field.cq_at = 0;
1812 DLB2_CSR_WR(hw, DLB2_SYS_LDB_CQ_AT(port->id.phys_id), r10.val);
1814 if (vdev_req && hw->virt_mode == DLB2_VIRT_SIOV) {
1815 r11.field.pasid = hw->pasid[vdev_id];
1820 DLB2_SYS_LDB_CQ_PASID(port->id.phys_id),
1823 r12.field.cq2vas = domain->id.phys_id;
1825 DLB2_CSR_WR(hw, DLB2_CHP_LDB_CQ2VAS(port->id.phys_id), r12.val);
1827 /* Disable the port's QID mappings */
1830 DLB2_CSR_WR(hw, DLB2_LSP_CQ2PRIOV(port->id.phys_id), r13.val);
1835 static int dlb2_configure_ldb_port(struct dlb2_hw *hw,
1836 struct dlb2_hw_domain *domain,
1837 struct dlb2_ldb_port *port,
1838 uintptr_t cq_dma_base,
1839 struct dlb2_create_ldb_port_args *args,
1841 unsigned int vdev_id)
1845 port->hist_list_entry_base = domain->hist_list_entry_base +
1846 domain->hist_list_entry_offset;
1847 port->hist_list_entry_limit = port->hist_list_entry_base +
1848 args->cq_history_list_size;
1850 domain->hist_list_entry_offset += args->cq_history_list_size;
1851 domain->avail_hist_list_entries -= args->cq_history_list_size;
1853 ret = dlb2_ldb_port_configure_cq(hw,
1863 dlb2_ldb_port_configure_pp(hw,
1869 dlb2_ldb_port_cq_enable(hw, port);
1871 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++)
1872 port->qid_map[i].state = DLB2_QUEUE_UNMAPPED;
1873 port->num_mappings = 0;
1875 port->enabled = true;
1877 port->configured = true;
1883 dlb2_log_create_ldb_port_args(struct dlb2_hw *hw,
1885 uintptr_t cq_dma_base,
1886 struct dlb2_create_ldb_port_args *args,
1888 unsigned int vdev_id)
1890 DLB2_HW_DBG(hw, "DLB2 create load-balanced port arguments:\n");
1892 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
1893 DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
1895 DLB2_HW_DBG(hw, "\tCQ depth: %d\n",
1897 DLB2_HW_DBG(hw, "\tCQ hist list size: %d\n",
1898 args->cq_history_list_size);
1899 DLB2_HW_DBG(hw, "\tCQ base address: 0x%lx\n",
1901 DLB2_HW_DBG(hw, "\tCoS ID: %u\n", args->cos_id);
1902 DLB2_HW_DBG(hw, "\tStrict CoS allocation: %u\n",
1907 dlb2_verify_create_ldb_port_args(struct dlb2_hw *hw,
1909 uintptr_t cq_dma_base,
1910 struct dlb2_create_ldb_port_args *args,
1911 struct dlb2_cmd_response *resp,
1913 unsigned int vdev_id)
1915 struct dlb2_hw_domain *domain;
1918 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
1920 if (domain == NULL) {
1921 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
1925 if (!domain->configured) {
1926 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
1930 if (domain->started) {
1931 resp->status = DLB2_ST_DOMAIN_STARTED;
1935 if (args->cos_id >= DLB2_NUM_COS_DOMAINS) {
1936 resp->status = DLB2_ST_INVALID_COS_ID;
1940 if (args->cos_strict) {
1941 if (dlb2_list_empty(&domain->avail_ldb_ports[args->cos_id])) {
1942 resp->status = DLB2_ST_LDB_PORTS_UNAVAILABLE;
1946 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
1947 if (!dlb2_list_empty(&domain->avail_ldb_ports[i]))
1951 if (i == DLB2_NUM_COS_DOMAINS) {
1952 resp->status = DLB2_ST_LDB_PORTS_UNAVAILABLE;
1957 /* Check cache-line alignment */
1958 if ((cq_dma_base & 0x3F) != 0) {
1959 resp->status = DLB2_ST_INVALID_CQ_VIRT_ADDR;
1963 if (args->cq_depth != 1 &&
1964 args->cq_depth != 2 &&
1965 args->cq_depth != 4 &&
1966 args->cq_depth != 8 &&
1967 args->cq_depth != 16 &&
1968 args->cq_depth != 32 &&
1969 args->cq_depth != 64 &&
1970 args->cq_depth != 128 &&
1971 args->cq_depth != 256 &&
1972 args->cq_depth != 512 &&
1973 args->cq_depth != 1024) {
1974 resp->status = DLB2_ST_INVALID_CQ_DEPTH;
1978 /* The history list size must be >= 1 */
1979 if (!args->cq_history_list_size) {
1980 resp->status = DLB2_ST_INVALID_HIST_LIST_DEPTH;
1984 if (args->cq_history_list_size > domain->avail_hist_list_entries) {
1985 resp->status = DLB2_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
1994 * dlb2_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
1996 * @hw: Contains the current state of the DLB2 hardware.
1997 * @domain_id: Domain ID
1998 * @args: User-provided arguments.
1999 * @cq_dma_base: Base DMA address for consumer queue memory
2000 * @resp: Response to user.
2001 * @vdev_req: Request came from a virtual device.
2002 * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
2004 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
2005 * satisfy a request, resp->status will be set accordingly.
2007 int dlb2_hw_create_ldb_port(struct dlb2_hw *hw,
2009 struct dlb2_create_ldb_port_args *args,
2010 uintptr_t cq_dma_base,
2011 struct dlb2_cmd_response *resp,
2013 unsigned int vdev_id)
2015 struct dlb2_hw_domain *domain;
2016 struct dlb2_ldb_port *port;
2019 dlb2_log_create_ldb_port_args(hw,
2027 * Verify that hardware resources are available before attempting to
2028 * satisfy the request. This simplifies the error unwinding code.
2030 ret = dlb2_verify_create_ldb_port_args(hw,
2040 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2041 if (domain == NULL) {
2043 "[%s():%d] Internal error: domain not found\n",
2044 __func__, __LINE__);
2048 if (args->cos_strict) {
2049 cos_id = args->cos_id;
2051 port = DLB2_DOM_LIST_HEAD(domain->avail_ldb_ports[cos_id],
2056 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
2057 idx = (args->cos_id + i) % DLB2_NUM_COS_DOMAINS;
2059 port = DLB2_DOM_LIST_HEAD(domain->avail_ldb_ports[idx],
2070 "[%s():%d] Internal error: no available ldb ports\n",
2071 __func__, __LINE__);
2075 if (port->configured) {
2077 "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
2082 ret = dlb2_configure_ldb_port(hw,
2093 * Configuration succeeded, so move the resource from the 'avail' to
2096 dlb2_list_del(&domain->avail_ldb_ports[cos_id], &port->domain_list);
2098 dlb2_list_add(&domain->used_ldb_ports[cos_id], &port->domain_list);
2101 resp->id = (vdev_req) ? port->id.virt_id : port->id.phys_id;
2107 dlb2_log_create_dir_port_args(struct dlb2_hw *hw,
2109 uintptr_t cq_dma_base,
2110 struct dlb2_create_dir_port_args *args,
2112 unsigned int vdev_id)
2114 DLB2_HW_DBG(hw, "DLB2 create directed port arguments:\n");
2116 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
2117 DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
2119 DLB2_HW_DBG(hw, "\tCQ depth: %d\n",
2121 DLB2_HW_DBG(hw, "\tCQ base address: 0x%lx\n",
2125 static struct dlb2_dir_pq_pair *
2126 dlb2_get_domain_used_dir_pq(struct dlb2_hw *hw,
2129 struct dlb2_hw_domain *domain)
2131 struct dlb2_list_entry *iter;
2132 struct dlb2_dir_pq_pair *port;
2135 if (id >= DLB2_MAX_NUM_DIR_PORTS(hw->ver))
2138 DLB2_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
2139 if ((!vdev_req && port->id.phys_id == id) ||
2140 (vdev_req && port->id.virt_id == id))
2147 dlb2_verify_create_dir_port_args(struct dlb2_hw *hw,
2149 uintptr_t cq_dma_base,
2150 struct dlb2_create_dir_port_args *args,
2151 struct dlb2_cmd_response *resp,
2153 unsigned int vdev_id)
2155 struct dlb2_hw_domain *domain;
2157 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2159 if (domain == NULL) {
2160 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2164 if (!domain->configured) {
2165 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
2169 if (domain->started) {
2170 resp->status = DLB2_ST_DOMAIN_STARTED;
2175 * If the user claims the queue is already configured, validate
2176 * the queue ID, its domain, and whether the queue is configured.
2178 if (args->queue_id != -1) {
2179 struct dlb2_dir_pq_pair *queue;
2181 queue = dlb2_get_domain_used_dir_pq(hw,
2186 if (queue == NULL || queue->domain_id.phys_id !=
2187 domain->id.phys_id ||
2188 !queue->queue_configured) {
2189 resp->status = DLB2_ST_INVALID_DIR_QUEUE_ID;
2195 * If the port's queue is not configured, validate that a free
2196 * port-queue pair is available.
2198 if (args->queue_id == -1 &&
2199 dlb2_list_empty(&domain->avail_dir_pq_pairs)) {
2200 resp->status = DLB2_ST_DIR_PORTS_UNAVAILABLE;
2204 /* Check cache-line alignment */
2205 if ((cq_dma_base & 0x3F) != 0) {
2206 resp->status = DLB2_ST_INVALID_CQ_VIRT_ADDR;
2210 if (args->cq_depth != 1 &&
2211 args->cq_depth != 2 &&
2212 args->cq_depth != 4 &&
2213 args->cq_depth != 8 &&
2214 args->cq_depth != 16 &&
2215 args->cq_depth != 32 &&
2216 args->cq_depth != 64 &&
2217 args->cq_depth != 128 &&
2218 args->cq_depth != 256 &&
2219 args->cq_depth != 512 &&
2220 args->cq_depth != 1024) {
2221 resp->status = DLB2_ST_INVALID_CQ_DEPTH;
2228 static void dlb2_dir_port_configure_pp(struct dlb2_hw *hw,
2229 struct dlb2_hw_domain *domain,
2230 struct dlb2_dir_pq_pair *port,
2232 unsigned int vdev_id)
2234 union dlb2_sys_dir_pp2vas r0 = { {0} };
2235 union dlb2_sys_dir_pp_v r4 = { {0} };
2237 r0.field.vas = domain->id.phys_id;
2239 DLB2_CSR_WR(hw, DLB2_SYS_DIR_PP2VAS(port->id.phys_id), r0.val);
2242 union dlb2_sys_vf_dir_vpp2pp r1 = { {0} };
2243 union dlb2_sys_dir_pp2vdev r2 = { {0} };
2244 union dlb2_sys_vf_dir_vpp_v r3 = { {0} };
2249 * DLB uses producer port address bits 17:12 to determine the
2250 * producer port ID. In Scalable IOV mode, PP accesses come
2251 * through the PF MMIO window for the physical producer port,
2252 * so for translation purposes the virtual and physical port
2255 if (hw->virt_mode == DLB2_VIRT_SRIOV)
2256 virt_id = port->id.virt_id;
2258 virt_id = port->id.phys_id;
2260 r1.field.pp = port->id.phys_id;
2262 offs = vdev_id * DLB2_MAX_NUM_DIR_PORTS(hw->ver) + virt_id;
2264 DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VPP2PP(offs), r1.val);
2266 r2.field.vdev = vdev_id;
2269 DLB2_SYS_DIR_PP2VDEV(port->id.phys_id),
2274 DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VPP_V(offs), r3.val);
2280 DLB2_SYS_DIR_PP_V(port->id.phys_id),
2284 static int dlb2_dir_port_configure_cq(struct dlb2_hw *hw,
2285 struct dlb2_hw_domain *domain,
2286 struct dlb2_dir_pq_pair *port,
2287 uintptr_t cq_dma_base,
2288 struct dlb2_create_dir_port_args *args,
2290 unsigned int vdev_id)
2292 union dlb2_sys_dir_cq_addr_l r0 = { {0} };
2293 union dlb2_sys_dir_cq_addr_u r1 = { {0} };
2294 union dlb2_sys_dir_cq2vf_pf_ro r2 = { {0} };
2295 union dlb2_chp_dir_cq_tkn_depth_sel r3 = { {0} };
2296 union dlb2_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
2297 union dlb2_sys_dir_cq_fmt r9 = { {0} };
2298 union dlb2_sys_dir_cq_at r10 = { {0} };
2299 union dlb2_sys_dir_cq_pasid r11 = { {0} };
2300 union dlb2_chp_dir_cq2vas r12 = { {0} };
2302 /* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
2303 r0.field.addr_l = cq_dma_base >> 6;
2305 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_ADDR_L(port->id.phys_id), r0.val);
2307 r1.field.addr_u = cq_dma_base >> 32;
2309 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_ADDR_U(port->id.phys_id), r1.val);
2312 * 'ro' == relaxed ordering. This setting allows DLB2 to write
2313 * cache lines out-of-order (but QEs within a cache line are always
2314 * updated in-order).
2316 r2.field.vf = vdev_id;
2317 r2.field.is_pf = !vdev_req && (hw->virt_mode != DLB2_VIRT_SIOV);
2320 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ2VF_PF_RO(port->id.phys_id), r2.val);
2322 if (args->cq_depth <= 8) {
2323 r3.field.token_depth_select = 1;
2324 } else if (args->cq_depth == 16) {
2325 r3.field.token_depth_select = 2;
2326 } else if (args->cq_depth == 32) {
2327 r3.field.token_depth_select = 3;
2328 } else if (args->cq_depth == 64) {
2329 r3.field.token_depth_select = 4;
2330 } else if (args->cq_depth == 128) {
2331 r3.field.token_depth_select = 5;
2332 } else if (args->cq_depth == 256) {
2333 r3.field.token_depth_select = 6;
2334 } else if (args->cq_depth == 512) {
2335 r3.field.token_depth_select = 7;
2336 } else if (args->cq_depth == 1024) {
2337 r3.field.token_depth_select = 8;
2340 "[%s():%d] Internal error: invalid CQ depth\n",
2341 __func__, __LINE__);
2346 DLB2_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id.phys_id),
2350 * To support CQs with depth less than 8, program the token count
2351 * register with a non-zero initial value. Operations such as domain
2352 * reset must take this initial value into account when quiescing the
2355 port->init_tkn_cnt = 0;
2357 if (args->cq_depth < 8) {
2358 union dlb2_lsp_cq_dir_tkn_cnt r13 = { {0} };
2360 port->init_tkn_cnt = 8 - args->cq_depth;
2362 r13.field.count = port->init_tkn_cnt;
2365 DLB2_LSP_CQ_DIR_TKN_CNT(port->id.phys_id),
2369 DLB2_LSP_CQ_DIR_TKN_CNT(port->id.phys_id),
2370 DLB2_LSP_CQ_DIR_TKN_CNT_RST);
2373 r4.field.token_depth_select = r3.field.token_depth_select;
2374 r4.field.disable_wb_opt = 0;
2375 r4.field.ignore_depth = 0;
2378 DLB2_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id.phys_id),
2381 /* Reset the CQ write pointer */
2383 DLB2_CHP_DIR_CQ_WPTR(port->id.phys_id),
2384 DLB2_CHP_DIR_CQ_WPTR_RST);
2386 /* Virtualize the PPID */
2387 r9.field.keep_pf_ppid = 0;
2389 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_FMT(port->id.phys_id), r9.val);
2392 * Address translation (AT) settings: 0: untranslated, 2: translated
2393 * (see ATS spec regarding Address Type field for more details)
2395 r10.field.cq_at = 0;
2397 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_AT(port->id.phys_id), r10.val);
2399 if (vdev_req && hw->virt_mode == DLB2_VIRT_SIOV) {
2400 r11.field.pasid = hw->pasid[vdev_id];
2405 DLB2_SYS_DIR_CQ_PASID(port->id.phys_id),
2408 r12.field.cq2vas = domain->id.phys_id;
2410 DLB2_CSR_WR(hw, DLB2_CHP_DIR_CQ2VAS(port->id.phys_id), r12.val);
2415 static int dlb2_configure_dir_port(struct dlb2_hw *hw,
2416 struct dlb2_hw_domain *domain,
2417 struct dlb2_dir_pq_pair *port,
2418 uintptr_t cq_dma_base,
2419 struct dlb2_create_dir_port_args *args,
2421 unsigned int vdev_id)
2425 ret = dlb2_dir_port_configure_cq(hw,
2436 dlb2_dir_port_configure_pp(hw,
2442 dlb2_dir_port_cq_enable(hw, port);
2444 port->enabled = true;
2446 port->port_configured = true;
2452 * dlb2_hw_create_dir_port() - Allocate and initialize a DLB directed port
2453 * and queue. The port/queue pair have the same ID and name.
2454 * @hw: Contains the current state of the DLB2 hardware.
2455 * @domain_id: Domain ID
2456 * @args: User-provided arguments.
2457 * @cq_dma_base: Base DMA address for consumer queue memory
2458 * @resp: Response to user.
2459 * @vdev_req: Request came from a virtual device.
2460 * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
2462 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
2463 * satisfy a request, resp->status will be set accordingly.
2465 int dlb2_hw_create_dir_port(struct dlb2_hw *hw,
2467 struct dlb2_create_dir_port_args *args,
2468 uintptr_t cq_dma_base,
2469 struct dlb2_cmd_response *resp,
2471 unsigned int vdev_id)
2473 struct dlb2_dir_pq_pair *port;
2474 struct dlb2_hw_domain *domain;
2477 dlb2_log_create_dir_port_args(hw,
2485 * Verify that hardware resources are available before attempting to
2486 * satisfy the request. This simplifies the error unwinding code.
2488 ret = dlb2_verify_create_dir_port_args(hw,
2498 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2500 if (args->queue_id != -1)
2501 port = dlb2_get_domain_used_dir_pq(hw,
2506 port = DLB2_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
2510 "[%s():%d] Internal error: no available dir ports\n",
2511 __func__, __LINE__);
2515 ret = dlb2_configure_dir_port(hw,
2526 * Configuration succeeded, so move the resource from the 'avail' to
2527 * the 'used' list (if it's not already there).
2529 if (args->queue_id == -1) {
2530 dlb2_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
2532 dlb2_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
2536 resp->id = (vdev_req) ? port->id.virt_id : port->id.phys_id;
2541 static void dlb2_configure_dir_queue(struct dlb2_hw *hw,
2542 struct dlb2_hw_domain *domain,
2543 struct dlb2_dir_pq_pair *queue,
2544 struct dlb2_create_dir_queue_args *args,
2546 unsigned int vdev_id)
2548 union dlb2_sys_dir_vasqid_v r0 = { {0} };
2549 union dlb2_sys_dir_qid_its r1 = { {0} };
2550 union dlb2_lsp_qid_dir_depth_thrsh r2 = { {0} };
2551 union dlb2_sys_dir_qid_v r5 = { {0} };
2555 /* QID write permissions are turned on when the domain is started */
2556 r0.field.vasqid_v = 0;
2558 offs = domain->id.phys_id * DLB2_MAX_NUM_DIR_QUEUES(hw->ver) +
2561 DLB2_CSR_WR(hw, DLB2_SYS_DIR_VASQID_V(offs), r0.val);
2563 /* Don't timestamp QEs that pass through this queue */
2564 r1.field.qid_its = 0;
2567 DLB2_SYS_DIR_QID_ITS(queue->id.phys_id),
2570 r2.field.thresh = args->depth_threshold;
2573 DLB2_LSP_QID_DIR_DEPTH_THRSH(queue->id.phys_id),
2577 union dlb2_sys_vf_dir_vqid_v r3 = { {0} };
2578 union dlb2_sys_vf_dir_vqid2qid r4 = { {0} };
2580 offs = vdev_id * DLB2_MAX_NUM_DIR_QUEUES(hw->ver)
2581 + queue->id.virt_id;
2583 r3.field.vqid_v = 1;
2585 DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VQID_V(offs), r3.val);
2587 r4.field.qid = queue->id.phys_id;
2589 DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VQID2QID(offs), r4.val);
2594 DLB2_CSR_WR(hw, DLB2_SYS_DIR_QID_V(queue->id.phys_id), r5.val);
2596 queue->queue_configured = true;
2600 dlb2_log_create_dir_queue_args(struct dlb2_hw *hw,
2602 struct dlb2_create_dir_queue_args *args,
2604 unsigned int vdev_id)
2606 DLB2_HW_DBG(hw, "DLB2 create directed queue arguments:\n");
2608 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
2609 DLB2_HW_DBG(hw, "\tDomain ID: %d\n", domain_id);
2610 DLB2_HW_DBG(hw, "\tPort ID: %d\n", args->port_id);
2614 dlb2_verify_create_dir_queue_args(struct dlb2_hw *hw,
2616 struct dlb2_create_dir_queue_args *args,
2617 struct dlb2_cmd_response *resp,
2619 unsigned int vdev_id)
2621 struct dlb2_hw_domain *domain;
2623 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2625 if (domain == NULL) {
2626 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2630 if (!domain->configured) {
2631 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
2635 if (domain->started) {
2636 resp->status = DLB2_ST_DOMAIN_STARTED;
2641 * If the user claims the port is already configured, validate the port
2642 * ID, its domain, and whether the port is configured.
2644 if (args->port_id != -1) {
2645 struct dlb2_dir_pq_pair *port;
2647 port = dlb2_get_domain_used_dir_pq(hw,
2652 if (port == NULL || port->domain_id.phys_id !=
2653 domain->id.phys_id || !port->port_configured) {
2654 resp->status = DLB2_ST_INVALID_PORT_ID;
2660 * If the queue's port is not configured, validate that a free
2661 * port-queue pair is available.
2663 if (args->port_id == -1 &&
2664 dlb2_list_empty(&domain->avail_dir_pq_pairs)) {
2665 resp->status = DLB2_ST_DIR_QUEUES_UNAVAILABLE;
2673 * dlb2_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
2674 * @hw: Contains the current state of the DLB2 hardware.
2675 * @domain_id: Domain ID
2676 * @args: User-provided arguments.
2677 * @resp: Response to user.
2678 * @vdev_req: Request came from a virtual device.
2679 * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
2681 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
2682 * satisfy a request, resp->status will be set accordingly.
2684 int dlb2_hw_create_dir_queue(struct dlb2_hw *hw,
2686 struct dlb2_create_dir_queue_args *args,
2687 struct dlb2_cmd_response *resp,
2689 unsigned int vdev_id)
2691 struct dlb2_dir_pq_pair *queue;
2692 struct dlb2_hw_domain *domain;
2695 dlb2_log_create_dir_queue_args(hw, domain_id, args, vdev_req, vdev_id);
2698 * Verify that hardware resources are available before attempting to
2699 * satisfy the request. This simplifies the error unwinding code.
2701 ret = dlb2_verify_create_dir_queue_args(hw,
2710 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2711 if (domain == NULL) {
2713 "[%s():%d] Internal error: domain not found\n",
2714 __func__, __LINE__);
2718 if (args->port_id != -1)
2719 queue = dlb2_get_domain_used_dir_pq(hw,
2724 queue = DLB2_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
2726 if (queue == NULL) {
2728 "[%s():%d] Internal error: no available dir queues\n",
2729 __func__, __LINE__);
2733 dlb2_configure_dir_queue(hw, domain, queue, args, vdev_req, vdev_id);
2736 * Configuration succeeded, so move the resource from the 'avail' to
2737 * the 'used' list (if it's not already there).
2739 if (args->port_id == -1) {
2740 dlb2_list_del(&domain->avail_dir_pq_pairs,
2741 &queue->domain_list);
2743 dlb2_list_add(&domain->used_dir_pq_pairs,
2744 &queue->domain_list);
2749 resp->id = (vdev_req) ? queue->id.virt_id : queue->id.phys_id;
2755 dlb2_port_find_slot_with_pending_map_queue(struct dlb2_ldb_port *port,
2756 struct dlb2_ldb_queue *queue,
2761 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
2762 struct dlb2_ldb_port_qid_map *map = &port->qid_map[i];
2764 if (map->state == DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP &&
2765 map->pending_qid == queue->id.phys_id)
2771 return (i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ);
2774 static void dlb2_ldb_port_change_qid_priority(struct dlb2_hw *hw,
2775 struct dlb2_ldb_port *port,
2777 struct dlb2_map_qid_args *args)
2779 union dlb2_lsp_cq2priov r0;
2781 /* Read-modify-write the priority and valid bit register */
2782 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2PRIOV(port->id.phys_id));
2784 r0.field.v |= 1 << slot;
2785 r0.field.prio |= (args->priority & 0x7) << slot * 3;
2787 DLB2_CSR_WR(hw, DLB2_LSP_CQ2PRIOV(port->id.phys_id), r0.val);
2791 port->qid_map[slot].priority = args->priority;
2794 static int dlb2_verify_map_qid_slot_available(struct dlb2_ldb_port *port,
2795 struct dlb2_ldb_queue *queue,
2796 struct dlb2_cmd_response *resp)
2798 enum dlb2_qid_map_state state;
2801 /* Unused slot available? */
2802 if (port->num_mappings < DLB2_MAX_NUM_QIDS_PER_LDB_CQ)
2806 * If the queue is already mapped (from the application's perspective),
2807 * this is simply a priority update.
2809 state = DLB2_QUEUE_MAPPED;
2810 if (dlb2_port_find_slot_queue(port, state, queue, &i))
2813 state = DLB2_QUEUE_MAP_IN_PROG;
2814 if (dlb2_port_find_slot_queue(port, state, queue, &i))
2817 if (dlb2_port_find_slot_with_pending_map_queue(port, queue, &i))
2821 * If the slot contains an unmap in progress, it's considered
2824 state = DLB2_QUEUE_UNMAP_IN_PROG;
2825 if (dlb2_port_find_slot(port, state, &i))
2828 state = DLB2_QUEUE_UNMAPPED;
2829 if (dlb2_port_find_slot(port, state, &i))
2832 resp->status = DLB2_ST_NO_QID_SLOTS_AVAILABLE;
2836 static struct dlb2_ldb_queue *
2837 dlb2_get_domain_ldb_queue(u32 id,
2839 struct dlb2_hw_domain *domain)
2841 struct dlb2_list_entry *iter;
2842 struct dlb2_ldb_queue *queue;
2845 if (id >= DLB2_MAX_NUM_LDB_QUEUES)
2848 DLB2_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
2849 if ((!vdev_req && queue->id.phys_id == id) ||
2850 (vdev_req && queue->id.virt_id == id))
2856 static struct dlb2_ldb_port *
2857 dlb2_get_domain_used_ldb_port(u32 id,
2859 struct dlb2_hw_domain *domain)
2861 struct dlb2_list_entry *iter;
2862 struct dlb2_ldb_port *port;
2866 if (id >= DLB2_MAX_NUM_LDB_PORTS)
2869 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
2870 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter)
2871 if ((!vdev_req && port->id.phys_id == id) ||
2872 (vdev_req && port->id.virt_id == id))
2875 DLB2_DOM_LIST_FOR(domain->avail_ldb_ports[i], port, iter)
2876 if ((!vdev_req && port->id.phys_id == id) ||
2877 (vdev_req && port->id.virt_id == id))
2884 static int dlb2_verify_map_qid_args(struct dlb2_hw *hw,
2886 struct dlb2_map_qid_args *args,
2887 struct dlb2_cmd_response *resp,
2889 unsigned int vdev_id)
2891 struct dlb2_hw_domain *domain;
2892 struct dlb2_ldb_port *port;
2893 struct dlb2_ldb_queue *queue;
2896 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2898 if (domain == NULL) {
2899 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2903 if (!domain->configured) {
2904 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
2910 port = dlb2_get_domain_used_ldb_port(id, vdev_req, domain);
2912 if (port == NULL || !port->configured) {
2913 resp->status = DLB2_ST_INVALID_PORT_ID;
2917 if (args->priority >= DLB2_QID_PRIORITIES) {
2918 resp->status = DLB2_ST_INVALID_PRIORITY;
2922 queue = dlb2_get_domain_ldb_queue(args->qid, vdev_req, domain);
2924 if (queue == NULL || !queue->configured) {
2925 resp->status = DLB2_ST_INVALID_QID;
2929 if (queue->domain_id.phys_id != domain->id.phys_id) {
2930 resp->status = DLB2_ST_INVALID_QID;
2934 if (port->domain_id.phys_id != domain->id.phys_id) {
2935 resp->status = DLB2_ST_INVALID_PORT_ID;
2942 static void dlb2_log_map_qid(struct dlb2_hw *hw,
2944 struct dlb2_map_qid_args *args,
2946 unsigned int vdev_id)
2948 DLB2_HW_DBG(hw, "DLB2 map QID arguments:\n");
2950 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
2951 DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
2953 DLB2_HW_DBG(hw, "\tPort ID: %d\n",
2955 DLB2_HW_DBG(hw, "\tQueue ID: %d\n",
2957 DLB2_HW_DBG(hw, "\tPriority: %d\n",
2961 int dlb2_hw_map_qid(struct dlb2_hw *hw,
2963 struct dlb2_map_qid_args *args,
2964 struct dlb2_cmd_response *resp,
2966 unsigned int vdev_id)
2968 struct dlb2_hw_domain *domain;
2969 struct dlb2_ldb_queue *queue;
2970 enum dlb2_qid_map_state st;
2971 struct dlb2_ldb_port *port;
2975 dlb2_log_map_qid(hw, domain_id, args, vdev_req, vdev_id);
2978 * Verify that hardware resources are available before attempting to
2979 * satisfy the request. This simplifies the error unwinding code.
2981 ret = dlb2_verify_map_qid_args(hw,
2990 prio = args->priority;
2992 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2993 if (domain == NULL) {
2995 "[%s():%d] Internal error: domain not found\n",
2996 __func__, __LINE__);
3002 port = dlb2_get_domain_used_ldb_port(id, vdev_req, domain);
3005 "[%s():%d] Internal error: port not found\n",
3006 __func__, __LINE__);
3010 queue = dlb2_get_domain_ldb_queue(args->qid, vdev_req, domain);
3011 if (queue == NULL) {
3013 "[%s():%d] Internal error: queue not found\n",
3014 __func__, __LINE__);
3019 * If there are any outstanding detach operations for this port,
3020 * attempt to complete them. This may be necessary to free up a QID
3021 * slot for this requested mapping.
3023 if (port->num_pending_removals)
3024 dlb2_domain_finish_unmap_port(hw, domain, port);
3026 ret = dlb2_verify_map_qid_slot_available(port, queue, resp);
3030 /* Hardware requires disabling the CQ before mapping QIDs. */
3032 dlb2_ldb_port_cq_disable(hw, port);
3035 * If this is only a priority change, don't perform the full QID->CQ
3038 st = DLB2_QUEUE_MAPPED;
3039 if (dlb2_port_find_slot_queue(port, st, queue, &i)) {
3040 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
3042 "[%s():%d] Internal error: port slot tracking failed\n",
3043 __func__, __LINE__);
3047 if (prio != port->qid_map[i].priority) {
3048 dlb2_ldb_port_change_qid_priority(hw, port, i, args);
3049 DLB2_HW_DBG(hw, "DLB2 map: priority change\n");
3052 st = DLB2_QUEUE_MAPPED;
3053 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
3060 st = DLB2_QUEUE_UNMAP_IN_PROG;
3061 if (dlb2_port_find_slot_queue(port, st, queue, &i)) {
3062 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
3064 "[%s():%d] Internal error: port slot tracking failed\n",
3065 __func__, __LINE__);
3069 if (prio != port->qid_map[i].priority) {
3070 dlb2_ldb_port_change_qid_priority(hw, port, i, args);
3071 DLB2_HW_DBG(hw, "DLB2 map: priority change\n");
3074 st = DLB2_QUEUE_MAPPED;
3075 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
3083 * If this is a priority change on an in-progress mapping, don't
3084 * perform the full QID->CQ mapping procedure.
3086 st = DLB2_QUEUE_MAP_IN_PROG;
3087 if (dlb2_port_find_slot_queue(port, st, queue, &i)) {
3088 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
3090 "[%s():%d] Internal error: port slot tracking failed\n",
3091 __func__, __LINE__);
3095 port->qid_map[i].priority = prio;
3097 DLB2_HW_DBG(hw, "DLB2 map: priority change only\n");
3103 * If this is a priority change on a pending mapping, update the
3106 if (dlb2_port_find_slot_with_pending_map_queue(port, queue, &i)) {
3107 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
3109 "[%s():%d] Internal error: port slot tracking failed\n",
3110 __func__, __LINE__);
3114 port->qid_map[i].pending_priority = prio;
3116 DLB2_HW_DBG(hw, "DLB2 map: priority change only\n");
3122 * If all the CQ's slots are in use, then there's an unmap in progress
3123 * (guaranteed by dlb2_verify_map_qid_slot_available()), so add this
3124 * mapping to pending_map and return. When the removal is completed for
3125 * the slot's current occupant, this mapping will be performed.
3127 if (!dlb2_port_find_slot(port, DLB2_QUEUE_UNMAPPED, &i)) {
3128 if (dlb2_port_find_slot(port, DLB2_QUEUE_UNMAP_IN_PROG, &i)) {
3129 enum dlb2_qid_map_state st;
3131 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
3133 "[%s():%d] Internal error: port slot tracking failed\n",
3134 __func__, __LINE__);
3138 port->qid_map[i].pending_qid = queue->id.phys_id;
3139 port->qid_map[i].pending_priority = prio;
3141 st = DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP;
3143 ret = dlb2_port_slot_state_transition(hw, port, queue,
3148 DLB2_HW_DBG(hw, "DLB2 map: map pending removal\n");
3155 * If the domain has started, a special "dynamic" CQ->queue mapping
3156 * procedure is required in order to safely update the CQ<->QID tables.
3157 * The "static" procedure cannot be used when traffic is flowing,
3158 * because the CQ<->QID tables cannot be updated atomically and the
3159 * scheduler won't see the new mapping unless the queue's if_status
3160 * changes, which isn't guaranteed.
3162 ret = dlb2_ldb_port_map_qid(hw, domain, port, queue, prio);
3164 /* If ret is less than zero, it's due to an internal error */
3170 dlb2_ldb_port_cq_enable(hw, port);
3177 static void dlb2_log_unmap_qid(struct dlb2_hw *hw,
3179 struct dlb2_unmap_qid_args *args,
3181 unsigned int vdev_id)
3183 DLB2_HW_DBG(hw, "DLB2 unmap QID arguments:\n");
3185 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
3186 DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
3188 DLB2_HW_DBG(hw, "\tPort ID: %d\n",
3190 DLB2_HW_DBG(hw, "\tQueue ID: %d\n",
3192 if (args->qid < DLB2_MAX_NUM_LDB_QUEUES)
3193 DLB2_HW_DBG(hw, "\tQueue's num mappings: %d\n",
3194 hw->rsrcs.ldb_queues[args->qid].num_mappings);
3197 static int dlb2_verify_unmap_qid_args(struct dlb2_hw *hw,
3199 struct dlb2_unmap_qid_args *args,
3200 struct dlb2_cmd_response *resp,
3202 unsigned int vdev_id)
3204 enum dlb2_qid_map_state state;
3205 struct dlb2_hw_domain *domain;
3206 struct dlb2_ldb_queue *queue;
3207 struct dlb2_ldb_port *port;
3211 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
3213 if (domain == NULL) {
3214 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
3218 if (!domain->configured) {
3219 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
3225 port = dlb2_get_domain_used_ldb_port(id, vdev_req, domain);
3227 if (port == NULL || !port->configured) {
3228 resp->status = DLB2_ST_INVALID_PORT_ID;
3232 if (port->domain_id.phys_id != domain->id.phys_id) {
3233 resp->status = DLB2_ST_INVALID_PORT_ID;
3237 queue = dlb2_get_domain_ldb_queue(args->qid, vdev_req, domain);
3239 if (queue == NULL || !queue->configured) {
3240 DLB2_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
3241 __func__, args->qid);
3242 resp->status = DLB2_ST_INVALID_QID;
3247 * Verify that the port has the queue mapped. From the application's
3248 * perspective a queue is mapped if it is actually mapped, the map is
3249 * in progress, or the map is blocked pending an unmap.
3251 state = DLB2_QUEUE_MAPPED;
3252 if (dlb2_port_find_slot_queue(port, state, queue, &slot))
3255 state = DLB2_QUEUE_MAP_IN_PROG;
3256 if (dlb2_port_find_slot_queue(port, state, queue, &slot))
3259 if (dlb2_port_find_slot_with_pending_map_queue(port, queue, &slot))
3262 resp->status = DLB2_ST_INVALID_QID;
3266 int dlb2_hw_unmap_qid(struct dlb2_hw *hw,
3268 struct dlb2_unmap_qid_args *args,
3269 struct dlb2_cmd_response *resp,
3271 unsigned int vdev_id)
3273 struct dlb2_hw_domain *domain;
3274 struct dlb2_ldb_queue *queue;
3275 enum dlb2_qid_map_state st;
3276 struct dlb2_ldb_port *port;
3277 bool unmap_complete;
3280 dlb2_log_unmap_qid(hw, domain_id, args, vdev_req, vdev_id);
3283 * Verify that hardware resources are available before attempting to
3284 * satisfy the request. This simplifies the error unwinding code.
3286 ret = dlb2_verify_unmap_qid_args(hw,
3295 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
3296 if (domain == NULL) {
3298 "[%s():%d] Internal error: domain not found\n",
3299 __func__, __LINE__);
3305 port = dlb2_get_domain_used_ldb_port(id, vdev_req, domain);
3308 "[%s():%d] Internal error: port not found\n",
3309 __func__, __LINE__);
3313 queue = dlb2_get_domain_ldb_queue(args->qid, vdev_req, domain);
3314 if (queue == NULL) {
3316 "[%s():%d] Internal error: queue not found\n",
3317 __func__, __LINE__);
3322 * If the queue hasn't been mapped yet, we need to update the slot's
3323 * state and re-enable the queue's inflights.
3325 st = DLB2_QUEUE_MAP_IN_PROG;
3326 if (dlb2_port_find_slot_queue(port, st, queue, &i)) {
3327 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
3329 "[%s():%d] Internal error: port slot tracking failed\n",
3330 __func__, __LINE__);
3335 * Since the in-progress map was aborted, re-enable the QID's
3338 if (queue->num_pending_additions == 0)
3339 dlb2_ldb_queue_set_inflight_limit(hw, queue);
3341 st = DLB2_QUEUE_UNMAPPED;
3342 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
3346 goto unmap_qid_done;
3350 * If the queue mapping is on hold pending an unmap, we simply need to
3351 * update the slot's state.
3353 if (dlb2_port_find_slot_with_pending_map_queue(port, queue, &i)) {
3354 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
3356 "[%s():%d] Internal error: port slot tracking failed\n",
3357 __func__, __LINE__);
3361 st = DLB2_QUEUE_UNMAP_IN_PROG;
3362 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
3366 goto unmap_qid_done;
3369 st = DLB2_QUEUE_MAPPED;
3370 if (!dlb2_port_find_slot_queue(port, st, queue, &i)) {
3372 "[%s()] Internal error: no available CQ slots\n",
3377 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
3379 "[%s():%d] Internal error: port slot tracking failed\n",
3380 __func__, __LINE__);
3385 * QID->CQ mapping removal is an asynchronous procedure. It requires
3386 * stopping the DLB2 from scheduling this CQ, draining all inflights
3387 * from the CQ, then unmapping the queue from the CQ. This function
3388 * simply marks the port as needing the queue unmapped, and (if
3389 * necessary) starts the unmapping worker thread.
3391 dlb2_ldb_port_cq_disable(hw, port);
3393 st = DLB2_QUEUE_UNMAP_IN_PROG;
3394 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
3399 * Attempt to finish the unmapping now, in case the port has no
3400 * outstanding inflights. If that's not the case, this will fail and
3401 * the unmapping will be completed at a later time.
3403 unmap_complete = dlb2_domain_finish_unmap_port(hw, domain, port);
3406 * If the unmapping couldn't complete immediately, launch the worker
3407 * thread (if it isn't already launched) to finish it later.
3409 if (!unmap_complete && !os_worker_active(hw))
3410 os_schedule_work(hw);
3419 dlb2_log_pending_port_unmaps_args(struct dlb2_hw *hw,
3420 struct dlb2_pending_port_unmaps_args *args,
3422 unsigned int vdev_id)
3424 DLB2_HW_DBG(hw, "DLB unmaps in progress arguments:\n");
3426 DLB2_HW_DBG(hw, "(Request from VF %d)\n", vdev_id);
3427 DLB2_HW_DBG(hw, "\tPort ID: %d\n", args->port_id);
3430 int dlb2_hw_pending_port_unmaps(struct dlb2_hw *hw,
3432 struct dlb2_pending_port_unmaps_args *args,
3433 struct dlb2_cmd_response *resp,
3435 unsigned int vdev_id)
3437 struct dlb2_hw_domain *domain;
3438 struct dlb2_ldb_port *port;
3440 dlb2_log_pending_port_unmaps_args(hw, args, vdev_req, vdev_id);
3442 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
3444 if (domain == NULL) {
3445 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
3449 port = dlb2_get_domain_used_ldb_port(args->port_id, vdev_req, domain);
3450 if (port == NULL || !port->configured) {
3451 resp->status = DLB2_ST_INVALID_PORT_ID;
3455 resp->id = port->num_pending_removals;
3460 static int dlb2_verify_start_domain_args(struct dlb2_hw *hw,
3462 struct dlb2_cmd_response *resp,
3464 unsigned int vdev_id)
3466 struct dlb2_hw_domain *domain;
3468 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
3470 if (domain == NULL) {
3471 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
3475 if (!domain->configured) {
3476 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
3480 if (domain->started) {
3481 resp->status = DLB2_ST_DOMAIN_STARTED;
3488 static void dlb2_log_start_domain(struct dlb2_hw *hw,
3491 unsigned int vdev_id)
3493 DLB2_HW_DBG(hw, "DLB2 start domain arguments:\n");
3495 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
3496 DLB2_HW_DBG(hw, "\tDomain ID: %d\n", domain_id);
3500 * dlb2_hw_start_domain() - Lock the domain configuration
3501 * @hw: Contains the current state of the DLB2 hardware.
3502 * @domain_id: Domain ID
3503 * @arg: User-provided arguments (unused, here for ioctl callback template).
3504 * @resp: Response to user.
3505 * @vdev_req: Request came from a virtual device.
3506 * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
3508 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
3509 * satisfy a request, resp->status will be set accordingly.
3512 dlb2_hw_start_domain(struct dlb2_hw *hw,
3514 struct dlb2_start_domain_args *arg,
3515 struct dlb2_cmd_response *resp,
3517 unsigned int vdev_id)
3519 struct dlb2_list_entry *iter;
3520 struct dlb2_dir_pq_pair *dir_queue;
3521 struct dlb2_ldb_queue *ldb_queue;
3522 struct dlb2_hw_domain *domain;
3527 dlb2_log_start_domain(hw, domain_id, vdev_req, vdev_id);
3529 ret = dlb2_verify_start_domain_args(hw,
3537 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
3538 if (domain == NULL) {
3540 "[%s():%d] Internal error: domain not found\n",
3541 __func__, __LINE__);
3546 * Enable load-balanced and directed queue write permissions for the
3547 * queues this domain owns. Without this, the DLB2 will drop all
3548 * incoming traffic to those queues.
3550 DLB2_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
3551 union dlb2_sys_ldb_vasqid_v r0 = { {0} };
3554 r0.field.vasqid_v = 1;
3556 offs = domain->id.phys_id * DLB2_MAX_NUM_LDB_QUEUES +
3557 ldb_queue->id.phys_id;
3559 DLB2_CSR_WR(hw, DLB2_SYS_LDB_VASQID_V(offs), r0.val);
3562 DLB2_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
3563 union dlb2_sys_dir_vasqid_v r0 = { {0} };
3566 r0.field.vasqid_v = 1;
3568 offs = domain->id.phys_id * DLB2_MAX_NUM_DIR_PORTS(hw->ver) +
3569 dir_queue->id.phys_id;
3571 DLB2_CSR_WR(hw, DLB2_SYS_DIR_VASQID_V(offs), r0.val);
3576 domain->started = true;
3583 static void dlb2_log_get_dir_queue_depth(struct dlb2_hw *hw,
3589 DLB2_HW_DBG(hw, "DLB get directed queue depth:\n");
3591 DLB2_HW_DBG(hw, "(Request from VF %d)\n", vf_id);
3592 DLB2_HW_DBG(hw, "\tDomain ID: %d\n", domain_id);
3593 DLB2_HW_DBG(hw, "\tQueue ID: %d\n", queue_id);
3596 int dlb2_hw_get_dir_queue_depth(struct dlb2_hw *hw,
3598 struct dlb2_get_dir_queue_depth_args *args,
3599 struct dlb2_cmd_response *resp,
3601 unsigned int vdev_id)
3603 struct dlb2_dir_pq_pair *queue;
3604 struct dlb2_hw_domain *domain;
3609 dlb2_log_get_dir_queue_depth(hw, domain_id, args->queue_id,
3612 domain = dlb2_get_domain_from_id(hw, id, vdev_req, vdev_id);
3613 if (domain == NULL) {
3614 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
3618 id = args->queue_id;
3620 queue = dlb2_get_domain_used_dir_pq(hw, id, vdev_req, domain);
3621 if (queue == NULL) {
3622 resp->status = DLB2_ST_INVALID_QID;
3626 resp->id = dlb2_dir_queue_depth(hw, queue);
3631 static void dlb2_log_get_ldb_queue_depth(struct dlb2_hw *hw,
3637 DLB2_HW_DBG(hw, "DLB get load-balanced queue depth:\n");
3639 DLB2_HW_DBG(hw, "(Request from VF %d)\n", vf_id);
3640 DLB2_HW_DBG(hw, "\tDomain ID: %d\n", domain_id);
3641 DLB2_HW_DBG(hw, "\tQueue ID: %d\n", queue_id);
3644 int dlb2_hw_get_ldb_queue_depth(struct dlb2_hw *hw,
3646 struct dlb2_get_ldb_queue_depth_args *args,
3647 struct dlb2_cmd_response *resp,
3649 unsigned int vdev_id)
3651 struct dlb2_hw_domain *domain;
3652 struct dlb2_ldb_queue *queue;
3654 dlb2_log_get_ldb_queue_depth(hw, domain_id, args->queue_id,
3657 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
3658 if (domain == NULL) {
3659 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
3663 queue = dlb2_get_domain_ldb_queue(args->queue_id, vdev_req, domain);
3664 if (queue == NULL) {
3665 resp->status = DLB2_ST_INVALID_QID;
3669 resp->id = dlb2_ldb_queue_depth(hw, queue);