1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
7 #include "dlb2_hw_types.h"
8 #include "dlb2_osdep.h"
9 #include "dlb2_osdep_bitmap.h"
10 #include "dlb2_osdep_types.h"
11 #include "dlb2_regs.h"
12 #include "dlb2_resource.h"
14 #include "../../dlb2_priv.h"
15 #include "../../dlb2_inline_fns.h"
17 #define DLB2_DOM_LIST_HEAD(head, type) \
18 DLB2_LIST_HEAD((head), type, domain_list)
20 #define DLB2_FUNC_LIST_HEAD(head, type) \
21 DLB2_LIST_HEAD((head), type, func_list)
23 #define DLB2_DOM_LIST_FOR(head, ptr, iter) \
24 DLB2_LIST_FOR_EACH(head, ptr, domain_list, iter)
26 #define DLB2_FUNC_LIST_FOR(head, ptr, iter) \
27 DLB2_LIST_FOR_EACH(head, ptr, func_list, iter)
29 #define DLB2_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
30 DLB2_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
32 #define DLB2_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
33 DLB2_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
35 void dlb2_hw_enable_sparse_dir_cq_mode(struct dlb2_hw *hw)
37 union dlb2_chp_cfg_chp_csr_ctrl r0;
39 r0.val = DLB2_CSR_RD(hw, DLB2_CHP_CFG_CHP_CSR_CTRL);
41 r0.field.cfg_64bytes_qe_dir_cq_mode = 1;
43 DLB2_CSR_WR(hw, DLB2_CHP_CFG_CHP_CSR_CTRL, r0.val);
46 void dlb2_hw_enable_sparse_ldb_cq_mode(struct dlb2_hw *hw)
48 union dlb2_chp_cfg_chp_csr_ctrl r0;
50 r0.val = DLB2_CSR_RD(hw, DLB2_CHP_CFG_CHP_CSR_CTRL);
52 r0.field.cfg_64bytes_qe_ldb_cq_mode = 1;
54 DLB2_CSR_WR(hw, DLB2_CHP_CFG_CHP_CSR_CTRL, r0.val);
58 * The PF driver cannot assume that a register write will affect subsequent HCW
59 * writes. To ensure a write completes, the driver must read back a CSR. This
60 * function only need be called for configuration that can occur after the
61 * domain has started; prior to starting, applications can't send HCWs.
63 static inline void dlb2_flush_csr(struct dlb2_hw *hw)
65 DLB2_CSR_RD(hw, DLB2_SYS_TOTAL_VAS);
68 static void dlb2_dir_port_cq_enable(struct dlb2_hw *hw,
69 struct dlb2_dir_pq_pair *port)
71 union dlb2_lsp_cq_dir_dsbl reg;
73 reg.field.disabled = 0;
75 DLB2_CSR_WR(hw, DLB2_LSP_CQ_DIR_DSBL(port->id.phys_id), reg.val);
80 static u32 dlb2_dir_queue_depth(struct dlb2_hw *hw,
81 struct dlb2_dir_pq_pair *queue)
83 union dlb2_lsp_qid_dir_enqueue_cnt r0;
85 r0.val = DLB2_CSR_RD(hw,
86 DLB2_LSP_QID_DIR_ENQUEUE_CNT(queue->id.phys_id));
88 return r0.field.count;
91 static void dlb2_ldb_port_cq_enable(struct dlb2_hw *hw,
92 struct dlb2_ldb_port *port)
94 union dlb2_lsp_cq_ldb_dsbl reg;
97 * Don't re-enable the port if a removal is pending. The caller should
98 * mark this port as enabled (if it isn't already), and when the
99 * removal completes the port will be enabled.
101 if (port->num_pending_removals)
104 reg.field.disabled = 0;
106 DLB2_CSR_WR(hw, DLB2_LSP_CQ_LDB_DSBL(port->id.phys_id), reg.val);
111 static void dlb2_ldb_port_cq_disable(struct dlb2_hw *hw,
112 struct dlb2_ldb_port *port)
114 union dlb2_lsp_cq_ldb_dsbl reg;
116 reg.field.disabled = 1;
118 DLB2_CSR_WR(hw, DLB2_LSP_CQ_LDB_DSBL(port->id.phys_id), reg.val);
123 static u32 dlb2_ldb_queue_depth(struct dlb2_hw *hw,
124 struct dlb2_ldb_queue *queue)
126 union dlb2_lsp_qid_aqed_active_cnt r0;
127 union dlb2_lsp_qid_atm_active r1;
128 union dlb2_lsp_qid_ldb_enqueue_cnt r2;
130 r0.val = DLB2_CSR_RD(hw,
131 DLB2_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
132 r1.val = DLB2_CSR_RD(hw,
133 DLB2_LSP_QID_ATM_ACTIVE(queue->id.phys_id));
135 r2.val = DLB2_CSR_RD(hw,
136 DLB2_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
138 return r0.field.count + r1.field.count + r2.field.count;
141 static struct dlb2_ldb_queue *
142 dlb2_get_ldb_queue_from_id(struct dlb2_hw *hw,
145 unsigned int vdev_id)
147 struct dlb2_list_entry *iter1;
148 struct dlb2_list_entry *iter2;
149 struct dlb2_function_resources *rsrcs;
150 struct dlb2_hw_domain *domain;
151 struct dlb2_ldb_queue *queue;
155 if (id >= DLB2_MAX_NUM_LDB_QUEUES)
158 rsrcs = (vdev_req) ? &hw->vdev[vdev_id] : &hw->pf;
161 return &hw->rsrcs.ldb_queues[id];
163 DLB2_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
164 DLB2_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter2)
165 if (queue->id.virt_id == id)
169 DLB2_FUNC_LIST_FOR(rsrcs->avail_ldb_queues, queue, iter1)
170 if (queue->id.virt_id == id)
176 static struct dlb2_hw_domain *dlb2_get_domain_from_id(struct dlb2_hw *hw,
179 unsigned int vdev_id)
181 struct dlb2_list_entry *iteration;
182 struct dlb2_function_resources *rsrcs;
183 struct dlb2_hw_domain *domain;
184 RTE_SET_USED(iteration);
186 if (id >= DLB2_MAX_NUM_DOMAINS)
190 return &hw->domains[id];
192 rsrcs = &hw->vdev[vdev_id];
194 DLB2_FUNC_LIST_FOR(rsrcs->used_domains, domain, iteration)
195 if (domain->id.virt_id == id)
201 static int dlb2_port_slot_state_transition(struct dlb2_hw *hw,
202 struct dlb2_ldb_port *port,
203 struct dlb2_ldb_queue *queue,
205 enum dlb2_qid_map_state new_state)
207 enum dlb2_qid_map_state curr_state = port->qid_map[slot].state;
208 struct dlb2_hw_domain *domain;
211 domain_id = port->domain_id.phys_id;
213 domain = dlb2_get_domain_from_id(hw, domain_id, false, 0);
214 if (domain == NULL) {
216 "[%s()] Internal error: unable to find domain %d\n",
217 __func__, domain_id);
221 switch (curr_state) {
222 case DLB2_QUEUE_UNMAPPED:
224 case DLB2_QUEUE_MAPPED:
225 queue->num_mappings++;
226 port->num_mappings++;
228 case DLB2_QUEUE_MAP_IN_PROG:
229 queue->num_pending_additions++;
230 domain->num_pending_additions++;
236 case DLB2_QUEUE_MAPPED:
238 case DLB2_QUEUE_UNMAPPED:
239 queue->num_mappings--;
240 port->num_mappings--;
242 case DLB2_QUEUE_UNMAP_IN_PROG:
243 port->num_pending_removals++;
244 domain->num_pending_removals++;
246 case DLB2_QUEUE_MAPPED:
247 /* Priority change, nothing to update */
253 case DLB2_QUEUE_MAP_IN_PROG:
255 case DLB2_QUEUE_UNMAPPED:
256 queue->num_pending_additions--;
257 domain->num_pending_additions--;
259 case DLB2_QUEUE_MAPPED:
260 queue->num_mappings++;
261 port->num_mappings++;
262 queue->num_pending_additions--;
263 domain->num_pending_additions--;
269 case DLB2_QUEUE_UNMAP_IN_PROG:
271 case DLB2_QUEUE_UNMAPPED:
272 port->num_pending_removals--;
273 domain->num_pending_removals--;
274 queue->num_mappings--;
275 port->num_mappings--;
277 case DLB2_QUEUE_MAPPED:
278 port->num_pending_removals--;
279 domain->num_pending_removals--;
281 case DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP:
282 /* Nothing to update */
288 case DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP:
290 case DLB2_QUEUE_UNMAP_IN_PROG:
291 /* Nothing to update */
293 case DLB2_QUEUE_UNMAPPED:
295 * An UNMAP_IN_PROG_PENDING_MAP slot briefly
296 * becomes UNMAPPED before it transitions to
299 queue->num_mappings--;
300 port->num_mappings--;
301 port->num_pending_removals--;
302 domain->num_pending_removals--;
312 port->qid_map[slot].state = new_state;
315 "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
316 __func__, queue->id.phys_id, port->id.phys_id,
317 curr_state, new_state);
322 "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
323 __func__, queue->id.phys_id, port->id.phys_id,
324 curr_state, new_state);
328 static bool dlb2_port_find_slot(struct dlb2_ldb_port *port,
329 enum dlb2_qid_map_state state,
334 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
335 if (port->qid_map[i].state == state)
341 return (i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ);
344 static bool dlb2_port_find_slot_queue(struct dlb2_ldb_port *port,
345 enum dlb2_qid_map_state state,
346 struct dlb2_ldb_queue *queue,
351 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
352 if (port->qid_map[i].state == state &&
353 port->qid_map[i].qid == queue->id.phys_id)
359 return (i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ);
363 * dlb2_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as
364 * their function names imply, and should only be called by the dynamic CQ
367 static void dlb2_ldb_queue_disable_mapped_cqs(struct dlb2_hw *hw,
368 struct dlb2_hw_domain *domain,
369 struct dlb2_ldb_queue *queue)
371 struct dlb2_list_entry *iter;
372 struct dlb2_ldb_port *port;
376 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
377 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter) {
378 enum dlb2_qid_map_state state = DLB2_QUEUE_MAPPED;
380 if (!dlb2_port_find_slot_queue(port, state,
385 dlb2_ldb_port_cq_disable(hw, port);
390 static void dlb2_ldb_queue_enable_mapped_cqs(struct dlb2_hw *hw,
391 struct dlb2_hw_domain *domain,
392 struct dlb2_ldb_queue *queue)
394 struct dlb2_list_entry *iter;
395 struct dlb2_ldb_port *port;
399 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
400 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter) {
401 enum dlb2_qid_map_state state = DLB2_QUEUE_MAPPED;
403 if (!dlb2_port_find_slot_queue(port, state,
408 dlb2_ldb_port_cq_enable(hw, port);
413 static void dlb2_ldb_port_clear_queue_if_status(struct dlb2_hw *hw,
414 struct dlb2_ldb_port *port,
417 union dlb2_lsp_ldb_sched_ctrl r0 = { {0} };
419 r0.field.cq = port->id.phys_id;
420 r0.field.qidix = slot;
422 r0.field.inflight_ok_v = 1;
424 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r0.val);
429 static void dlb2_ldb_port_set_queue_if_status(struct dlb2_hw *hw,
430 struct dlb2_ldb_port *port,
433 union dlb2_lsp_ldb_sched_ctrl r0 = { {0} };
435 r0.field.cq = port->id.phys_id;
436 r0.field.qidix = slot;
438 r0.field.inflight_ok_v = 1;
440 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r0.val);
445 static int dlb2_ldb_port_map_qid_static(struct dlb2_hw *hw,
446 struct dlb2_ldb_port *p,
447 struct dlb2_ldb_queue *q,
450 union dlb2_lsp_cq2priov r0;
451 union dlb2_lsp_cq2qid0 r1;
452 union dlb2_atm_qid2cqidix_00 r2;
453 union dlb2_lsp_qid2cqidix_00 r3;
454 union dlb2_lsp_qid2cqidix2_00 r4;
455 enum dlb2_qid_map_state state;
458 /* Look for a pending or already mapped slot, else an unused slot */
459 if (!dlb2_port_find_slot_queue(p, DLB2_QUEUE_MAP_IN_PROG, q, &i) &&
460 !dlb2_port_find_slot_queue(p, DLB2_QUEUE_MAPPED, q, &i) &&
461 !dlb2_port_find_slot(p, DLB2_QUEUE_UNMAPPED, &i)) {
463 "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
468 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
470 "[%s():%d] Internal error: port slot tracking failed\n",
475 /* Read-modify-write the priority and valid bit register */
476 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2PRIOV(p->id.phys_id));
478 r0.field.v |= 1 << i;
479 r0.field.prio |= (priority & 0x7) << i * 3;
481 DLB2_CSR_WR(hw, DLB2_LSP_CQ2PRIOV(p->id.phys_id), r0.val);
483 /* Read-modify-write the QID map register */
485 r1.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2QID0(p->id.phys_id));
487 r1.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2QID1(p->id.phys_id));
489 if (i == 0 || i == 4)
490 r1.field.qid_p0 = q->id.phys_id;
491 if (i == 1 || i == 5)
492 r1.field.qid_p1 = q->id.phys_id;
493 if (i == 2 || i == 6)
494 r1.field.qid_p2 = q->id.phys_id;
495 if (i == 3 || i == 7)
496 r1.field.qid_p3 = q->id.phys_id;
499 DLB2_CSR_WR(hw, DLB2_LSP_CQ2QID0(p->id.phys_id), r1.val);
501 DLB2_CSR_WR(hw, DLB2_LSP_CQ2QID1(p->id.phys_id), r1.val);
503 r2.val = DLB2_CSR_RD(hw,
504 DLB2_ATM_QID2CQIDIX(q->id.phys_id,
507 r3.val = DLB2_CSR_RD(hw,
508 DLB2_LSP_QID2CQIDIX(q->id.phys_id,
511 r4.val = DLB2_CSR_RD(hw,
512 DLB2_LSP_QID2CQIDIX2(q->id.phys_id,
515 switch (p->id.phys_id % 4) {
517 r2.field.cq_p0 |= 1 << i;
518 r3.field.cq_p0 |= 1 << i;
519 r4.field.cq_p0 |= 1 << i;
523 r2.field.cq_p1 |= 1 << i;
524 r3.field.cq_p1 |= 1 << i;
525 r4.field.cq_p1 |= 1 << i;
529 r2.field.cq_p2 |= 1 << i;
530 r3.field.cq_p2 |= 1 << i;
531 r4.field.cq_p2 |= 1 << i;
535 r2.field.cq_p3 |= 1 << i;
536 r3.field.cq_p3 |= 1 << i;
537 r4.field.cq_p3 |= 1 << i;
542 DLB2_ATM_QID2CQIDIX(q->id.phys_id, p->id.phys_id / 4),
546 DLB2_LSP_QID2CQIDIX(q->id.phys_id, p->id.phys_id / 4),
550 DLB2_LSP_QID2CQIDIX2(q->id.phys_id, p->id.phys_id / 4),
555 p->qid_map[i].qid = q->id.phys_id;
556 p->qid_map[i].priority = priority;
558 state = DLB2_QUEUE_MAPPED;
560 return dlb2_port_slot_state_transition(hw, p, q, i, state);
563 static int dlb2_ldb_port_set_has_work_bits(struct dlb2_hw *hw,
564 struct dlb2_ldb_port *port,
565 struct dlb2_ldb_queue *queue,
568 union dlb2_lsp_qid_aqed_active_cnt r0;
569 union dlb2_lsp_qid_ldb_enqueue_cnt r1;
570 union dlb2_lsp_ldb_sched_ctrl r2 = { {0} };
572 /* Set the atomic scheduling haswork bit */
573 r0.val = DLB2_CSR_RD(hw,
574 DLB2_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
576 r2.field.cq = port->id.phys_id;
577 r2.field.qidix = slot;
579 r2.field.rlist_haswork_v = r0.field.count > 0;
581 /* Set the non-atomic scheduling haswork bit */
582 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r2.val);
584 r1.val = DLB2_CSR_RD(hw,
585 DLB2_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
587 memset(&r2, 0, sizeof(r2));
589 r2.field.cq = port->id.phys_id;
590 r2.field.qidix = slot;
592 r2.field.nalb_haswork_v = (r1.field.count > 0);
594 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r2.val);
601 static void dlb2_ldb_port_clear_has_work_bits(struct dlb2_hw *hw,
602 struct dlb2_ldb_port *port,
605 union dlb2_lsp_ldb_sched_ctrl r2 = { {0} };
607 r2.field.cq = port->id.phys_id;
608 r2.field.qidix = slot;
610 r2.field.rlist_haswork_v = 1;
612 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r2.val);
614 memset(&r2, 0, sizeof(r2));
616 r2.field.cq = port->id.phys_id;
617 r2.field.qidix = slot;
619 r2.field.nalb_haswork_v = 1;
621 DLB2_CSR_WR(hw, DLB2_LSP_LDB_SCHED_CTRL, r2.val);
626 static void dlb2_ldb_queue_set_inflight_limit(struct dlb2_hw *hw,
627 struct dlb2_ldb_queue *queue)
629 union dlb2_lsp_qid_ldb_infl_lim r0 = { {0} };
631 r0.field.limit = queue->num_qid_inflights;
633 DLB2_CSR_WR(hw, DLB2_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), r0.val);
636 static void dlb2_ldb_queue_clear_inflight_limit(struct dlb2_hw *hw,
637 struct dlb2_ldb_queue *queue)
640 DLB2_LSP_QID_LDB_INFL_LIM(queue->id.phys_id),
641 DLB2_LSP_QID_LDB_INFL_LIM_RST);
644 static int dlb2_ldb_port_finish_map_qid_dynamic(struct dlb2_hw *hw,
645 struct dlb2_hw_domain *domain,
646 struct dlb2_ldb_port *port,
647 struct dlb2_ldb_queue *queue)
649 struct dlb2_list_entry *iter;
650 union dlb2_lsp_qid_ldb_infl_cnt r0;
651 enum dlb2_qid_map_state state;
656 r0.val = DLB2_CSR_RD(hw,
657 DLB2_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
659 if (r0.field.count) {
661 "[%s()] Internal error: non-zero QID inflight count\n",
667 * Static map the port and set its corresponding has_work bits.
669 state = DLB2_QUEUE_MAP_IN_PROG;
670 if (!dlb2_port_find_slot_queue(port, state, queue, &slot))
673 if (slot >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
675 "[%s():%d] Internal error: port slot tracking failed\n",
680 prio = port->qid_map[slot].priority;
683 * Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
684 * the port's qid_map state.
686 ret = dlb2_ldb_port_map_qid_static(hw, port, queue, prio);
690 ret = dlb2_ldb_port_set_has_work_bits(hw, port, queue, slot);
695 * Ensure IF_status(cq,qid) is 0 before enabling the port to
696 * prevent spurious schedules to cause the queue's inflight
699 dlb2_ldb_port_clear_queue_if_status(hw, port, slot);
701 /* Reset the queue's inflight status */
702 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
703 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter) {
704 state = DLB2_QUEUE_MAPPED;
705 if (!dlb2_port_find_slot_queue(port, state,
709 dlb2_ldb_port_set_queue_if_status(hw, port, slot);
713 dlb2_ldb_queue_set_inflight_limit(hw, queue);
715 /* Re-enable CQs mapped to this queue */
716 dlb2_ldb_queue_enable_mapped_cqs(hw, domain, queue);
718 /* If this queue has other mappings pending, clear its inflight limit */
719 if (queue->num_pending_additions > 0)
720 dlb2_ldb_queue_clear_inflight_limit(hw, queue);
726 * dlb2_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
727 * @hw: dlb2_hw handle for a particular device.
728 * @port: load-balanced port
729 * @queue: load-balanced queue
730 * @priority: queue servicing priority
732 * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
733 * at a later point, and <0 if an error occurred.
735 static int dlb2_ldb_port_map_qid_dynamic(struct dlb2_hw *hw,
736 struct dlb2_ldb_port *port,
737 struct dlb2_ldb_queue *queue,
740 union dlb2_lsp_qid_ldb_infl_cnt r0 = { {0} };
741 enum dlb2_qid_map_state state;
742 struct dlb2_hw_domain *domain;
743 int domain_id, slot, ret;
745 domain_id = port->domain_id.phys_id;
747 domain = dlb2_get_domain_from_id(hw, domain_id, false, 0);
748 if (domain == NULL) {
750 "[%s()] Internal error: unable to find domain %d\n",
751 __func__, port->domain_id.phys_id);
756 * Set the QID inflight limit to 0 to prevent further scheduling of the
759 DLB2_CSR_WR(hw, DLB2_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), 0);
761 if (!dlb2_port_find_slot(port, DLB2_QUEUE_UNMAPPED, &slot)) {
763 "Internal error: No available unmapped slots\n");
767 if (slot >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
769 "[%s():%d] Internal error: port slot tracking failed\n",
774 port->qid_map[slot].qid = queue->id.phys_id;
775 port->qid_map[slot].priority = priority;
777 state = DLB2_QUEUE_MAP_IN_PROG;
778 ret = dlb2_port_slot_state_transition(hw, port, queue, slot, state);
782 r0.val = DLB2_CSR_RD(hw,
783 DLB2_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
785 if (r0.field.count) {
787 * The queue is owed completions so it's not safe to map it
788 * yet. Schedule a kernel thread to complete the mapping later,
789 * once software has completed all the queue's inflight events.
791 if (!os_worker_active(hw))
792 os_schedule_work(hw);
798 * Disable the affected CQ, and the CQs already mapped to the QID,
799 * before reading the QID's inflight count a second time. There is an
800 * unlikely race in which the QID may schedule one more QE after we
801 * read an inflight count of 0, and disabling the CQs guarantees that
802 * the race will not occur after a re-read of the inflight count
806 dlb2_ldb_port_cq_disable(hw, port);
808 dlb2_ldb_queue_disable_mapped_cqs(hw, domain, queue);
810 r0.val = DLB2_CSR_RD(hw,
811 DLB2_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
813 if (r0.field.count) {
815 dlb2_ldb_port_cq_enable(hw, port);
817 dlb2_ldb_queue_enable_mapped_cqs(hw, domain, queue);
820 * The queue is owed completions so it's not safe to map it
821 * yet. Schedule a kernel thread to complete the mapping later,
822 * once software has completed all the queue's inflight events.
824 if (!os_worker_active(hw))
825 os_schedule_work(hw);
830 return dlb2_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
833 static void dlb2_domain_finish_map_port(struct dlb2_hw *hw,
834 struct dlb2_hw_domain *domain,
835 struct dlb2_ldb_port *port)
839 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
840 union dlb2_lsp_qid_ldb_infl_cnt r0;
841 struct dlb2_ldb_queue *queue;
844 if (port->qid_map[i].state != DLB2_QUEUE_MAP_IN_PROG)
847 qid = port->qid_map[i].qid;
849 queue = dlb2_get_ldb_queue_from_id(hw, qid, false, 0);
853 "[%s()] Internal error: unable to find queue %d\n",
858 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_QID_LDB_INFL_CNT(qid));
864 * Disable the affected CQ, and the CQs already mapped to the
865 * QID, before reading the QID's inflight count a second time.
866 * There is an unlikely race in which the QID may schedule one
867 * more QE after we read an inflight count of 0, and disabling
868 * the CQs guarantees that the race will not occur after a
869 * re-read of the inflight count register.
872 dlb2_ldb_port_cq_disable(hw, port);
874 dlb2_ldb_queue_disable_mapped_cqs(hw, domain, queue);
876 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_QID_LDB_INFL_CNT(qid));
878 if (r0.field.count) {
880 dlb2_ldb_port_cq_enable(hw, port);
882 dlb2_ldb_queue_enable_mapped_cqs(hw, domain, queue);
887 dlb2_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
892 dlb2_domain_finish_map_qid_procedures(struct dlb2_hw *hw,
893 struct dlb2_hw_domain *domain)
895 struct dlb2_list_entry *iter;
896 struct dlb2_ldb_port *port;
900 if (!domain->configured || domain->num_pending_additions == 0)
903 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
904 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter)
905 dlb2_domain_finish_map_port(hw, domain, port);
908 return domain->num_pending_additions;
911 static int dlb2_ldb_port_unmap_qid(struct dlb2_hw *hw,
912 struct dlb2_ldb_port *port,
913 struct dlb2_ldb_queue *queue)
915 enum dlb2_qid_map_state mapped, in_progress, pending_map, unmapped;
916 union dlb2_lsp_cq2priov r0;
917 union dlb2_atm_qid2cqidix_00 r1;
918 union dlb2_lsp_qid2cqidix_00 r2;
919 union dlb2_lsp_qid2cqidix2_00 r3;
924 /* Find the queue's slot */
925 mapped = DLB2_QUEUE_MAPPED;
926 in_progress = DLB2_QUEUE_UNMAP_IN_PROG;
927 pending_map = DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP;
929 if (!dlb2_port_find_slot_queue(port, mapped, queue, &i) &&
930 !dlb2_port_find_slot_queue(port, in_progress, queue, &i) &&
931 !dlb2_port_find_slot_queue(port, pending_map, queue, &i)) {
933 "[%s():%d] Internal error: QID %d isn't mapped\n",
934 __func__, __LINE__, queue->id.phys_id);
938 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
940 "[%s():%d] Internal error: port slot tracking failed\n",
945 port_id = port->id.phys_id;
946 queue_id = queue->id.phys_id;
948 /* Read-modify-write the priority and valid bit register */
949 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2PRIOV(port_id));
951 r0.field.v &= ~(1 << i);
953 DLB2_CSR_WR(hw, DLB2_LSP_CQ2PRIOV(port_id), r0.val);
955 r1.val = DLB2_CSR_RD(hw,
956 DLB2_ATM_QID2CQIDIX(queue_id, port_id / 4));
958 r2.val = DLB2_CSR_RD(hw,
959 DLB2_LSP_QID2CQIDIX(queue_id, port_id / 4));
961 r3.val = DLB2_CSR_RD(hw,
962 DLB2_LSP_QID2CQIDIX2(queue_id, port_id / 4));
964 switch (port_id % 4) {
966 r1.field.cq_p0 &= ~(1 << i);
967 r2.field.cq_p0 &= ~(1 << i);
968 r3.field.cq_p0 &= ~(1 << i);
972 r1.field.cq_p1 &= ~(1 << i);
973 r2.field.cq_p1 &= ~(1 << i);
974 r3.field.cq_p1 &= ~(1 << i);
978 r1.field.cq_p2 &= ~(1 << i);
979 r2.field.cq_p2 &= ~(1 << i);
980 r3.field.cq_p2 &= ~(1 << i);
984 r1.field.cq_p3 &= ~(1 << i);
985 r2.field.cq_p3 &= ~(1 << i);
986 r3.field.cq_p3 &= ~(1 << i);
991 DLB2_ATM_QID2CQIDIX(queue_id, port_id / 4),
995 DLB2_LSP_QID2CQIDIX(queue_id, port_id / 4),
999 DLB2_LSP_QID2CQIDIX2(queue_id, port_id / 4),
1004 unmapped = DLB2_QUEUE_UNMAPPED;
1006 return dlb2_port_slot_state_transition(hw, port, queue, i, unmapped);
1009 static int dlb2_ldb_port_map_qid(struct dlb2_hw *hw,
1010 struct dlb2_hw_domain *domain,
1011 struct dlb2_ldb_port *port,
1012 struct dlb2_ldb_queue *queue,
1015 if (domain->started)
1016 return dlb2_ldb_port_map_qid_dynamic(hw, port, queue, prio);
1018 return dlb2_ldb_port_map_qid_static(hw, port, queue, prio);
1022 dlb2_domain_finish_unmap_port_slot(struct dlb2_hw *hw,
1023 struct dlb2_hw_domain *domain,
1024 struct dlb2_ldb_port *port,
1027 enum dlb2_qid_map_state state;
1028 struct dlb2_ldb_queue *queue;
1030 queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
1032 state = port->qid_map[slot].state;
1034 /* Update the QID2CQIDX and CQ2QID vectors */
1035 dlb2_ldb_port_unmap_qid(hw, port, queue);
1038 * Ensure the QID will not be serviced by this {CQ, slot} by clearing
1041 dlb2_ldb_port_clear_has_work_bits(hw, port, slot);
1043 /* Reset the {CQ, slot} to its default state */
1044 dlb2_ldb_port_set_queue_if_status(hw, port, slot);
1046 /* Re-enable the CQ if it wasn't manually disabled by the user */
1048 dlb2_ldb_port_cq_enable(hw, port);
1051 * If there is a mapping that is pending this slot's removal, perform
1054 if (state == DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP) {
1055 struct dlb2_ldb_port_qid_map *map;
1056 struct dlb2_ldb_queue *map_queue;
1059 map = &port->qid_map[slot];
1061 map->qid = map->pending_qid;
1062 map->priority = map->pending_priority;
1064 map_queue = &hw->rsrcs.ldb_queues[map->qid];
1065 prio = map->priority;
1067 dlb2_ldb_port_map_qid(hw, domain, port, map_queue, prio);
1071 static bool dlb2_domain_finish_unmap_port(struct dlb2_hw *hw,
1072 struct dlb2_hw_domain *domain,
1073 struct dlb2_ldb_port *port)
1075 union dlb2_lsp_cq_ldb_infl_cnt r0;
1078 if (port->num_pending_removals == 0)
1082 * The unmap requires all the CQ's outstanding inflights to be
1085 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ_LDB_INFL_CNT(port->id.phys_id));
1086 if (r0.field.count > 0)
1089 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
1090 struct dlb2_ldb_port_qid_map *map;
1092 map = &port->qid_map[i];
1094 if (map->state != DLB2_QUEUE_UNMAP_IN_PROG &&
1095 map->state != DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP)
1098 dlb2_domain_finish_unmap_port_slot(hw, domain, port, i);
1105 dlb2_domain_finish_unmap_qid_procedures(struct dlb2_hw *hw,
1106 struct dlb2_hw_domain *domain)
1108 struct dlb2_list_entry *iter;
1109 struct dlb2_ldb_port *port;
1113 if (!domain->configured || domain->num_pending_removals == 0)
1116 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
1117 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter)
1118 dlb2_domain_finish_unmap_port(hw, domain, port);
1121 return domain->num_pending_removals;
1124 unsigned int dlb2_finish_unmap_qid_procedures(struct dlb2_hw *hw)
1128 /* Finish queue unmap jobs for any domain that needs it */
1129 for (i = 0; i < DLB2_MAX_NUM_DOMAINS; i++) {
1130 struct dlb2_hw_domain *domain = &hw->domains[i];
1132 num += dlb2_domain_finish_unmap_qid_procedures(hw, domain);
1138 unsigned int dlb2_finish_map_qid_procedures(struct dlb2_hw *hw)
1142 /* Finish queue map jobs for any domain that needs it */
1143 for (i = 0; i < DLB2_MAX_NUM_DOMAINS; i++) {
1144 struct dlb2_hw_domain *domain = &hw->domains[i];
1146 num += dlb2_domain_finish_map_qid_procedures(hw, domain);
1152 int dlb2_get_group_sequence_numbers(struct dlb2_hw *hw, unsigned int group_id)
1154 if (group_id >= DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
1157 return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
1160 int dlb2_get_group_sequence_number_occupancy(struct dlb2_hw *hw,
1161 unsigned int group_id)
1163 if (group_id >= DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
1166 return dlb2_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
1169 static void dlb2_log_set_group_sequence_numbers(struct dlb2_hw *hw,
1170 unsigned int group_id,
1173 DLB2_HW_DBG(hw, "DLB2 set group sequence numbers:\n");
1174 DLB2_HW_DBG(hw, "\tGroup ID: %u\n", group_id);
1175 DLB2_HW_DBG(hw, "\tValue: %lu\n", val);
1178 int dlb2_set_group_sequence_numbers(struct dlb2_hw *hw,
1179 unsigned int group_id,
1182 u32 valid_allocations[] = {64, 128, 256, 512, 1024};
1183 union dlb2_ro_pipe_grp_sn_mode r0 = { {0} };
1184 struct dlb2_sn_group *group;
1187 if (group_id >= DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
1190 group = &hw->rsrcs.sn_groups[group_id];
1193 * Once the first load-balanced queue using an SN group is configured,
1194 * the group cannot be changed.
1196 if (group->slot_use_bitmap != 0)
1199 for (mode = 0; mode < DLB2_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
1200 if (val == valid_allocations[mode])
1203 if (mode == DLB2_MAX_NUM_SEQUENCE_NUMBER_MODES)
1207 group->sequence_numbers_per_queue = val;
1209 r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
1210 r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
1212 DLB2_CSR_WR(hw, DLB2_RO_PIPE_GRP_SN_MODE, r0.val);
1214 dlb2_log_set_group_sequence_numbers(hw, group_id, val);
1220 dlb2_log_create_dir_port_args(struct dlb2_hw *hw,
1222 uintptr_t cq_dma_base,
1223 struct dlb2_create_dir_port_args *args,
1225 unsigned int vdev_id)
1227 DLB2_HW_DBG(hw, "DLB2 create directed port arguments:\n");
1229 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
1230 DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
1232 DLB2_HW_DBG(hw, "\tCQ depth: %d\n",
1234 DLB2_HW_DBG(hw, "\tCQ base address: 0x%lx\n",
1238 static struct dlb2_dir_pq_pair *
1239 dlb2_get_domain_used_dir_pq(struct dlb2_hw *hw,
1242 struct dlb2_hw_domain *domain)
1244 struct dlb2_list_entry *iter;
1245 struct dlb2_dir_pq_pair *port;
1248 if (id >= DLB2_MAX_NUM_DIR_PORTS(hw->ver))
1251 DLB2_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
1252 if ((!vdev_req && port->id.phys_id == id) ||
1253 (vdev_req && port->id.virt_id == id))
1260 dlb2_verify_create_dir_port_args(struct dlb2_hw *hw,
1262 uintptr_t cq_dma_base,
1263 struct dlb2_create_dir_port_args *args,
1264 struct dlb2_cmd_response *resp,
1266 unsigned int vdev_id)
1268 struct dlb2_hw_domain *domain;
1270 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
1272 if (domain == NULL) {
1273 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
1277 if (!domain->configured) {
1278 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
1282 if (domain->started) {
1283 resp->status = DLB2_ST_DOMAIN_STARTED;
1288 * If the user claims the queue is already configured, validate
1289 * the queue ID, its domain, and whether the queue is configured.
1291 if (args->queue_id != -1) {
1292 struct dlb2_dir_pq_pair *queue;
1294 queue = dlb2_get_domain_used_dir_pq(hw,
1299 if (queue == NULL || queue->domain_id.phys_id !=
1300 domain->id.phys_id ||
1301 !queue->queue_configured) {
1302 resp->status = DLB2_ST_INVALID_DIR_QUEUE_ID;
1308 * If the port's queue is not configured, validate that a free
1309 * port-queue pair is available.
1311 if (args->queue_id == -1 &&
1312 dlb2_list_empty(&domain->avail_dir_pq_pairs)) {
1313 resp->status = DLB2_ST_DIR_PORTS_UNAVAILABLE;
1317 /* Check cache-line alignment */
1318 if ((cq_dma_base & 0x3F) != 0) {
1319 resp->status = DLB2_ST_INVALID_CQ_VIRT_ADDR;
1323 if (args->cq_depth != 1 &&
1324 args->cq_depth != 2 &&
1325 args->cq_depth != 4 &&
1326 args->cq_depth != 8 &&
1327 args->cq_depth != 16 &&
1328 args->cq_depth != 32 &&
1329 args->cq_depth != 64 &&
1330 args->cq_depth != 128 &&
1331 args->cq_depth != 256 &&
1332 args->cq_depth != 512 &&
1333 args->cq_depth != 1024) {
1334 resp->status = DLB2_ST_INVALID_CQ_DEPTH;
1341 static void dlb2_dir_port_configure_pp(struct dlb2_hw *hw,
1342 struct dlb2_hw_domain *domain,
1343 struct dlb2_dir_pq_pair *port,
1345 unsigned int vdev_id)
1347 union dlb2_sys_dir_pp2vas r0 = { {0} };
1348 union dlb2_sys_dir_pp_v r4 = { {0} };
1350 r0.field.vas = domain->id.phys_id;
1352 DLB2_CSR_WR(hw, DLB2_SYS_DIR_PP2VAS(port->id.phys_id), r0.val);
1355 union dlb2_sys_vf_dir_vpp2pp r1 = { {0} };
1356 union dlb2_sys_dir_pp2vdev r2 = { {0} };
1357 union dlb2_sys_vf_dir_vpp_v r3 = { {0} };
1362 * DLB uses producer port address bits 17:12 to determine the
1363 * producer port ID. In Scalable IOV mode, PP accesses come
1364 * through the PF MMIO window for the physical producer port,
1365 * so for translation purposes the virtual and physical port
1368 if (hw->virt_mode == DLB2_VIRT_SRIOV)
1369 virt_id = port->id.virt_id;
1371 virt_id = port->id.phys_id;
1373 r1.field.pp = port->id.phys_id;
1375 offs = vdev_id * DLB2_MAX_NUM_DIR_PORTS(hw->ver) + virt_id;
1377 DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VPP2PP(offs), r1.val);
1379 r2.field.vdev = vdev_id;
1382 DLB2_SYS_DIR_PP2VDEV(port->id.phys_id),
1387 DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VPP_V(offs), r3.val);
1393 DLB2_SYS_DIR_PP_V(port->id.phys_id),
1397 static int dlb2_dir_port_configure_cq(struct dlb2_hw *hw,
1398 struct dlb2_hw_domain *domain,
1399 struct dlb2_dir_pq_pair *port,
1400 uintptr_t cq_dma_base,
1401 struct dlb2_create_dir_port_args *args,
1403 unsigned int vdev_id)
1405 union dlb2_sys_dir_cq_addr_l r0 = { {0} };
1406 union dlb2_sys_dir_cq_addr_u r1 = { {0} };
1407 union dlb2_sys_dir_cq2vf_pf_ro r2 = { {0} };
1408 union dlb2_chp_dir_cq_tkn_depth_sel r3 = { {0} };
1409 union dlb2_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
1410 union dlb2_sys_dir_cq_fmt r9 = { {0} };
1411 union dlb2_sys_dir_cq_at r10 = { {0} };
1412 union dlb2_sys_dir_cq_pasid r11 = { {0} };
1413 union dlb2_chp_dir_cq2vas r12 = { {0} };
1415 /* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
1416 r0.field.addr_l = cq_dma_base >> 6;
1418 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_ADDR_L(port->id.phys_id), r0.val);
1420 r1.field.addr_u = cq_dma_base >> 32;
1422 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_ADDR_U(port->id.phys_id), r1.val);
1425 * 'ro' == relaxed ordering. This setting allows DLB2 to write
1426 * cache lines out-of-order (but QEs within a cache line are always
1427 * updated in-order).
1429 r2.field.vf = vdev_id;
1430 r2.field.is_pf = !vdev_req && (hw->virt_mode != DLB2_VIRT_SIOV);
1433 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ2VF_PF_RO(port->id.phys_id), r2.val);
1435 if (args->cq_depth <= 8) {
1436 r3.field.token_depth_select = 1;
1437 } else if (args->cq_depth == 16) {
1438 r3.field.token_depth_select = 2;
1439 } else if (args->cq_depth == 32) {
1440 r3.field.token_depth_select = 3;
1441 } else if (args->cq_depth == 64) {
1442 r3.field.token_depth_select = 4;
1443 } else if (args->cq_depth == 128) {
1444 r3.field.token_depth_select = 5;
1445 } else if (args->cq_depth == 256) {
1446 r3.field.token_depth_select = 6;
1447 } else if (args->cq_depth == 512) {
1448 r3.field.token_depth_select = 7;
1449 } else if (args->cq_depth == 1024) {
1450 r3.field.token_depth_select = 8;
1453 "[%s():%d] Internal error: invalid CQ depth\n",
1454 __func__, __LINE__);
1459 DLB2_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id.phys_id),
1463 * To support CQs with depth less than 8, program the token count
1464 * register with a non-zero initial value. Operations such as domain
1465 * reset must take this initial value into account when quiescing the
1468 port->init_tkn_cnt = 0;
1470 if (args->cq_depth < 8) {
1471 union dlb2_lsp_cq_dir_tkn_cnt r13 = { {0} };
1473 port->init_tkn_cnt = 8 - args->cq_depth;
1475 r13.field.count = port->init_tkn_cnt;
1478 DLB2_LSP_CQ_DIR_TKN_CNT(port->id.phys_id),
1482 DLB2_LSP_CQ_DIR_TKN_CNT(port->id.phys_id),
1483 DLB2_LSP_CQ_DIR_TKN_CNT_RST);
1486 r4.field.token_depth_select = r3.field.token_depth_select;
1487 r4.field.disable_wb_opt = 0;
1488 r4.field.ignore_depth = 0;
1491 DLB2_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id.phys_id),
1494 /* Reset the CQ write pointer */
1496 DLB2_CHP_DIR_CQ_WPTR(port->id.phys_id),
1497 DLB2_CHP_DIR_CQ_WPTR_RST);
1499 /* Virtualize the PPID */
1500 r9.field.keep_pf_ppid = 0;
1502 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_FMT(port->id.phys_id), r9.val);
1505 * Address translation (AT) settings: 0: untranslated, 2: translated
1506 * (see ATS spec regarding Address Type field for more details)
1508 r10.field.cq_at = 0;
1510 DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_AT(port->id.phys_id), r10.val);
1512 if (vdev_req && hw->virt_mode == DLB2_VIRT_SIOV) {
1513 r11.field.pasid = hw->pasid[vdev_id];
1518 DLB2_SYS_DIR_CQ_PASID(port->id.phys_id),
1521 r12.field.cq2vas = domain->id.phys_id;
1523 DLB2_CSR_WR(hw, DLB2_CHP_DIR_CQ2VAS(port->id.phys_id), r12.val);
1528 static int dlb2_configure_dir_port(struct dlb2_hw *hw,
1529 struct dlb2_hw_domain *domain,
1530 struct dlb2_dir_pq_pair *port,
1531 uintptr_t cq_dma_base,
1532 struct dlb2_create_dir_port_args *args,
1534 unsigned int vdev_id)
1538 ret = dlb2_dir_port_configure_cq(hw,
1549 dlb2_dir_port_configure_pp(hw,
1555 dlb2_dir_port_cq_enable(hw, port);
1557 port->enabled = true;
1559 port->port_configured = true;
1565 * dlb2_hw_create_dir_port() - Allocate and initialize a DLB directed port
1566 * and queue. The port/queue pair have the same ID and name.
1567 * @hw: Contains the current state of the DLB2 hardware.
1568 * @domain_id: Domain ID
1569 * @args: User-provided arguments.
1570 * @cq_dma_base: Base DMA address for consumer queue memory
1571 * @resp: Response to user.
1572 * @vdev_req: Request came from a virtual device.
1573 * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
1575 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
1576 * satisfy a request, resp->status will be set accordingly.
1578 int dlb2_hw_create_dir_port(struct dlb2_hw *hw,
1580 struct dlb2_create_dir_port_args *args,
1581 uintptr_t cq_dma_base,
1582 struct dlb2_cmd_response *resp,
1584 unsigned int vdev_id)
1586 struct dlb2_dir_pq_pair *port;
1587 struct dlb2_hw_domain *domain;
1590 dlb2_log_create_dir_port_args(hw,
1598 * Verify that hardware resources are available before attempting to
1599 * satisfy the request. This simplifies the error unwinding code.
1601 ret = dlb2_verify_create_dir_port_args(hw,
1611 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
1613 if (args->queue_id != -1)
1614 port = dlb2_get_domain_used_dir_pq(hw,
1619 port = DLB2_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
1623 "[%s():%d] Internal error: no available dir ports\n",
1624 __func__, __LINE__);
1628 ret = dlb2_configure_dir_port(hw,
1639 * Configuration succeeded, so move the resource from the 'avail' to
1640 * the 'used' list (if it's not already there).
1642 if (args->queue_id == -1) {
1643 dlb2_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
1645 dlb2_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
1649 resp->id = (vdev_req) ? port->id.virt_id : port->id.phys_id;
1654 static void dlb2_configure_dir_queue(struct dlb2_hw *hw,
1655 struct dlb2_hw_domain *domain,
1656 struct dlb2_dir_pq_pair *queue,
1657 struct dlb2_create_dir_queue_args *args,
1659 unsigned int vdev_id)
1661 union dlb2_sys_dir_vasqid_v r0 = { {0} };
1662 union dlb2_sys_dir_qid_its r1 = { {0} };
1663 union dlb2_lsp_qid_dir_depth_thrsh r2 = { {0} };
1664 union dlb2_sys_dir_qid_v r5 = { {0} };
1668 /* QID write permissions are turned on when the domain is started */
1669 r0.field.vasqid_v = 0;
1671 offs = domain->id.phys_id * DLB2_MAX_NUM_DIR_QUEUES(hw->ver) +
1674 DLB2_CSR_WR(hw, DLB2_SYS_DIR_VASQID_V(offs), r0.val);
1676 /* Don't timestamp QEs that pass through this queue */
1677 r1.field.qid_its = 0;
1680 DLB2_SYS_DIR_QID_ITS(queue->id.phys_id),
1683 r2.field.thresh = args->depth_threshold;
1686 DLB2_LSP_QID_DIR_DEPTH_THRSH(queue->id.phys_id),
1690 union dlb2_sys_vf_dir_vqid_v r3 = { {0} };
1691 union dlb2_sys_vf_dir_vqid2qid r4 = { {0} };
1693 offs = vdev_id * DLB2_MAX_NUM_DIR_QUEUES(hw->ver)
1694 + queue->id.virt_id;
1696 r3.field.vqid_v = 1;
1698 DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VQID_V(offs), r3.val);
1700 r4.field.qid = queue->id.phys_id;
1702 DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VQID2QID(offs), r4.val);
1707 DLB2_CSR_WR(hw, DLB2_SYS_DIR_QID_V(queue->id.phys_id), r5.val);
1709 queue->queue_configured = true;
1713 dlb2_log_create_dir_queue_args(struct dlb2_hw *hw,
1715 struct dlb2_create_dir_queue_args *args,
1717 unsigned int vdev_id)
1719 DLB2_HW_DBG(hw, "DLB2 create directed queue arguments:\n");
1721 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
1722 DLB2_HW_DBG(hw, "\tDomain ID: %d\n", domain_id);
1723 DLB2_HW_DBG(hw, "\tPort ID: %d\n", args->port_id);
1727 dlb2_verify_create_dir_queue_args(struct dlb2_hw *hw,
1729 struct dlb2_create_dir_queue_args *args,
1730 struct dlb2_cmd_response *resp,
1732 unsigned int vdev_id)
1734 struct dlb2_hw_domain *domain;
1736 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
1738 if (domain == NULL) {
1739 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
1743 if (!domain->configured) {
1744 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
1748 if (domain->started) {
1749 resp->status = DLB2_ST_DOMAIN_STARTED;
1754 * If the user claims the port is already configured, validate the port
1755 * ID, its domain, and whether the port is configured.
1757 if (args->port_id != -1) {
1758 struct dlb2_dir_pq_pair *port;
1760 port = dlb2_get_domain_used_dir_pq(hw,
1765 if (port == NULL || port->domain_id.phys_id !=
1766 domain->id.phys_id || !port->port_configured) {
1767 resp->status = DLB2_ST_INVALID_PORT_ID;
1773 * If the queue's port is not configured, validate that a free
1774 * port-queue pair is available.
1776 if (args->port_id == -1 &&
1777 dlb2_list_empty(&domain->avail_dir_pq_pairs)) {
1778 resp->status = DLB2_ST_DIR_QUEUES_UNAVAILABLE;
1786 * dlb2_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
1787 * @hw: Contains the current state of the DLB2 hardware.
1788 * @domain_id: Domain ID
1789 * @args: User-provided arguments.
1790 * @resp: Response to user.
1791 * @vdev_req: Request came from a virtual device.
1792 * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
1794 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
1795 * satisfy a request, resp->status will be set accordingly.
1797 int dlb2_hw_create_dir_queue(struct dlb2_hw *hw,
1799 struct dlb2_create_dir_queue_args *args,
1800 struct dlb2_cmd_response *resp,
1802 unsigned int vdev_id)
1804 struct dlb2_dir_pq_pair *queue;
1805 struct dlb2_hw_domain *domain;
1808 dlb2_log_create_dir_queue_args(hw, domain_id, args, vdev_req, vdev_id);
1811 * Verify that hardware resources are available before attempting to
1812 * satisfy the request. This simplifies the error unwinding code.
1814 ret = dlb2_verify_create_dir_queue_args(hw,
1823 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
1824 if (domain == NULL) {
1826 "[%s():%d] Internal error: domain not found\n",
1827 __func__, __LINE__);
1831 if (args->port_id != -1)
1832 queue = dlb2_get_domain_used_dir_pq(hw,
1837 queue = DLB2_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
1839 if (queue == NULL) {
1841 "[%s():%d] Internal error: no available dir queues\n",
1842 __func__, __LINE__);
1846 dlb2_configure_dir_queue(hw, domain, queue, args, vdev_req, vdev_id);
1849 * Configuration succeeded, so move the resource from the 'avail' to
1850 * the 'used' list (if it's not already there).
1852 if (args->port_id == -1) {
1853 dlb2_list_del(&domain->avail_dir_pq_pairs,
1854 &queue->domain_list);
1856 dlb2_list_add(&domain->used_dir_pq_pairs,
1857 &queue->domain_list);
1862 resp->id = (vdev_req) ? queue->id.virt_id : queue->id.phys_id;
1868 dlb2_port_find_slot_with_pending_map_queue(struct dlb2_ldb_port *port,
1869 struct dlb2_ldb_queue *queue,
1874 for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
1875 struct dlb2_ldb_port_qid_map *map = &port->qid_map[i];
1877 if (map->state == DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP &&
1878 map->pending_qid == queue->id.phys_id)
1884 return (i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ);
1887 static void dlb2_ldb_port_change_qid_priority(struct dlb2_hw *hw,
1888 struct dlb2_ldb_port *port,
1890 struct dlb2_map_qid_args *args)
1892 union dlb2_lsp_cq2priov r0;
1894 /* Read-modify-write the priority and valid bit register */
1895 r0.val = DLB2_CSR_RD(hw, DLB2_LSP_CQ2PRIOV(port->id.phys_id));
1897 r0.field.v |= 1 << slot;
1898 r0.field.prio |= (args->priority & 0x7) << slot * 3;
1900 DLB2_CSR_WR(hw, DLB2_LSP_CQ2PRIOV(port->id.phys_id), r0.val);
1904 port->qid_map[slot].priority = args->priority;
1907 static int dlb2_verify_map_qid_slot_available(struct dlb2_ldb_port *port,
1908 struct dlb2_ldb_queue *queue,
1909 struct dlb2_cmd_response *resp)
1911 enum dlb2_qid_map_state state;
1914 /* Unused slot available? */
1915 if (port->num_mappings < DLB2_MAX_NUM_QIDS_PER_LDB_CQ)
1919 * If the queue is already mapped (from the application's perspective),
1920 * this is simply a priority update.
1922 state = DLB2_QUEUE_MAPPED;
1923 if (dlb2_port_find_slot_queue(port, state, queue, &i))
1926 state = DLB2_QUEUE_MAP_IN_PROG;
1927 if (dlb2_port_find_slot_queue(port, state, queue, &i))
1930 if (dlb2_port_find_slot_with_pending_map_queue(port, queue, &i))
1934 * If the slot contains an unmap in progress, it's considered
1937 state = DLB2_QUEUE_UNMAP_IN_PROG;
1938 if (dlb2_port_find_slot(port, state, &i))
1941 state = DLB2_QUEUE_UNMAPPED;
1942 if (dlb2_port_find_slot(port, state, &i))
1945 resp->status = DLB2_ST_NO_QID_SLOTS_AVAILABLE;
1949 static struct dlb2_ldb_queue *
1950 dlb2_get_domain_ldb_queue(u32 id,
1952 struct dlb2_hw_domain *domain)
1954 struct dlb2_list_entry *iter;
1955 struct dlb2_ldb_queue *queue;
1958 if (id >= DLB2_MAX_NUM_LDB_QUEUES)
1961 DLB2_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
1962 if ((!vdev_req && queue->id.phys_id == id) ||
1963 (vdev_req && queue->id.virt_id == id))
1969 static struct dlb2_ldb_port *
1970 dlb2_get_domain_used_ldb_port(u32 id,
1972 struct dlb2_hw_domain *domain)
1974 struct dlb2_list_entry *iter;
1975 struct dlb2_ldb_port *port;
1979 if (id >= DLB2_MAX_NUM_LDB_PORTS)
1982 for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
1983 DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter)
1984 if ((!vdev_req && port->id.phys_id == id) ||
1985 (vdev_req && port->id.virt_id == id))
1988 DLB2_DOM_LIST_FOR(domain->avail_ldb_ports[i], port, iter)
1989 if ((!vdev_req && port->id.phys_id == id) ||
1990 (vdev_req && port->id.virt_id == id))
1997 static int dlb2_verify_map_qid_args(struct dlb2_hw *hw,
1999 struct dlb2_map_qid_args *args,
2000 struct dlb2_cmd_response *resp,
2002 unsigned int vdev_id)
2004 struct dlb2_hw_domain *domain;
2005 struct dlb2_ldb_port *port;
2006 struct dlb2_ldb_queue *queue;
2009 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2011 if (domain == NULL) {
2012 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2016 if (!domain->configured) {
2017 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
2023 port = dlb2_get_domain_used_ldb_port(id, vdev_req, domain);
2025 if (port == NULL || !port->configured) {
2026 resp->status = DLB2_ST_INVALID_PORT_ID;
2030 if (args->priority >= DLB2_QID_PRIORITIES) {
2031 resp->status = DLB2_ST_INVALID_PRIORITY;
2035 queue = dlb2_get_domain_ldb_queue(args->qid, vdev_req, domain);
2037 if (queue == NULL || !queue->configured) {
2038 resp->status = DLB2_ST_INVALID_QID;
2042 if (queue->domain_id.phys_id != domain->id.phys_id) {
2043 resp->status = DLB2_ST_INVALID_QID;
2047 if (port->domain_id.phys_id != domain->id.phys_id) {
2048 resp->status = DLB2_ST_INVALID_PORT_ID;
2055 static void dlb2_log_map_qid(struct dlb2_hw *hw,
2057 struct dlb2_map_qid_args *args,
2059 unsigned int vdev_id)
2061 DLB2_HW_DBG(hw, "DLB2 map QID arguments:\n");
2063 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
2064 DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
2066 DLB2_HW_DBG(hw, "\tPort ID: %d\n",
2068 DLB2_HW_DBG(hw, "\tQueue ID: %d\n",
2070 DLB2_HW_DBG(hw, "\tPriority: %d\n",
2074 int dlb2_hw_map_qid(struct dlb2_hw *hw,
2076 struct dlb2_map_qid_args *args,
2077 struct dlb2_cmd_response *resp,
2079 unsigned int vdev_id)
2081 struct dlb2_hw_domain *domain;
2082 struct dlb2_ldb_queue *queue;
2083 enum dlb2_qid_map_state st;
2084 struct dlb2_ldb_port *port;
2088 dlb2_log_map_qid(hw, domain_id, args, vdev_req, vdev_id);
2091 * Verify that hardware resources are available before attempting to
2092 * satisfy the request. This simplifies the error unwinding code.
2094 ret = dlb2_verify_map_qid_args(hw,
2103 prio = args->priority;
2105 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2106 if (domain == NULL) {
2108 "[%s():%d] Internal error: domain not found\n",
2109 __func__, __LINE__);
2115 port = dlb2_get_domain_used_ldb_port(id, vdev_req, domain);
2118 "[%s():%d] Internal error: port not found\n",
2119 __func__, __LINE__);
2123 queue = dlb2_get_domain_ldb_queue(args->qid, vdev_req, domain);
2124 if (queue == NULL) {
2126 "[%s():%d] Internal error: queue not found\n",
2127 __func__, __LINE__);
2132 * If there are any outstanding detach operations for this port,
2133 * attempt to complete them. This may be necessary to free up a QID
2134 * slot for this requested mapping.
2136 if (port->num_pending_removals)
2137 dlb2_domain_finish_unmap_port(hw, domain, port);
2139 ret = dlb2_verify_map_qid_slot_available(port, queue, resp);
2143 /* Hardware requires disabling the CQ before mapping QIDs. */
2145 dlb2_ldb_port_cq_disable(hw, port);
2148 * If this is only a priority change, don't perform the full QID->CQ
2151 st = DLB2_QUEUE_MAPPED;
2152 if (dlb2_port_find_slot_queue(port, st, queue, &i)) {
2153 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2155 "[%s():%d] Internal error: port slot tracking failed\n",
2156 __func__, __LINE__);
2160 if (prio != port->qid_map[i].priority) {
2161 dlb2_ldb_port_change_qid_priority(hw, port, i, args);
2162 DLB2_HW_DBG(hw, "DLB2 map: priority change\n");
2165 st = DLB2_QUEUE_MAPPED;
2166 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
2173 st = DLB2_QUEUE_UNMAP_IN_PROG;
2174 if (dlb2_port_find_slot_queue(port, st, queue, &i)) {
2175 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2177 "[%s():%d] Internal error: port slot tracking failed\n",
2178 __func__, __LINE__);
2182 if (prio != port->qid_map[i].priority) {
2183 dlb2_ldb_port_change_qid_priority(hw, port, i, args);
2184 DLB2_HW_DBG(hw, "DLB2 map: priority change\n");
2187 st = DLB2_QUEUE_MAPPED;
2188 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
2196 * If this is a priority change on an in-progress mapping, don't
2197 * perform the full QID->CQ mapping procedure.
2199 st = DLB2_QUEUE_MAP_IN_PROG;
2200 if (dlb2_port_find_slot_queue(port, st, queue, &i)) {
2201 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2203 "[%s():%d] Internal error: port slot tracking failed\n",
2204 __func__, __LINE__);
2208 port->qid_map[i].priority = prio;
2210 DLB2_HW_DBG(hw, "DLB2 map: priority change only\n");
2216 * If this is a priority change on a pending mapping, update the
2219 if (dlb2_port_find_slot_with_pending_map_queue(port, queue, &i)) {
2220 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2222 "[%s():%d] Internal error: port slot tracking failed\n",
2223 __func__, __LINE__);
2227 port->qid_map[i].pending_priority = prio;
2229 DLB2_HW_DBG(hw, "DLB2 map: priority change only\n");
2235 * If all the CQ's slots are in use, then there's an unmap in progress
2236 * (guaranteed by dlb2_verify_map_qid_slot_available()), so add this
2237 * mapping to pending_map and return. When the removal is completed for
2238 * the slot's current occupant, this mapping will be performed.
2240 if (!dlb2_port_find_slot(port, DLB2_QUEUE_UNMAPPED, &i)) {
2241 if (dlb2_port_find_slot(port, DLB2_QUEUE_UNMAP_IN_PROG, &i)) {
2242 enum dlb2_qid_map_state st;
2244 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2246 "[%s():%d] Internal error: port slot tracking failed\n",
2247 __func__, __LINE__);
2251 port->qid_map[i].pending_qid = queue->id.phys_id;
2252 port->qid_map[i].pending_priority = prio;
2254 st = DLB2_QUEUE_UNMAP_IN_PROG_PENDING_MAP;
2256 ret = dlb2_port_slot_state_transition(hw, port, queue,
2261 DLB2_HW_DBG(hw, "DLB2 map: map pending removal\n");
2268 * If the domain has started, a special "dynamic" CQ->queue mapping
2269 * procedure is required in order to safely update the CQ<->QID tables.
2270 * The "static" procedure cannot be used when traffic is flowing,
2271 * because the CQ<->QID tables cannot be updated atomically and the
2272 * scheduler won't see the new mapping unless the queue's if_status
2273 * changes, which isn't guaranteed.
2275 ret = dlb2_ldb_port_map_qid(hw, domain, port, queue, prio);
2277 /* If ret is less than zero, it's due to an internal error */
2283 dlb2_ldb_port_cq_enable(hw, port);
2290 static void dlb2_log_unmap_qid(struct dlb2_hw *hw,
2292 struct dlb2_unmap_qid_args *args,
2294 unsigned int vdev_id)
2296 DLB2_HW_DBG(hw, "DLB2 unmap QID arguments:\n");
2298 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
2299 DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
2301 DLB2_HW_DBG(hw, "\tPort ID: %d\n",
2303 DLB2_HW_DBG(hw, "\tQueue ID: %d\n",
2305 if (args->qid < DLB2_MAX_NUM_LDB_QUEUES)
2306 DLB2_HW_DBG(hw, "\tQueue's num mappings: %d\n",
2307 hw->rsrcs.ldb_queues[args->qid].num_mappings);
2310 static int dlb2_verify_unmap_qid_args(struct dlb2_hw *hw,
2312 struct dlb2_unmap_qid_args *args,
2313 struct dlb2_cmd_response *resp,
2315 unsigned int vdev_id)
2317 enum dlb2_qid_map_state state;
2318 struct dlb2_hw_domain *domain;
2319 struct dlb2_ldb_queue *queue;
2320 struct dlb2_ldb_port *port;
2324 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2326 if (domain == NULL) {
2327 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2331 if (!domain->configured) {
2332 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
2338 port = dlb2_get_domain_used_ldb_port(id, vdev_req, domain);
2340 if (port == NULL || !port->configured) {
2341 resp->status = DLB2_ST_INVALID_PORT_ID;
2345 if (port->domain_id.phys_id != domain->id.phys_id) {
2346 resp->status = DLB2_ST_INVALID_PORT_ID;
2350 queue = dlb2_get_domain_ldb_queue(args->qid, vdev_req, domain);
2352 if (queue == NULL || !queue->configured) {
2353 DLB2_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
2354 __func__, args->qid);
2355 resp->status = DLB2_ST_INVALID_QID;
2360 * Verify that the port has the queue mapped. From the application's
2361 * perspective a queue is mapped if it is actually mapped, the map is
2362 * in progress, or the map is blocked pending an unmap.
2364 state = DLB2_QUEUE_MAPPED;
2365 if (dlb2_port_find_slot_queue(port, state, queue, &slot))
2368 state = DLB2_QUEUE_MAP_IN_PROG;
2369 if (dlb2_port_find_slot_queue(port, state, queue, &slot))
2372 if (dlb2_port_find_slot_with_pending_map_queue(port, queue, &slot))
2375 resp->status = DLB2_ST_INVALID_QID;
2379 int dlb2_hw_unmap_qid(struct dlb2_hw *hw,
2381 struct dlb2_unmap_qid_args *args,
2382 struct dlb2_cmd_response *resp,
2384 unsigned int vdev_id)
2386 struct dlb2_hw_domain *domain;
2387 struct dlb2_ldb_queue *queue;
2388 enum dlb2_qid_map_state st;
2389 struct dlb2_ldb_port *port;
2390 bool unmap_complete;
2393 dlb2_log_unmap_qid(hw, domain_id, args, vdev_req, vdev_id);
2396 * Verify that hardware resources are available before attempting to
2397 * satisfy the request. This simplifies the error unwinding code.
2399 ret = dlb2_verify_unmap_qid_args(hw,
2408 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2409 if (domain == NULL) {
2411 "[%s():%d] Internal error: domain not found\n",
2412 __func__, __LINE__);
2418 port = dlb2_get_domain_used_ldb_port(id, vdev_req, domain);
2421 "[%s():%d] Internal error: port not found\n",
2422 __func__, __LINE__);
2426 queue = dlb2_get_domain_ldb_queue(args->qid, vdev_req, domain);
2427 if (queue == NULL) {
2429 "[%s():%d] Internal error: queue not found\n",
2430 __func__, __LINE__);
2435 * If the queue hasn't been mapped yet, we need to update the slot's
2436 * state and re-enable the queue's inflights.
2438 st = DLB2_QUEUE_MAP_IN_PROG;
2439 if (dlb2_port_find_slot_queue(port, st, queue, &i)) {
2440 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2442 "[%s():%d] Internal error: port slot tracking failed\n",
2443 __func__, __LINE__);
2448 * Since the in-progress map was aborted, re-enable the QID's
2451 if (queue->num_pending_additions == 0)
2452 dlb2_ldb_queue_set_inflight_limit(hw, queue);
2454 st = DLB2_QUEUE_UNMAPPED;
2455 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
2459 goto unmap_qid_done;
2463 * If the queue mapping is on hold pending an unmap, we simply need to
2464 * update the slot's state.
2466 if (dlb2_port_find_slot_with_pending_map_queue(port, queue, &i)) {
2467 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2469 "[%s():%d] Internal error: port slot tracking failed\n",
2470 __func__, __LINE__);
2474 st = DLB2_QUEUE_UNMAP_IN_PROG;
2475 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
2479 goto unmap_qid_done;
2482 st = DLB2_QUEUE_MAPPED;
2483 if (!dlb2_port_find_slot_queue(port, st, queue, &i)) {
2485 "[%s()] Internal error: no available CQ slots\n",
2490 if (i >= DLB2_MAX_NUM_QIDS_PER_LDB_CQ) {
2492 "[%s():%d] Internal error: port slot tracking failed\n",
2493 __func__, __LINE__);
2498 * QID->CQ mapping removal is an asynchronous procedure. It requires
2499 * stopping the DLB2 from scheduling this CQ, draining all inflights
2500 * from the CQ, then unmapping the queue from the CQ. This function
2501 * simply marks the port as needing the queue unmapped, and (if
2502 * necessary) starts the unmapping worker thread.
2504 dlb2_ldb_port_cq_disable(hw, port);
2506 st = DLB2_QUEUE_UNMAP_IN_PROG;
2507 ret = dlb2_port_slot_state_transition(hw, port, queue, i, st);
2512 * Attempt to finish the unmapping now, in case the port has no
2513 * outstanding inflights. If that's not the case, this will fail and
2514 * the unmapping will be completed at a later time.
2516 unmap_complete = dlb2_domain_finish_unmap_port(hw, domain, port);
2519 * If the unmapping couldn't complete immediately, launch the worker
2520 * thread (if it isn't already launched) to finish it later.
2522 if (!unmap_complete && !os_worker_active(hw))
2523 os_schedule_work(hw);
2532 dlb2_log_pending_port_unmaps_args(struct dlb2_hw *hw,
2533 struct dlb2_pending_port_unmaps_args *args,
2535 unsigned int vdev_id)
2537 DLB2_HW_DBG(hw, "DLB unmaps in progress arguments:\n");
2539 DLB2_HW_DBG(hw, "(Request from VF %d)\n", vdev_id);
2540 DLB2_HW_DBG(hw, "\tPort ID: %d\n", args->port_id);
2543 int dlb2_hw_pending_port_unmaps(struct dlb2_hw *hw,
2545 struct dlb2_pending_port_unmaps_args *args,
2546 struct dlb2_cmd_response *resp,
2548 unsigned int vdev_id)
2550 struct dlb2_hw_domain *domain;
2551 struct dlb2_ldb_port *port;
2553 dlb2_log_pending_port_unmaps_args(hw, args, vdev_req, vdev_id);
2555 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2557 if (domain == NULL) {
2558 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2562 port = dlb2_get_domain_used_ldb_port(args->port_id, vdev_req, domain);
2563 if (port == NULL || !port->configured) {
2564 resp->status = DLB2_ST_INVALID_PORT_ID;
2568 resp->id = port->num_pending_removals;
2573 static int dlb2_verify_start_domain_args(struct dlb2_hw *hw,
2575 struct dlb2_cmd_response *resp,
2577 unsigned int vdev_id)
2579 struct dlb2_hw_domain *domain;
2581 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2583 if (domain == NULL) {
2584 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2588 if (!domain->configured) {
2589 resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
2593 if (domain->started) {
2594 resp->status = DLB2_ST_DOMAIN_STARTED;
2601 static void dlb2_log_start_domain(struct dlb2_hw *hw,
2604 unsigned int vdev_id)
2606 DLB2_HW_DBG(hw, "DLB2 start domain arguments:\n");
2608 DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
2609 DLB2_HW_DBG(hw, "\tDomain ID: %d\n", domain_id);
2613 * dlb2_hw_start_domain() - Lock the domain configuration
2614 * @hw: Contains the current state of the DLB2 hardware.
2615 * @domain_id: Domain ID
2616 * @arg: User-provided arguments (unused, here for ioctl callback template).
2617 * @resp: Response to user.
2618 * @vdev_req: Request came from a virtual device.
2619 * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
2621 * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
2622 * satisfy a request, resp->status will be set accordingly.
2625 dlb2_hw_start_domain(struct dlb2_hw *hw,
2627 struct dlb2_start_domain_args *arg,
2628 struct dlb2_cmd_response *resp,
2630 unsigned int vdev_id)
2632 struct dlb2_list_entry *iter;
2633 struct dlb2_dir_pq_pair *dir_queue;
2634 struct dlb2_ldb_queue *ldb_queue;
2635 struct dlb2_hw_domain *domain;
2640 dlb2_log_start_domain(hw, domain_id, vdev_req, vdev_id);
2642 ret = dlb2_verify_start_domain_args(hw,
2650 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2651 if (domain == NULL) {
2653 "[%s():%d] Internal error: domain not found\n",
2654 __func__, __LINE__);
2659 * Enable load-balanced and directed queue write permissions for the
2660 * queues this domain owns. Without this, the DLB2 will drop all
2661 * incoming traffic to those queues.
2663 DLB2_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
2664 union dlb2_sys_ldb_vasqid_v r0 = { {0} };
2667 r0.field.vasqid_v = 1;
2669 offs = domain->id.phys_id * DLB2_MAX_NUM_LDB_QUEUES +
2670 ldb_queue->id.phys_id;
2672 DLB2_CSR_WR(hw, DLB2_SYS_LDB_VASQID_V(offs), r0.val);
2675 DLB2_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
2676 union dlb2_sys_dir_vasqid_v r0 = { {0} };
2679 r0.field.vasqid_v = 1;
2681 offs = domain->id.phys_id * DLB2_MAX_NUM_DIR_PORTS(hw->ver) +
2682 dir_queue->id.phys_id;
2684 DLB2_CSR_WR(hw, DLB2_SYS_DIR_VASQID_V(offs), r0.val);
2689 domain->started = true;
2696 static void dlb2_log_get_dir_queue_depth(struct dlb2_hw *hw,
2702 DLB2_HW_DBG(hw, "DLB get directed queue depth:\n");
2704 DLB2_HW_DBG(hw, "(Request from VF %d)\n", vf_id);
2705 DLB2_HW_DBG(hw, "\tDomain ID: %d\n", domain_id);
2706 DLB2_HW_DBG(hw, "\tQueue ID: %d\n", queue_id);
2709 int dlb2_hw_get_dir_queue_depth(struct dlb2_hw *hw,
2711 struct dlb2_get_dir_queue_depth_args *args,
2712 struct dlb2_cmd_response *resp,
2714 unsigned int vdev_id)
2716 struct dlb2_dir_pq_pair *queue;
2717 struct dlb2_hw_domain *domain;
2722 dlb2_log_get_dir_queue_depth(hw, domain_id, args->queue_id,
2725 domain = dlb2_get_domain_from_id(hw, id, vdev_req, vdev_id);
2726 if (domain == NULL) {
2727 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2731 id = args->queue_id;
2733 queue = dlb2_get_domain_used_dir_pq(hw, id, vdev_req, domain);
2734 if (queue == NULL) {
2735 resp->status = DLB2_ST_INVALID_QID;
2739 resp->id = dlb2_dir_queue_depth(hw, queue);
2744 static void dlb2_log_get_ldb_queue_depth(struct dlb2_hw *hw,
2750 DLB2_HW_DBG(hw, "DLB get load-balanced queue depth:\n");
2752 DLB2_HW_DBG(hw, "(Request from VF %d)\n", vf_id);
2753 DLB2_HW_DBG(hw, "\tDomain ID: %d\n", domain_id);
2754 DLB2_HW_DBG(hw, "\tQueue ID: %d\n", queue_id);
2757 int dlb2_hw_get_ldb_queue_depth(struct dlb2_hw *hw,
2759 struct dlb2_get_ldb_queue_depth_args *args,
2760 struct dlb2_cmd_response *resp,
2762 unsigned int vdev_id)
2764 struct dlb2_hw_domain *domain;
2765 struct dlb2_ldb_queue *queue;
2767 dlb2_log_get_ldb_queue_depth(hw, domain_id, args->queue_id,
2770 domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
2771 if (domain == NULL) {
2772 resp->status = DLB2_ST_INVALID_DOMAIN_ID;
2776 queue = dlb2_get_domain_ldb_queue(args->queue_id, vdev_req, domain);
2777 if (queue == NULL) {
2778 resp->status = DLB2_ST_INVALID_QID;
2782 resp->id = dlb2_ldb_queue_depth(hw, queue);