1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
9 roc_nix_tm_sq_aura_fc(struct roc_nix_sq *sq, bool enable)
11 struct npa_aq_enq_req *req;
12 struct npa_aq_enq_rsp *rsp;
18 plt_tm_dbg("Setting SQ %u SQB aura FC to %s", sq->qid,
19 enable ? "enable" : "disable");
21 lf = idev_npa_obj_get();
23 return NPA_ERR_DEVICE_NOT_BOUNDED;
26 /* Set/clear sqb aura fc_ena */
27 aura_handle = sq->aura_handle;
28 req = mbox_alloc_msg_npa_aq_enq(mbox);
32 req->aura_id = roc_npa_aura_handle_to_aura(aura_handle);
33 req->ctype = NPA_AQ_CTYPE_AURA;
34 req->op = NPA_AQ_INSTOP_WRITE;
35 /* Below is not needed for aura writes but AF driver needs it */
36 /* AF will translate to associated poolctx */
37 req->aura.pool_addr = req->aura_id;
39 req->aura.fc_ena = enable;
40 req->aura_mask.fc_ena = 1;
42 rc = mbox_process(mbox);
46 /* Read back npa aura ctx */
47 req = mbox_alloc_msg_npa_aq_enq(mbox);
51 req->aura_id = roc_npa_aura_handle_to_aura(aura_handle);
52 req->ctype = NPA_AQ_CTYPE_AURA;
53 req->op = NPA_AQ_INSTOP_READ;
55 rc = mbox_process_msg(mbox, (void *)&rsp);
59 /* Init when enabled as there might be no triggers */
61 *(volatile uint64_t *)sq->fc = rsp->aura.count;
63 *(volatile uint64_t *)sq->fc = sq->nb_sqb_bufs;
64 /* Sync write barrier */
70 roc_nix_tm_free_resources(struct roc_nix *roc_nix, bool hw_only)
72 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
74 if (nix->tm_flags & NIX_TM_HIERARCHY_ENA)
77 return nix_tm_free_resources(roc_nix, BIT(ROC_NIX_TM_USER), hw_only);
81 nix_tm_shaper_profile_add(struct roc_nix *roc_nix,
82 struct nix_tm_shaper_profile *profile, int skip_ins)
84 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
85 uint64_t commit_rate, commit_sz;
86 uint64_t peak_rate, peak_sz;
90 commit_rate = profile->commit.rate;
91 commit_sz = profile->commit.size;
92 peak_rate = profile->peak.rate;
93 peak_sz = profile->peak.size;
95 if (nix_tm_shaper_profile_search(nix, id) && !skip_ins)
96 return NIX_ERR_TM_SHAPER_PROFILE_EXISTS;
98 if (profile->pkt_len_adj < NIX_TM_LENGTH_ADJUST_MIN ||
99 profile->pkt_len_adj > NIX_TM_LENGTH_ADJUST_MAX)
100 return NIX_ERR_TM_SHAPER_PKT_LEN_ADJUST;
102 /* We cannot support both pkt length adjust and pkt mode */
103 if (profile->pkt_mode && profile->pkt_len_adj)
104 return NIX_ERR_TM_SHAPER_PKT_LEN_ADJUST;
106 /* commit rate and burst size can be enabled/disabled */
107 if (commit_rate || commit_sz) {
108 if (commit_sz < NIX_TM_MIN_SHAPER_BURST ||
109 commit_sz > NIX_TM_MAX_SHAPER_BURST)
110 return NIX_ERR_TM_INVALID_COMMIT_SZ;
111 else if (!nix_tm_shaper_rate_conv(commit_rate, NULL, NULL,
113 return NIX_ERR_TM_INVALID_COMMIT_RATE;
116 /* Peak rate and burst size can be enabled/disabled */
117 if (peak_sz || peak_rate) {
118 if (peak_sz < NIX_TM_MIN_SHAPER_BURST ||
119 peak_sz > NIX_TM_MAX_SHAPER_BURST)
120 return NIX_ERR_TM_INVALID_PEAK_SZ;
121 else if (!nix_tm_shaper_rate_conv(peak_rate, NULL, NULL, NULL))
122 return NIX_ERR_TM_INVALID_PEAK_RATE;
126 TAILQ_INSERT_TAIL(&nix->shaper_profile_list, profile, shaper);
128 plt_tm_dbg("Added TM shaper profile %u, "
129 " pir %" PRIu64 " , pbs %" PRIu64 ", cir %" PRIu64
130 ", cbs %" PRIu64 " , adj %u, pkt_mode %u",
131 id, profile->peak.rate, profile->peak.size,
132 profile->commit.rate, profile->commit.size,
133 profile->pkt_len_adj, profile->pkt_mode);
135 /* Always use PIR for single rate shaping */
136 if (!peak_rate && commit_rate) {
137 profile->peak.rate = profile->commit.rate;
138 profile->peak.size = profile->commit.size;
139 profile->commit.rate = 0;
140 profile->commit.size = 0;
143 /* update min rate */
144 nix->tm_rate_min = nix_tm_shaper_profile_rate_min(nix);
149 roc_nix_tm_shaper_profile_add(struct roc_nix *roc_nix,
150 struct roc_nix_tm_shaper_profile *roc_profile)
152 struct nix_tm_shaper_profile *profile;
154 profile = (struct nix_tm_shaper_profile *)roc_profile->reserved;
156 profile->ref_cnt = 0;
157 profile->id = roc_profile->id;
158 if (roc_profile->pkt_mode) {
159 /* Each packet accomulate single count, whereas HW
160 * considers each unit as Byte, so we need convert
163 profile->commit.rate = roc_profile->commit_rate * 8;
164 profile->peak.rate = roc_profile->peak_rate * 8;
166 profile->commit.rate = roc_profile->commit_rate;
167 profile->peak.rate = roc_profile->peak_rate;
169 profile->commit.size = roc_profile->commit_sz;
170 profile->peak.size = roc_profile->peak_sz;
171 profile->pkt_len_adj = roc_profile->pkt_len_adj;
172 profile->pkt_mode = roc_profile->pkt_mode;
173 profile->free_fn = roc_profile->free_fn;
175 return nix_tm_shaper_profile_add(roc_nix, profile, 0);
179 roc_nix_tm_shaper_profile_update(struct roc_nix *roc_nix,
180 struct roc_nix_tm_shaper_profile *roc_profile)
182 struct nix_tm_shaper_profile *profile;
184 profile = (struct nix_tm_shaper_profile *)roc_profile->reserved;
186 if (roc_profile->pkt_mode) {
187 /* Each packet accomulate single count, whereas HW
188 * considers each unit as Byte, so we need convert
191 profile->commit.rate = roc_profile->commit_rate * 8;
192 profile->peak.rate = roc_profile->peak_rate * 8;
194 profile->commit.rate = roc_profile->commit_rate;
195 profile->peak.rate = roc_profile->peak_rate;
197 profile->commit.size = roc_profile->commit_sz;
198 profile->peak.size = roc_profile->peak_sz;
200 return nix_tm_shaper_profile_add(roc_nix, profile, 1);
204 roc_nix_tm_shaper_profile_delete(struct roc_nix *roc_nix, uint32_t id)
206 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
207 struct nix_tm_shaper_profile *profile;
209 profile = nix_tm_shaper_profile_search(nix, id);
211 return NIX_ERR_TM_INVALID_SHAPER_PROFILE;
213 if (profile->ref_cnt)
214 return NIX_ERR_TM_SHAPER_PROFILE_IN_USE;
216 plt_tm_dbg("Removing TM shaper profile %u", id);
217 TAILQ_REMOVE(&nix->shaper_profile_list, profile, shaper);
218 nix_tm_shaper_profile_free(profile);
220 /* update min rate */
221 nix->tm_rate_min = nix_tm_shaper_profile_rate_min(nix);
226 roc_nix_tm_node_add(struct roc_nix *roc_nix, struct roc_nix_tm_node *roc_node)
228 struct nix_tm_node *node;
230 node = (struct nix_tm_node *)&roc_node->reserved;
231 node->id = roc_node->id;
232 node->priority = roc_node->priority;
233 node->weight = roc_node->weight;
234 node->lvl = roc_node->lvl;
235 node->parent_id = roc_node->parent_id;
236 node->shaper_profile_id = roc_node->shaper_profile_id;
237 node->pkt_mode = roc_node->pkt_mode;
238 node->pkt_mode_set = roc_node->pkt_mode_set;
239 node->free_fn = roc_node->free_fn;
240 node->tree = ROC_NIX_TM_USER;
242 return nix_tm_node_add(roc_nix, node);
246 roc_nix_tm_node_pkt_mode_update(struct roc_nix *roc_nix, uint32_t node_id,
249 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
250 struct nix_tm_node *node, *child;
251 struct nix_tm_node_list *list;
252 int num_children = 0;
254 node = nix_tm_node_search(nix, node_id, ROC_NIX_TM_USER);
256 return NIX_ERR_TM_INVALID_NODE;
258 if (node->pkt_mode == pkt_mode) {
259 node->pkt_mode_set = true;
263 /* Check for any existing children, if there are any,
264 * then we cannot update the pkt mode as children's quantum
265 * are already taken in.
267 list = nix_tm_node_list(nix, ROC_NIX_TM_USER);
268 TAILQ_FOREACH(child, list, node) {
269 if (child->parent == node)
273 /* Cannot update mode if it has children or tree is enabled */
274 if ((nix->tm_flags & NIX_TM_HIERARCHY_ENA) && num_children)
277 if (node->pkt_mode_set && num_children)
278 return NIX_ERR_TM_PKT_MODE_MISMATCH;
280 node->pkt_mode = pkt_mode;
281 node->pkt_mode_set = true;
287 roc_nix_tm_node_name_get(struct roc_nix *roc_nix, uint32_t node_id, char *buf,
290 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
291 struct nix_tm_node *node;
293 node = nix_tm_node_search(nix, node_id, ROC_NIX_TM_USER);
295 plt_strlcpy(buf, "???", buflen);
296 return NIX_ERR_TM_INVALID_NODE;
299 if (node->hw_lvl == NIX_TXSCH_LVL_CNT)
300 snprintf(buf, buflen, "SQ_%d", node->id);
302 snprintf(buf, buflen, "%s_%d", nix_tm_hwlvl2str(node->hw_lvl),
308 roc_nix_tm_node_delete(struct roc_nix *roc_nix, uint32_t node_id, bool free)
310 return nix_tm_node_delete(roc_nix, node_id, ROC_NIX_TM_USER, free);
314 roc_nix_tm_hierarchy_disable(struct roc_nix *roc_nix)
316 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
317 uint16_t sqb_cnt, head_off, tail_off;
318 uint16_t sq_cnt = nix->nb_tx_queues;
319 struct mbox *mbox = (&nix->dev)->mbox;
320 struct nix_tm_node_list *list;
321 enum roc_nix_tm_tree tree;
322 struct nix_tm_node *node;
323 struct roc_nix_sq *sq;
328 if (!(nix->tm_flags & NIX_TM_HIERARCHY_ENA))
331 plt_tm_dbg("Disabling hierarchy on %s", nix->pci_dev->name);
334 list = nix_tm_node_list(nix, tree);
336 /* Enable CGX RXTX to drain pkts */
337 if (!roc_nix->io_enabled) {
338 /* Though it enables both RX MCAM Entries and CGX Link
339 * we assume all the rx queues are stopped way back.
341 mbox_alloc_msg_nix_lf_start_rx(mbox);
342 rc = mbox_process(mbox);
344 plt_err("cgx start failed, rc=%d", rc);
350 TAILQ_FOREACH(node, list, node) {
351 if (node->hw_lvl != NIX_TXSCH_LVL_SMQ)
353 if (!(node->flags & NIX_TM_NODE_HWRES))
356 rc = nix_tm_smq_xoff(nix, node, false);
358 plt_err("Failed to enable smq %u, rc=%d", node->hw_id,
364 /* Flush all tx queues */
365 for (i = 0; i < sq_cnt; i++) {
370 rc = roc_nix_tm_sq_aura_fc(sq, false);
372 plt_err("Failed to disable sqb aura fc, rc=%d", rc);
376 /* Wait for sq entries to be flushed */
377 rc = roc_nix_tm_sq_flush_spin(sq);
379 plt_err("Failed to drain sq, rc=%d\n", rc);
384 /* XOFF & Flush all SMQ's. HRM mandates
385 * all SQ's empty before SMQ flush is issued.
387 TAILQ_FOREACH(node, list, node) {
388 if (node->hw_lvl != NIX_TXSCH_LVL_SMQ)
390 if (!(node->flags & NIX_TM_NODE_HWRES))
393 rc = nix_tm_smq_xoff(nix, node, true);
395 plt_err("Failed to enable smq %u, rc=%d", node->hw_id,
400 node->flags &= ~NIX_TM_NODE_ENABLED;
403 /* Verify sanity of all tx queues */
404 for (i = 0; i < sq_cnt; i++) {
409 wdata = ((uint64_t)sq->qid << 32);
410 regaddr = nix->base + NIX_LF_SQ_OP_STATUS;
411 val = roc_atomic64_add_nosync(wdata, (int64_t *)regaddr);
413 sqb_cnt = val & 0xFFFF;
414 head_off = (val >> 20) & 0x3F;
415 tail_off = (val >> 28) & 0x3F;
417 if (sqb_cnt > 1 || head_off != tail_off ||
418 (*(uint64_t *)sq->fc != sq->nb_sqb_bufs))
419 plt_err("Failed to gracefully flush sq %u", sq->qid);
422 nix->tm_flags &= ~NIX_TM_HIERARCHY_ENA;
424 /* Restore cgx state */
425 if (!roc_nix->io_enabled) {
426 mbox_alloc_msg_nix_lf_stop_rx(mbox);
427 rc |= mbox_process(mbox);
433 roc_nix_tm_hierarchy_enable(struct roc_nix *roc_nix, enum roc_nix_tm_tree tree,
436 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
437 struct nix_tm_node_list *list;
438 struct nix_tm_node *node;
439 struct roc_nix_sq *sq;
444 if (tree >= ROC_NIX_TM_TREE_MAX)
445 return NIX_ERR_PARAM;
447 if (nix->tm_flags & NIX_TM_HIERARCHY_ENA) {
448 if (nix->tm_tree != tree)
453 plt_tm_dbg("Enabling hierarchy on %s, xmit_ena %u, tree %u",
454 nix->pci_dev->name, xmit_enable, tree);
456 /* Free hw resources of other trees */
457 tree_mask = NIX_TM_TREE_MASK_ALL;
458 tree_mask &= ~BIT(tree);
460 rc = nix_tm_free_resources(roc_nix, tree_mask, true);
462 plt_err("failed to free resources of other trees, rc=%d", rc);
466 /* Update active tree before starting to do anything */
469 nix_tm_update_parent_info(nix, tree);
471 rc = nix_tm_alloc_txschq(nix, tree);
473 plt_err("TM failed to alloc tm resources=%d", rc);
477 rc = nix_tm_assign_resources(nix, tree);
479 plt_err("TM failed to assign tm resources=%d", rc);
483 rc = nix_tm_txsch_reg_config(nix, tree);
485 plt_err("TM failed to configure sched registers=%d", rc);
489 list = nix_tm_node_list(nix, tree);
490 /* Mark all non-leaf's as enabled */
491 TAILQ_FOREACH(node, list, node) {
492 if (!nix_tm_is_leaf(nix, node->lvl))
493 node->flags |= NIX_TM_NODE_ENABLED;
499 /* Update SQ Sched Data while SQ is idle */
500 TAILQ_FOREACH(node, list, node) {
501 if (!nix_tm_is_leaf(nix, node->lvl))
504 rc = nix_tm_sq_sched_conf(nix, node, false);
506 plt_err("SQ %u sched update failed, rc=%d", node->id,
512 /* Finally XON all SMQ's */
513 TAILQ_FOREACH(node, list, node) {
514 if (node->hw_lvl != NIX_TXSCH_LVL_SMQ)
517 rc = nix_tm_smq_xoff(nix, node, false);
519 plt_err("Failed to enable smq %u, rc=%d", node->hw_id,
525 /* Enable xmit as all the topology is ready */
526 TAILQ_FOREACH(node, list, node) {
527 if (!nix_tm_is_leaf(nix, node->lvl))
531 sq = nix->sqs[sq_id];
533 rc = roc_nix_tm_sq_aura_fc(sq, true);
535 plt_err("TM sw xon failed on SQ %u, rc=%d", node->id,
539 node->flags |= NIX_TM_NODE_ENABLED;
543 nix->tm_flags |= NIX_TM_HIERARCHY_ENA;