1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
8 npc_prep_mcam_ldata(uint8_t *ptr, const uint8_t *data, int len)
12 for (idx = 0; idx < len; idx++)
13 ptr[idx] = data[len - 1 - idx];
17 npc_check_copysz(size_t size, size_t len)
25 npc_mem_is_zero(const void *mem, int len)
30 for (i = 0; i < len; i++) {
38 npc_set_hw_mask(struct npc_parse_item_info *info, struct npc_xtract_info *xinfo,
44 if (xinfo->enable == 0)
47 if (xinfo->hdr_off < info->hw_hdr_len)
50 max_off = xinfo->hdr_off + xinfo->len - info->hw_hdr_len;
52 if (max_off > info->len)
55 offset = xinfo->hdr_off - info->hw_hdr_len;
56 for (j = offset; j < max_off; j++)
61 npc_get_hw_supp_mask(struct npc_parse_state *pst,
62 struct npc_parse_item_info *info, int lid, int lt)
64 struct npc_xtract_info *xinfo, *lfinfo;
65 char *hw_mask = info->hw_mask;
71 xinfo = pst->npc->prx_dxcfg[intf][lid][lt].xtract;
72 memset(hw_mask, 0, info->len);
74 for (i = 0; i < NPC_MAX_LD; i++)
75 npc_set_hw_mask(info, &xinfo[i], hw_mask);
77 for (i = 0; i < NPC_MAX_LD; i++) {
78 if (xinfo[i].flags_enable == 0)
81 lf_cfg = pst->npc->prx_lfcfg[i].i;
83 for (j = 0; j < NPC_MAX_LFL; j++) {
84 lfinfo = pst->npc->prx_fxcfg[intf][i][j].xtract;
85 npc_set_hw_mask(info, &lfinfo[0], hw_mask);
92 npc_mask_is_supported(const char *mask, const char *hw_mask, int len)
95 * If no hw_mask, assume nothing is supported.
99 return npc_mem_is_zero(mask, len);
102 if ((mask[len] | hw_mask[len]) != hw_mask[len])
103 return 0; /* False */
109 npc_parse_item_basic(const struct roc_npc_item_info *item,
110 struct npc_parse_item_info *info)
112 /* Item must not be NULL */
114 return NPC_ERR_PARAM;
116 /* Don't support ranges */
117 if (item->last != NULL)
118 return NPC_ERR_INVALID_RANGE;
120 /* If spec is NULL, both mask and last must be NULL, this
121 * makes it to match ANY value (eq to mask = 0).
122 * Setting either mask or last without spec is an error
124 if (item->spec == NULL) {
125 if (item->last == NULL && item->mask == NULL) {
129 return NPC_ERR_INVALID_SPEC;
132 /* We have valid spec */
133 if (item->type != ROC_NPC_ITEM_TYPE_RAW)
134 info->spec = item->spec;
136 /* If mask is not set, use default mask, err if default mask is
139 if (item->mask == NULL) {
140 if (info->def_mask == NULL)
141 return NPC_ERR_PARAM;
142 info->mask = info->def_mask;
144 if (item->type != ROC_NPC_ITEM_TYPE_RAW)
145 info->mask = item->mask;
148 /* mask specified must be subset of hw supported mask
149 * mask | hw_mask == hw_mask
151 if (!npc_mask_is_supported(info->mask, info->hw_mask, info->len))
152 return NPC_ERR_INVALID_MASK;
158 npc_update_extraction_data(struct npc_parse_state *pst,
159 struct npc_parse_item_info *info,
160 struct npc_xtract_info *xinfo)
162 uint8_t int_info_mask[NPC_MAX_EXTRACT_DATA_LEN];
163 uint8_t int_info[NPC_MAX_EXTRACT_DATA_LEN];
164 struct npc_xtract_info *x;
170 hdr_off = x->hdr_off;
172 if (hdr_off < info->hw_hdr_len)
178 hdr_off -= info->hw_hdr_len;
180 if (hdr_off >= info->len)
183 if (hdr_off + len > info->len)
184 len = info->len - hdr_off;
186 len = npc_check_copysz((ROC_NPC_MAX_MCAM_WIDTH_DWORDS * 8) - x->key_off,
189 return NPC_ERR_INVALID_SIZE;
191 /* Need to reverse complete structure so that dest addr is at
192 * MSB so as to program the MCAM using mcam_data & mcam_mask
195 npc_prep_mcam_ldata(int_info, (const uint8_t *)info->spec + hdr_off,
197 npc_prep_mcam_ldata(int_info_mask,
198 (const uint8_t *)info->mask + hdr_off, x->len);
200 memcpy(pst->mcam_mask + x->key_off, int_info_mask, len);
201 memcpy(pst->mcam_data + x->key_off, int_info, len);
206 npc_update_parse_state(struct npc_parse_state *pst,
207 struct npc_parse_item_info *info, int lid, int lt,
210 struct npc_lid_lt_xtract_info *xinfo;
211 struct roc_npc_flow_dump_data *dump;
212 struct npc_xtract_info *lfinfo;
216 pst->layer_mask |= lid;
218 pst->flags[lid] = flags;
220 intf = pst->nix_intf;
221 xinfo = &pst->npc->prx_dxcfg[intf][lid][lt];
222 if (xinfo->is_terminating)
225 if (info->spec == NULL)
228 for (i = 0; i < NPC_MAX_LD; i++) {
229 rc = npc_update_extraction_data(pst, info, &xinfo->xtract[i]);
234 for (i = 0; i < NPC_MAX_LD; i++) {
235 if (xinfo->xtract[i].flags_enable == 0)
238 lf_cfg = pst->npc->prx_lfcfg[i].i;
240 for (j = 0; j < NPC_MAX_LFL; j++) {
241 lfinfo = pst->npc->prx_fxcfg[intf][i][j].xtract;
242 rc = npc_update_extraction_data(pst, info,
247 if (lfinfo[0].enable)
254 dump = &pst->flow->dump_data[pst->flow->num_patterns++];
262 npc_first_set_bit(uint64_t slab)
266 if ((slab & 0xffffffff) == 0) {
270 if ((slab & 0xffff) == 0) {
274 if ((slab & 0xff) == 0) {
278 if ((slab & 0xf) == 0) {
282 if ((slab & 0x3) == 0) {
286 if ((slab & 0x1) == 0)
293 npc_shift_lv_ent(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc,
294 uint32_t old_ent, uint32_t new_ent)
296 struct npc_mcam_shift_entry_req *req;
297 struct npc_mcam_shift_entry_rsp *rsp;
298 struct npc_flow_list *list;
299 struct roc_npc_flow *flow_iter;
302 list = &npc->flow_list[flow->priority];
304 /* Old entry is disabled & it's contents are moved to new_entry,
305 * new entry is enabled finally.
307 req = mbox_alloc_msg_npc_mcam_shift_entry(mbox);
310 req->curr_entry[0] = old_ent;
311 req->new_entry[0] = new_ent;
312 req->shift_count = 1;
314 rc = mbox_process_msg(mbox, (void *)&rsp);
318 /* Remove old node from list */
319 TAILQ_FOREACH(flow_iter, list, next) {
320 if (flow_iter->mcam_id == old_ent)
321 TAILQ_REMOVE(list, flow_iter, next);
324 /* Insert node with new mcam id at right place */
325 TAILQ_FOREACH(flow_iter, list, next) {
326 if (flow_iter->mcam_id > new_ent)
327 TAILQ_INSERT_BEFORE(flow_iter, flow, next);
332 /* Exchange all required entries with a given priority level */
334 npc_shift_ent(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc,
335 struct npc_mcam_alloc_entry_rsp *rsp, int dir, int prio_lvl)
337 struct plt_bitmap *fr_bmp, *fr_bmp_rev, *lv_bmp, *lv_bmp_rev, *bmp;
338 uint32_t e_fr = 0, e_lv = 0, e, e_id = 0, mcam_entries;
339 uint64_t fr_bit_pos = 0, lv_bit_pos = 0, bit_pos = 0;
340 /* Bit position within the slab */
341 uint32_t sl_fr_bit_off = 0, sl_lv_bit_off = 0;
342 /* Overall bit position of the start of slab */
343 /* free & live entry index */
344 int rc_fr = 0, rc_lv = 0, rc = 0, idx = 0;
345 struct npc_mcam_ents_info *ent_info;
346 /* free & live bitmap slab */
347 uint64_t sl_fr = 0, sl_lv = 0, *sl;
349 fr_bmp = npc->free_entries[prio_lvl];
350 fr_bmp_rev = npc->free_entries_rev[prio_lvl];
351 lv_bmp = npc->live_entries[prio_lvl];
352 lv_bmp_rev = npc->live_entries_rev[prio_lvl];
353 ent_info = &npc->flow_entry_info[prio_lvl];
354 mcam_entries = npc->mcam_entries;
356 /* New entries allocated are always contiguous, but older entries
357 * already in free/live bitmap can be non-contiguous: so return
358 * shifted entries should be in non-contiguous format.
360 while (idx <= rsp->count) {
361 if (!sl_fr && !sl_lv) {
362 /* Lower index elements to be exchanged */
364 rc_fr = plt_bitmap_scan(fr_bmp, &e_fr, &sl_fr);
365 rc_lv = plt_bitmap_scan(lv_bmp, &e_lv, &sl_lv);
367 rc_fr = plt_bitmap_scan(fr_bmp_rev,
368 &sl_fr_bit_off, &sl_fr);
369 rc_lv = plt_bitmap_scan(lv_bmp_rev,
370 &sl_lv_bit_off, &sl_lv);
375 fr_bit_pos = npc_first_set_bit(sl_fr);
376 e_fr = sl_fr_bit_off + fr_bit_pos;
382 lv_bit_pos = npc_first_set_bit(sl_lv);
383 e_lv = sl_lv_bit_off + lv_bit_pos;
388 /* First entry is from free_bmap */
393 bit_pos = fr_bit_pos;
395 e_id = mcam_entries - e - 1;
402 bit_pos = lv_bit_pos;
404 e_id = mcam_entries - e - 1;
408 if (idx < rsp->count)
409 rc = npc_shift_lv_ent(mbox, flow, npc, e_id,
413 plt_bitmap_clear(bmp, e);
414 plt_bitmap_set(bmp, rsp->entry + idx);
415 /* Update entry list, use non-contiguous
418 rsp->entry_list[idx] = e_id;
419 *sl &= ~(1UL << bit_pos);
421 /* Update min & max entry identifiers in current
425 ent_info->max_id = rsp->entry + idx;
426 ent_info->min_id = e_id;
428 ent_info->max_id = e_id;
429 ent_info->min_id = rsp->entry;
437 /* Validate if newly allocated entries lie in the correct priority zone
438 * since NPC_MCAM_LOWER_PRIO & NPC_MCAM_HIGHER_PRIO don't ensure zone accuracy.
439 * If not properly aligned, shift entries to do so
442 npc_validate_and_shift_prio_ent(struct mbox *mbox, struct roc_npc_flow *flow,
444 struct npc_mcam_alloc_entry_rsp *rsp,
447 int prio_idx = 0, rc = 0, needs_shift = 0, idx, prio = flow->priority;
448 struct npc_mcam_ents_info *info = npc->flow_entry_info;
449 int dir = (req_prio == NPC_MCAM_HIGHER_PRIO) ? 1 : -1;
450 uint32_t tot_ent = 0;
453 prio_idx = npc->flow_max_priority - 1;
455 /* Only live entries needs to be shifted, free entries can just be
456 * moved by bits manipulation.
459 /* For dir = -1(NPC_MCAM_LOWER_PRIO), when shifting,
460 * NPC_MAX_PREALLOC_ENT are exchanged with adjoining higher priority
461 * level entries(lower indexes).
463 * For dir = +1(NPC_MCAM_HIGHER_PRIO), during shift,
464 * NPC_MAX_PREALLOC_ENT are exchanged with adjoining lower priority
465 * level entries(higher indexes) with highest indexes.
468 tot_ent = info[prio_idx].free_ent + info[prio_idx].live_ent;
470 if (dir < 0 && prio_idx != prio &&
471 rsp->entry > info[prio_idx].max_id && tot_ent) {
473 } else if ((dir > 0) && (prio_idx != prio) &&
474 (rsp->entry < info[prio_idx].min_id) && tot_ent) {
480 rc = npc_shift_ent(mbox, flow, npc, rsp, dir, prio_idx);
482 for (idx = 0; idx < rsp->count; idx++)
483 rsp->entry_list[idx] = rsp->entry + idx;
485 } while ((prio_idx != prio) && (prio_idx += dir));
491 npc_find_ref_entry(struct npc *npc, int *prio, int prio_lvl)
493 struct npc_mcam_ents_info *info = npc->flow_entry_info;
496 while (step < npc->flow_max_priority) {
497 if (((prio_lvl + step) < npc->flow_max_priority) &&
498 info[prio_lvl + step].live_ent) {
499 *prio = NPC_MCAM_HIGHER_PRIO;
500 return info[prio_lvl + step].min_id;
503 if (((prio_lvl - step) >= 0) &&
504 info[prio_lvl - step].live_ent) {
505 *prio = NPC_MCAM_LOWER_PRIO;
506 return info[prio_lvl - step].max_id;
510 *prio = NPC_MCAM_ANY_PRIO;
515 npc_fill_entry_cache(struct mbox *mbox, struct roc_npc_flow *flow,
516 struct npc *npc, uint32_t *free_ent)
518 struct plt_bitmap *free_bmp, *free_bmp_rev, *live_bmp, *live_bmp_rev;
519 struct npc_mcam_alloc_entry_rsp rsp_local;
520 struct npc_mcam_alloc_entry_rsp *rsp_cmd;
521 struct npc_mcam_alloc_entry_req *req;
522 struct npc_mcam_alloc_entry_rsp *rsp;
523 struct npc_mcam_ents_info *info;
524 int rc = -ENOSPC, prio;
525 uint16_t ref_ent, idx;
527 info = &npc->flow_entry_info[flow->priority];
528 free_bmp = npc->free_entries[flow->priority];
529 free_bmp_rev = npc->free_entries_rev[flow->priority];
530 live_bmp = npc->live_entries[flow->priority];
531 live_bmp_rev = npc->live_entries_rev[flow->priority];
533 ref_ent = npc_find_ref_entry(npc, &prio, flow->priority);
535 req = mbox_alloc_msg_npc_mcam_alloc_entry(mbox);
539 req->count = npc->flow_prealloc_size;
540 req->priority = prio;
541 req->ref_entry = ref_ent;
543 rc = mbox_process_msg(mbox, (void *)&rsp_cmd);
548 memcpy(rsp, rsp_cmd, sizeof(*rsp));
550 /* Non-first ent cache fill */
551 if (prio != NPC_MCAM_ANY_PRIO) {
552 npc_validate_and_shift_prio_ent(mbox, flow, npc, rsp, prio);
554 /* Copy into response entry list */
555 for (idx = 0; idx < rsp->count; idx++)
556 rsp->entry_list[idx] = rsp->entry + idx;
559 /* Update free entries, reverse free entries list,
560 * min & max entry ids.
562 for (idx = 0; idx < rsp->count; idx++) {
563 if (unlikely(rsp->entry_list[idx] < info->min_id))
564 info->min_id = rsp->entry_list[idx];
566 if (unlikely(rsp->entry_list[idx] > info->max_id))
567 info->max_id = rsp->entry_list[idx];
569 /* Skip entry to be returned, not to be part of free
572 if (prio == NPC_MCAM_HIGHER_PRIO) {
573 if (unlikely(idx == (rsp->count - 1))) {
574 *free_ent = rsp->entry_list[idx];
578 if (unlikely(!idx)) {
579 *free_ent = rsp->entry_list[idx];
584 plt_bitmap_set(free_bmp, rsp->entry_list[idx]);
585 plt_bitmap_set(free_bmp_rev,
586 npc->mcam_entries - rsp->entry_list[idx] - 1);
590 plt_bitmap_set(live_bmp, *free_ent);
591 plt_bitmap_set(live_bmp_rev, npc->mcam_entries - *free_ent - 1);
597 npc_check_preallocated_entry_cache(struct mbox *mbox, struct roc_npc_flow *flow,
600 struct plt_bitmap *free, *free_rev, *live, *live_rev;
601 uint32_t pos = 0, free_ent = 0, mcam_entries;
602 struct npc_mcam_ents_info *info;
606 info = &npc->flow_entry_info[flow->priority];
608 free_rev = npc->free_entries_rev[flow->priority];
609 free = npc->free_entries[flow->priority];
610 live_rev = npc->live_entries_rev[flow->priority];
611 live = npc->live_entries[flow->priority];
612 mcam_entries = npc->mcam_entries;
614 if (info->free_ent) {
615 rc = plt_bitmap_scan(free, &pos, &slab);
617 /* Get free_ent from free entry bitmap */
618 free_ent = pos + __builtin_ctzll(slab);
619 /* Remove from free bitmaps and add to live ones */
620 plt_bitmap_clear(free, free_ent);
621 plt_bitmap_set(live, free_ent);
622 plt_bitmap_clear(free_rev, mcam_entries - free_ent - 1);
623 plt_bitmap_set(live_rev, mcam_entries - free_ent - 1);
629 return NPC_ERR_INTERNAL;
632 rc = npc_fill_entry_cache(mbox, flow, npc, &free_ent);