drivers/common/cnxk/roc_npc_utils.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4 #include "roc_api.h"
   5 #include "roc_priv.h"
   6
   7 static void
   8 npc_prep_mcam_ldata(uint8_t *ptr, const uint8_t *data, int len)
   9 {
  10         int idx;
  11
  12         for (idx = 0; idx < len; idx++)
  13                 ptr[idx] = data[len - 1 - idx];
  14 }
  15
  16 static int
  17 npc_check_copysz(size_t size, size_t len)
  18 {
  19         if (len <= size)
  20                 return len;
  21         return NPC_ERR_PARAM;
  22 }
  23
  24 static inline int
  25 npc_mem_is_zero(const void *mem, int len)
  26 {
  27         const char *m = mem;
  28         int i;
  29
  30         for (i = 0; i < len; i++) {
  31                 if (m[i] != 0)
  32                         return 0;
  33         }
  34         return 1;
  35 }
  36
  37 static void
  38 npc_set_hw_mask(struct npc_parse_item_info *info, struct npc_xtract_info *xinfo,
  39                 char *hw_mask)
  40 {
  41         int max_off, offset;
  42         int j;
  43
  44         if (xinfo->enable == 0)
  45                 return;
  46
  47         if (xinfo->hdr_off < info->hw_hdr_len)
  48                 return;
  49
  50         max_off = xinfo->hdr_off + xinfo->len - info->hw_hdr_len;
  51
  52         if (max_off > info->len)
  53                 max_off = info->len;
  54
  55         offset = xinfo->hdr_off - info->hw_hdr_len;
  56         for (j = offset; j < max_off; j++)
  57                 hw_mask[j] = 0xff;
  58 }
  59
  60 void
  61 npc_get_hw_supp_mask(struct npc_parse_state *pst,
  62                      struct npc_parse_item_info *info, int lid, int lt)
  63 {
  64         struct npc_xtract_info *xinfo, *lfinfo;
  65         char *hw_mask = info->hw_mask;
  66         int lf_cfg = 0;
  67         int i, j;
  68         int intf;
  69
  70         intf = pst->nix_intf;
  71         xinfo = pst->npc->prx_dxcfg[intf][lid][lt].xtract;
  72         memset(hw_mask, 0, info->len);
  73
  74         for (i = 0; i < NPC_MAX_LD; i++)
  75                 npc_set_hw_mask(info, &xinfo[i], hw_mask);
  76
  77         for (i = 0; i < NPC_MAX_LD; i++) {
  78                 if (xinfo[i].flags_enable == 0)
  79                         continue;
  80
  81                 lf_cfg = pst->npc->prx_lfcfg[i].i;
  82                 if (lf_cfg == lid) {
  83                         for (j = 0; j < NPC_MAX_LFL; j++) {
  84                                 lfinfo = pst->npc->prx_fxcfg[intf][i][j].xtract;
  85                                 npc_set_hw_mask(info, &lfinfo[0], hw_mask);
  86                         }
  87                 }
  88         }
  89 }
  90
  91 static inline int
  92 npc_mask_is_supported(const char *mask, const char *hw_mask, int len)
  93 {
  94         /*
  95          * If no hw_mask, assume nothing is supported.
  96          * mask is never NULL
  97          */
  98         if (hw_mask == NULL)
  99                 return npc_mem_is_zero(mask, len);
 100
 101         while (len--) {
 102                 if ((mask[len] | hw_mask[len]) != hw_mask[len])
 103                         return 0; /* False */
 104         }
 105         return 1;
 106 }
 107
 108 int
 109 npc_parse_item_basic(const struct roc_npc_item_info *item,
 110                      struct npc_parse_item_info *info)
 111 {
 112         /* Item must not be NULL */
 113         if (item == NULL)
 114                 return NPC_ERR_PARAM;
 115
 116         /* Don't support ranges */
 117         if (item->last != NULL)
 118                 return NPC_ERR_INVALID_RANGE;
 119
 120         /* If spec is NULL, both mask and last must be NULL, this
 121          * makes it to match ANY value (eq to mask = 0).
 122          * Setting either mask or last without spec is an error
 123          */
 124         if (item->spec == NULL) {
 125                 if (item->last == NULL && item->mask == NULL) {
 126                         info->spec = NULL;
 127                         return 0;
 128                 }
 129                 return NPC_ERR_INVALID_SPEC;
 130         }
 131
 132         /* We have valid spec */
 133         if (item->type != ROC_NPC_ITEM_TYPE_RAW)
 134                 info->spec = item->spec;
 135
 136         /* If mask is not set, use default mask, err if default mask is
 137          * also NULL.
 138          */
 139         if (item->mask == NULL) {
 140                 if (info->def_mask == NULL)
 141                         return NPC_ERR_PARAM;
 142                 info->mask = info->def_mask;
 143         } else {
 144                 if (item->type != ROC_NPC_ITEM_TYPE_RAW)
 145                         info->mask = item->mask;
 146         }
 147
 148         /* mask specified must be subset of hw supported mask
 149          * mask | hw_mask == hw_mask
 150          */
 151         if (!npc_mask_is_supported(info->mask, info->hw_mask, info->len))
 152                 return NPC_ERR_INVALID_MASK;
 153
 154         return 0;
 155 }
 156
 157 static int
 158 npc_update_extraction_data(struct npc_parse_state *pst,
 159                            struct npc_parse_item_info *info,
 160                            struct npc_xtract_info *xinfo)
 161 {
 162         uint8_t int_info_mask[NPC_MAX_EXTRACT_DATA_LEN];
 163         uint8_t int_info[NPC_MAX_EXTRACT_DATA_LEN];
 164         struct npc_xtract_info *x;
 165         int hdr_off;
 166         int len = 0;
 167
 168         x = xinfo;
 169         len = x->len;
 170         hdr_off = x->hdr_off;
 171
 172         if (hdr_off < info->hw_hdr_len)
 173                 return 0;
 174
 175         if (x->enable == 0)
 176                 return 0;
 177
 178         hdr_off -= info->hw_hdr_len;
 179
 180         if (hdr_off >= info->len)
 181                 return 0;
 182
 183         if (hdr_off + len > info->len)
 184                 len = info->len - hdr_off;
 185
 186         len = npc_check_copysz((ROC_NPC_MAX_MCAM_WIDTH_DWORDS * 8) - x->key_off,
 187                                len);
 188         if (len < 0)
 189                 return NPC_ERR_INVALID_SIZE;
 190
 191         /* Need to reverse complete structure so that dest addr is at
 192          * MSB so as to program the MCAM using mcam_data & mcam_mask
 193          * arrays
 194          */
 195         npc_prep_mcam_ldata(int_info, (const uint8_t *)info->spec + hdr_off,
 196                             x->len);
 197         npc_prep_mcam_ldata(int_info_mask,
 198                             (const uint8_t *)info->mask + hdr_off, x->len);
 199
 200         memcpy(pst->mcam_mask + x->key_off, int_info_mask, len);
 201         memcpy(pst->mcam_data + x->key_off, int_info, len);
 202         return 0;
 203 }
 204
 205 int
 206 npc_update_parse_state(struct npc_parse_state *pst,
 207                        struct npc_parse_item_info *info, int lid, int lt,
 208                        uint8_t flags)
 209 {
 210         struct npc_lid_lt_xtract_info *xinfo;
 211         struct roc_npc_flow_dump_data *dump;
 212         struct npc_xtract_info *lfinfo;
 213         int intf, lf_cfg;
 214         int i, j, rc = 0;
 215
 216         pst->layer_mask |= lid;
 217         pst->lt[lid] = lt;
 218         pst->flags[lid] = flags;
 219
 220         intf = pst->nix_intf;
 221         xinfo = &pst->npc->prx_dxcfg[intf][lid][lt];
 222         if (xinfo->is_terminating)
 223                 pst->terminate = 1;
 224
 225         if (info->spec == NULL)
 226                 goto done;
 227
 228         for (i = 0; i < NPC_MAX_LD; i++) {
 229                 rc = npc_update_extraction_data(pst, info, &xinfo->xtract[i]);
 230                 if (rc != 0)
 231                         return rc;
 232         }
 233
 234         for (i = 0; i < NPC_MAX_LD; i++) {
 235                 if (xinfo->xtract[i].flags_enable == 0)
 236                         continue;
 237
 238                 lf_cfg = pst->npc->prx_lfcfg[i].i;
 239                 if (lf_cfg == lid) {
 240                         for (j = 0; j < NPC_MAX_LFL; j++) {
 241                                 lfinfo = pst->npc->prx_fxcfg[intf][i][j].xtract;
 242                                 rc = npc_update_extraction_data(pst, info,
 243                                                                 &lfinfo[0]);
 244                                 if (rc != 0)
 245                                         return rc;
 246
 247                                 if (lfinfo[0].enable)
 248                                         pst->flags[lid] = j;
 249                         }
 250                 }
 251         }
 252
 253 done:
 254         dump = &pst->flow->dump_data[pst->flow->num_patterns++];
 255         dump->lid = lid;
 256         dump->ltype = lt;
 257         pst->pattern++;
 258         return 0;
 259 }
 260
 261 static int
 262 npc_first_set_bit(uint64_t slab)
 263 {
 264         int num = 0;
 265
 266         if ((slab & 0xffffffff) == 0) {
 267                 num += 32;
 268                 slab >>= 32;
 269         }
 270         if ((slab & 0xffff) == 0) {
 271                 num += 16;
 272                 slab >>= 16;
 273         }
 274         if ((slab & 0xff) == 0) {
 275                 num += 8;
 276                 slab >>= 8;
 277         }
 278         if ((slab & 0xf) == 0) {
 279                 num += 4;
 280                 slab >>= 4;
 281         }
 282         if ((slab & 0x3) == 0) {
 283                 num += 2;
 284                 slab >>= 2;
 285         }
 286         if ((slab & 0x1) == 0)
 287                 num += 1;
 288
 289         return num;
 290 }
 291
 292 static int
 293 npc_shift_lv_ent(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc,
 294                  uint32_t old_ent, uint32_t new_ent)
 295 {
 296         struct npc_mcam_shift_entry_req *req;
 297         struct npc_mcam_shift_entry_rsp *rsp;
 298         struct npc_flow_list *list;
 299         struct roc_npc_flow *flow_iter;
 300         int rc = -ENOSPC;
 301
 302         list = &npc->flow_list[flow->priority];
 303
 304         /* Old entry is disabled & it's contents are moved to new_entry,
 305          * new entry is enabled finally.
 306          */
 307         req = mbox_alloc_msg_npc_mcam_shift_entry(mbox);
 308         if (req == NULL)
 309                 return rc;
 310         req->curr_entry[0] = old_ent;
 311         req->new_entry[0] = new_ent;
 312         req->shift_count = 1;
 313
 314         rc = mbox_process_msg(mbox, (void *)&rsp);
 315         if (rc)
 316                 return rc;
 317
 318         /* Remove old node from list */
 319         TAILQ_FOREACH(flow_iter, list, next) {
 320                 if (flow_iter->mcam_id == old_ent)
 321                         TAILQ_REMOVE(list, flow_iter, next);
 322         }
 323
 324         /* Insert node with new mcam id at right place */
 325         TAILQ_FOREACH(flow_iter, list, next) {
 326                 if (flow_iter->mcam_id > new_ent)
 327                         TAILQ_INSERT_BEFORE(flow_iter, flow, next);
 328         }
 329         return rc;
 330 }
 331
 332 /* Exchange all required entries with a given priority level */
 333 static int
 334 npc_shift_ent(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc,
 335               struct npc_mcam_alloc_entry_rsp *rsp, int dir, int prio_lvl)
 336 {
 337         struct plt_bitmap *fr_bmp, *fr_bmp_rev, *lv_bmp, *lv_bmp_rev, *bmp;
 338         uint32_t e_fr = 0, e_lv = 0, e, e_id = 0, mcam_entries;
 339         uint64_t fr_bit_pos = 0, lv_bit_pos = 0, bit_pos = 0;
 340         /* Bit position within the slab */
 341         uint32_t sl_fr_bit_off = 0, sl_lv_bit_off = 0;
 342         /* Overall bit position of the start of slab */
 343         /* free & live entry index */
 344         int rc_fr = 0, rc_lv = 0, rc = 0, idx = 0;
 345         struct npc_mcam_ents_info *ent_info;
 346         /* free & live bitmap slab */
 347         uint64_t sl_fr = 0, sl_lv = 0, *sl;
 348
 349         fr_bmp = npc->free_entries[prio_lvl];
 350         fr_bmp_rev = npc->free_entries_rev[prio_lvl];
 351         lv_bmp = npc->live_entries[prio_lvl];
 352         lv_bmp_rev = npc->live_entries_rev[prio_lvl];
 353         ent_info = &npc->flow_entry_info[prio_lvl];
 354         mcam_entries = npc->mcam_entries;
 355
 356         /* New entries allocated are always contiguous, but older entries
 357          * already in free/live bitmap can be non-contiguous: so return
 358          * shifted entries should be in non-contiguous format.
 359          */
 360         while (idx <= rsp->count) {
 361                 if (!sl_fr && !sl_lv) {
 362                         /* Lower index elements to be exchanged */
 363                         if (dir < 0) {
 364                                 rc_fr = plt_bitmap_scan(fr_bmp, &e_fr, &sl_fr);
 365                                 rc_lv = plt_bitmap_scan(lv_bmp, &e_lv, &sl_lv);
 366                         } else {
 367                                 rc_fr = plt_bitmap_scan(fr_bmp_rev,
 368                                                         &sl_fr_bit_off, &sl_fr);
 369                                 rc_lv = plt_bitmap_scan(lv_bmp_rev,
 370                                                         &sl_lv_bit_off, &sl_lv);
 371                         }
 372                 }
 373
 374                 if (rc_fr) {
 375                         fr_bit_pos = npc_first_set_bit(sl_fr);
 376                         e_fr = sl_fr_bit_off + fr_bit_pos;
 377                 } else {
 378                         e_fr = ~(0);
 379                 }
 380
 381                 if (rc_lv) {
 382                         lv_bit_pos = npc_first_set_bit(sl_lv);
 383                         e_lv = sl_lv_bit_off + lv_bit_pos;
 384                 } else {
 385                         e_lv = ~(0);
 386                 }
 387
 388                 /* First entry is from free_bmap */
 389                 if (e_fr < e_lv) {
 390                         bmp = fr_bmp;
 391                         e = e_fr;
 392                         sl = &sl_fr;
 393                         bit_pos = fr_bit_pos;
 394                         if (dir > 0)
 395                                 e_id = mcam_entries - e - 1;
 396                         else
 397                                 e_id = e;
 398                 } else {
 399                         bmp = lv_bmp;
 400                         e = e_lv;
 401                         sl = &sl_lv;
 402                         bit_pos = lv_bit_pos;
 403                         if (dir > 0)
 404                                 e_id = mcam_entries - e - 1;
 405                         else
 406                                 e_id = e;
 407
 408                         if (idx < rsp->count)
 409                                 rc = npc_shift_lv_ent(mbox, flow, npc, e_id,
 410                                                       rsp->entry + idx);
 411                 }
 412
 413                 plt_bitmap_clear(bmp, e);
 414                 plt_bitmap_set(bmp, rsp->entry + idx);
 415                 /* Update entry list, use non-contiguous
 416                  * list now.
 417                  */
 418                 rsp->entry_list[idx] = e_id;
 419                 *sl &= ~(1UL << bit_pos);
 420
 421                 /* Update min & max entry identifiers in current
 422                  * priority level.
 423                  */
 424                 if (dir < 0) {
 425                         ent_info->max_id = rsp->entry + idx;
 426                         ent_info->min_id = e_id;
 427                 } else {
 428                         ent_info->max_id = e_id;
 429                         ent_info->min_id = rsp->entry;
 430                 }
 431
 432                 idx++;
 433         }
 434         return rc;
 435 }
 436
 437 /* Validate if newly allocated entries lie in the correct priority zone
 438  * since NPC_MCAM_LOWER_PRIO & NPC_MCAM_HIGHER_PRIO don't ensure zone accuracy.
 439  * If not properly aligned, shift entries to do so
 440  */
 441 static int
 442 npc_validate_and_shift_prio_ent(struct mbox *mbox, struct roc_npc_flow *flow,
 443                                 struct npc *npc,
 444                                 struct npc_mcam_alloc_entry_rsp *rsp,
 445                                 int req_prio)
 446 {
 447         int prio_idx = 0, rc = 0, needs_shift = 0, idx, prio = flow->priority;
 448         struct npc_mcam_ents_info *info = npc->flow_entry_info;
 449         int dir = (req_prio == NPC_MCAM_HIGHER_PRIO) ? 1 : -1;
 450         uint32_t tot_ent = 0;
 451
 452         if (dir < 0)
 453                 prio_idx = npc->flow_max_priority - 1;
 454
 455         /* Only live entries needs to be shifted, free entries can just be
 456          * moved by bits manipulation.
 457          */
 458
 459         /* For dir = -1(NPC_MCAM_LOWER_PRIO), when shifting,
 460          * NPC_MAX_PREALLOC_ENT are exchanged with adjoining higher priority
 461          * level entries(lower indexes).
 462          *
 463          * For dir = +1(NPC_MCAM_HIGHER_PRIO), during shift,
 464          * NPC_MAX_PREALLOC_ENT are exchanged with adjoining lower priority
 465          * level entries(higher indexes) with highest indexes.
 466          */
 467         do {
 468                 tot_ent = info[prio_idx].free_ent + info[prio_idx].live_ent;
 469
 470                 if (dir < 0 && prio_idx != prio &&
 471                     rsp->entry > info[prio_idx].max_id && tot_ent) {
 472                         needs_shift = 1;
 473                 } else if ((dir > 0) && (prio_idx != prio) &&
 474                            (rsp->entry < info[prio_idx].min_id) && tot_ent) {
 475                         needs_shift = 1;
 476                 }
 477
 478                 if (needs_shift) {
 479                         needs_shift = 0;
 480                         rc = npc_shift_ent(mbox, flow, npc, rsp, dir, prio_idx);
 481                 } else {
 482                         for (idx = 0; idx < rsp->count; idx++)
 483                                 rsp->entry_list[idx] = rsp->entry + idx;
 484                 }
 485         } while ((prio_idx != prio) && (prio_idx += dir));
 486
 487         return rc;
 488 }
 489
 490 static int
 491 npc_find_ref_entry(struct npc *npc, int *prio, int prio_lvl)
 492 {
 493         struct npc_mcam_ents_info *info = npc->flow_entry_info;
 494         int step = 1;
 495
 496         while (step < npc->flow_max_priority) {
 497                 if (((prio_lvl + step) < npc->flow_max_priority) &&
 498                     info[prio_lvl + step].live_ent) {
 499                         *prio = NPC_MCAM_HIGHER_PRIO;
 500                         return info[prio_lvl + step].min_id;
 501                 }
 502
 503                 if (((prio_lvl - step) >= 0) &&
 504                     info[prio_lvl - step].live_ent) {
 505                         *prio = NPC_MCAM_LOWER_PRIO;
 506                         return info[prio_lvl - step].max_id;
 507                 }
 508                 step++;
 509         }
 510         *prio = NPC_MCAM_ANY_PRIO;
 511         return 0;
 512 }
 513
 514 static int
 515 npc_fill_entry_cache(struct mbox *mbox, struct roc_npc_flow *flow,
 516                      struct npc *npc, uint32_t *free_ent)
 517 {
 518         struct plt_bitmap *free_bmp, *free_bmp_rev, *live_bmp, *live_bmp_rev;
 519         struct npc_mcam_alloc_entry_rsp rsp_local;
 520         struct npc_mcam_alloc_entry_rsp *rsp_cmd;
 521         struct npc_mcam_alloc_entry_req *req;
 522         struct npc_mcam_alloc_entry_rsp *rsp;
 523         struct npc_mcam_ents_info *info;
 524         int rc = -ENOSPC, prio;
 525         uint16_t ref_ent, idx;
 526
 527         info = &npc->flow_entry_info[flow->priority];
 528         free_bmp = npc->free_entries[flow->priority];
 529         free_bmp_rev = npc->free_entries_rev[flow->priority];
 530         live_bmp = npc->live_entries[flow->priority];
 531         live_bmp_rev = npc->live_entries_rev[flow->priority];
 532
 533         ref_ent = npc_find_ref_entry(npc, &prio, flow->priority);
 534
 535         req = mbox_alloc_msg_npc_mcam_alloc_entry(mbox);
 536         if (req == NULL)
 537                 return rc;
 538         req->contig = 1;
 539         req->count = npc->flow_prealloc_size;
 540         req->priority = prio;
 541         req->ref_entry = ref_ent;
 542
 543         rc = mbox_process_msg(mbox, (void *)&rsp_cmd);
 544         if (rc)
 545                 return rc;
 546
 547         rsp = &rsp_local;
 548         memcpy(rsp, rsp_cmd, sizeof(*rsp));
 549
 550         /* Non-first ent cache fill */
 551         if (prio != NPC_MCAM_ANY_PRIO) {
 552                 npc_validate_and_shift_prio_ent(mbox, flow, npc, rsp, prio);
 553         } else {
 554                 /* Copy into response entry list */
 555                 for (idx = 0; idx < rsp->count; idx++)
 556                         rsp->entry_list[idx] = rsp->entry + idx;
 557         }
 558
 559         /* Update free entries, reverse free entries list,
 560          * min & max entry ids.
 561          */
 562         for (idx = 0; idx < rsp->count; idx++) {
 563                 if (unlikely(rsp->entry_list[idx] < info->min_id))
 564                         info->min_id = rsp->entry_list[idx];
 565
 566                 if (unlikely(rsp->entry_list[idx] > info->max_id))
 567                         info->max_id = rsp->entry_list[idx];
 568
 569                 /* Skip entry to be returned, not to be part of free
 570                  * list.
 571                  */
 572                 if (prio == NPC_MCAM_HIGHER_PRIO) {
 573                         if (unlikely(idx == (rsp->count - 1))) {
 574                                 *free_ent = rsp->entry_list[idx];
 575                                 continue;
 576                         }
 577                 } else {
 578                         if (unlikely(!idx)) {
 579                                 *free_ent = rsp->entry_list[idx];
 580                                 continue;
 581                         }
 582                 }
 583                 info->free_ent++;
 584                 plt_bitmap_set(free_bmp, rsp->entry_list[idx]);
 585                 plt_bitmap_set(free_bmp_rev,
 586                                npc->mcam_entries - rsp->entry_list[idx] - 1);
 587         }
 588
 589         info->live_ent++;
 590         plt_bitmap_set(live_bmp, *free_ent);
 591         plt_bitmap_set(live_bmp_rev, npc->mcam_entries - *free_ent - 1);
 592
 593         return 0;
 594 }
 595
 596 int
 597 npc_check_preallocated_entry_cache(struct mbox *mbox, struct roc_npc_flow *flow,
 598                                    struct npc *npc)
 599 {
 600         struct plt_bitmap *free, *free_rev, *live, *live_rev;
 601         uint32_t pos = 0, free_ent = 0, mcam_entries;
 602         struct npc_mcam_ents_info *info;
 603         uint64_t slab = 0;
 604         int rc;
 605
 606         info = &npc->flow_entry_info[flow->priority];
 607
 608         free_rev = npc->free_entries_rev[flow->priority];
 609         free = npc->free_entries[flow->priority];
 610         live_rev = npc->live_entries_rev[flow->priority];
 611         live = npc->live_entries[flow->priority];
 612         mcam_entries = npc->mcam_entries;
 613
 614         if (info->free_ent) {
 615                 rc = plt_bitmap_scan(free, &pos, &slab);
 616                 if (rc) {
 617                         /* Get free_ent from free entry bitmap */
 618                         free_ent = pos + __builtin_ctzll(slab);
 619                         /* Remove from free bitmaps and add to live ones */
 620                         plt_bitmap_clear(free, free_ent);
 621                         plt_bitmap_set(live, free_ent);
 622                         plt_bitmap_clear(free_rev, mcam_entries - free_ent - 1);
 623                         plt_bitmap_set(live_rev, mcam_entries - free_ent - 1);
 624
 625                         info->free_ent--;
 626                         info->live_ent++;
 627                         return free_ent;
 628                 }
 629                 return NPC_ERR_INTERNAL;
 630         }
 631
 632         rc = npc_fill_entry_cache(mbox, flow, npc, &free_ent);
 633         if (rc)
 634                 return rc;
 635
 636         return free_ent;
 637 }