net/mlx5: add VLAN push/pop DR commands to glue
[dpdk.git] / drivers / net / octeontx2 / otx2_flow.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(C) 2019 Marvell International Ltd.
3  */
4
5 #include "otx2_ethdev.h"
6 #include "otx2_flow.h"
7
8 int
9 otx2_flow_free_all_resources(struct otx2_eth_dev *hw)
10 {
11         struct otx2_npc_flow_info *npc = &hw->npc_flow;
12         struct otx2_mbox *mbox = hw->mbox;
13         struct otx2_mcam_ents_info *info;
14         struct rte_bitmap *bmap;
15         struct rte_flow *flow;
16         int entry_count = 0;
17         int rc, idx;
18
19         for (idx = 0; idx < npc->flow_max_priority; idx++) {
20                 info = &npc->flow_entry_info[idx];
21                 entry_count += info->live_ent;
22         }
23
24         if (entry_count == 0)
25                 return 0;
26
27         /* Free all MCAM entries allocated */
28         rc = otx2_flow_mcam_free_all_entries(mbox);
29
30         /* Free any MCAM counters and delete flow list */
31         for (idx = 0; idx < npc->flow_max_priority; idx++) {
32                 while ((flow = TAILQ_FIRST(&npc->flow_list[idx])) != NULL) {
33                         if (flow->ctr_id != NPC_COUNTER_NONE)
34                                 rc |= otx2_flow_mcam_free_counter(mbox,
35                                                              flow->ctr_id);
36
37                         TAILQ_REMOVE(&npc->flow_list[idx], flow, next);
38                         rte_free(flow);
39                         bmap = npc->live_entries[flow->priority];
40                         rte_bitmap_clear(bmap, flow->mcam_id);
41                 }
42                 info = &npc->flow_entry_info[idx];
43                 info->free_ent = 0;
44                 info->live_ent = 0;
45         }
46         return rc;
47 }
48
49
50 static int
51 flow_program_npc(struct otx2_parse_state *pst, struct otx2_mbox *mbox,
52                  struct otx2_npc_flow_info *flow_info)
53 {
54         /* This is non-LDATA part in search key */
55         uint64_t key_data[2] = {0ULL, 0ULL};
56         uint64_t key_mask[2] = {0ULL, 0ULL};
57         int intf = pst->flow->nix_intf;
58         int key_len, bit = 0, index;
59         int off, idx, data_off = 0;
60         uint8_t lid, mask, data;
61         uint16_t layer_info;
62         uint64_t lt, flags;
63
64
65         /* Skip till Layer A data start */
66         while (bit < NPC_PARSE_KEX_S_LA_OFFSET) {
67                 if (flow_info->keyx_supp_nmask[intf] & (1 << bit))
68                         data_off++;
69                 bit++;
70         }
71
72         /* Each bit represents 1 nibble */
73         data_off *= 4;
74
75         index = 0;
76         for (lid = 0; lid < NPC_MAX_LID; lid++) {
77                 /* Offset in key */
78                 off = NPC_PARSE_KEX_S_LID_OFFSET(lid);
79                 lt = pst->lt[lid] & 0xf;
80                 flags = pst->flags[lid] & 0xff;
81
82                 /* NPC_LAYER_KEX_S */
83                 layer_info = ((flow_info->keyx_supp_nmask[intf] >> off) & 0x7);
84
85                 if (layer_info) {
86                         for (idx = 0; idx <= 2 ; idx++) {
87                                 if (layer_info & (1 << idx)) {
88                                         if (idx == 2)
89                                                 data = lt;
90                                         else if (idx == 1)
91                                                 data = ((flags >> 4) & 0xf);
92                                         else
93                                                 data = (flags & 0xf);
94
95                                         if (data_off >= 64) {
96                                                 data_off = 0;
97                                                 index++;
98                                         }
99                                         key_data[index] |= ((uint64_t)data <<
100                                                             data_off);
101                                         mask = 0xf;
102                                         if (lt == 0)
103                                                 mask = 0;
104                                         key_mask[index] |= ((uint64_t)mask <<
105                                                             data_off);
106                                         data_off += 4;
107                                 }
108                         }
109                 }
110         }
111
112         otx2_npc_dbg("Npc prog key data0: 0x%" PRIx64 ", data1: 0x%" PRIx64,
113                      key_data[0], key_data[1]);
114
115         /* Copy this into mcam string */
116         key_len = (pst->npc->keyx_len[intf] + 7) / 8;
117         otx2_npc_dbg("Key_len  = %d", key_len);
118         memcpy(pst->flow->mcam_data, key_data, key_len);
119         memcpy(pst->flow->mcam_mask, key_mask, key_len);
120
121         otx2_npc_dbg("Final flow data");
122         for (idx = 0; idx < OTX2_MAX_MCAM_WIDTH_DWORDS; idx++) {
123                 otx2_npc_dbg("data[%d]: 0x%" PRIx64 ", mask[%d]: 0x%" PRIx64,
124                              idx, pst->flow->mcam_data[idx],
125                              idx, pst->flow->mcam_mask[idx]);
126         }
127
128         /*
129          * Now we have mcam data and mask formatted as
130          * [Key_len/4 nibbles][0 or 1 nibble hole][data]
131          * hole is present if key_len is odd number of nibbles.
132          * mcam data must be split into 64 bits + 48 bits segments
133          * for each back W0, W1.
134          */
135
136         return otx2_flow_mcam_alloc_and_write(pst->flow, mbox, pst, flow_info);
137 }
138
139 static int
140 flow_parse_attr(struct rte_eth_dev *eth_dev,
141                 const struct rte_flow_attr *attr,
142                 struct rte_flow_error *error,
143                 struct rte_flow *flow)
144 {
145         struct otx2_eth_dev *dev = eth_dev->data->dev_private;
146         const char *errmsg = NULL;
147
148         if (attr == NULL)
149                 errmsg = "Attribute can't be empty";
150         else if (attr->group)
151                 errmsg = "Groups are not supported";
152         else if (attr->priority >= dev->npc_flow.flow_max_priority)
153                 errmsg = "Priority should be with in specified range";
154         else if ((!attr->egress && !attr->ingress) ||
155                  (attr->egress && attr->ingress))
156                 errmsg = "Exactly one of ingress or egress must be set";
157
158         if (errmsg != NULL) {
159                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR,
160                                    attr, errmsg);
161                 return -ENOTSUP;
162         }
163
164         if (attr->ingress)
165                 flow->nix_intf = OTX2_INTF_RX;
166         else
167                 flow->nix_intf = OTX2_INTF_TX;
168
169         flow->priority = attr->priority;
170         return 0;
171 }
172
173 static inline int
174 flow_get_free_rss_grp(struct rte_bitmap *bmap,
175                       uint32_t size, uint32_t *pos)
176 {
177         for (*pos = 0; *pos < size; ++*pos) {
178                 if (!rte_bitmap_get(bmap, *pos))
179                         break;
180         }
181
182         return *pos < size ? 0 : -1;
183 }
184
185 static int
186 flow_configure_rss_action(struct otx2_eth_dev *dev,
187                           const struct rte_flow_action_rss *rss,
188                           uint8_t *alg_idx, uint32_t *rss_grp,
189                           int mcam_index)
190 {
191         struct otx2_npc_flow_info *flow_info = &dev->npc_flow;
192         uint16_t reta[NIX_RSS_RETA_SIZE_MAX];
193         uint32_t flowkey_cfg, grp_aval, i;
194         uint16_t *ind_tbl = NULL;
195         uint8_t flowkey_algx;
196         int rc;
197
198         rc = flow_get_free_rss_grp(flow_info->rss_grp_entries,
199                                    flow_info->rss_grps, &grp_aval);
200         /* RSS group :0 is not usable for flow rss action */
201         if (rc < 0 || grp_aval == 0)
202                 return -ENOSPC;
203
204         *rss_grp = grp_aval;
205
206         otx2_nix_rss_set_key(dev, (uint8_t *)(uintptr_t)rss->key,
207                              rss->key_len);
208
209         /* If queue count passed in the rss action is less than
210          * HW configured reta size, replicate rss action reta
211          * across HW reta table.
212          */
213         if (dev->rss_info.rss_size > rss->queue_num) {
214                 ind_tbl = reta;
215
216                 for (i = 0; i < (dev->rss_info.rss_size / rss->queue_num); i++)
217                         memcpy(reta + i * rss->queue_num, rss->queue,
218                                sizeof(uint16_t) * rss->queue_num);
219
220                 i = dev->rss_info.rss_size % rss->queue_num;
221                 if (i)
222                         memcpy(&reta[dev->rss_info.rss_size] - i,
223                                rss->queue, i * sizeof(uint16_t));
224         } else {
225                 ind_tbl = (uint16_t *)(uintptr_t)rss->queue;
226         }
227
228         rc = otx2_nix_rss_tbl_init(dev, *rss_grp, ind_tbl);
229         if (rc) {
230                 otx2_err("Failed to init rss table rc = %d", rc);
231                 return rc;
232         }
233
234         flowkey_cfg = otx2_rss_ethdev_to_nix(dev, rss->types, rss->level);
235
236         rc = otx2_rss_set_hf(dev, flowkey_cfg, &flowkey_algx,
237                              *rss_grp, mcam_index);
238         if (rc) {
239                 otx2_err("Failed to set rss hash function rc = %d", rc);
240                 return rc;
241         }
242
243         *alg_idx = flowkey_algx;
244
245         rte_bitmap_set(flow_info->rss_grp_entries, *rss_grp);
246
247         return 0;
248 }
249
250
251 static int
252 flow_program_rss_action(struct rte_eth_dev *eth_dev,
253                         const struct rte_flow_action actions[],
254                         struct rte_flow *flow)
255 {
256         struct otx2_eth_dev *dev = eth_dev->data->dev_private;
257         const struct rte_flow_action_rss *rss;
258         uint32_t rss_grp;
259         uint8_t alg_idx;
260         int rc;
261
262         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
263                 if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
264                         rss = (const struct rte_flow_action_rss *)actions->conf;
265
266                         rc = flow_configure_rss_action(dev,
267                                                        rss, &alg_idx, &rss_grp,
268                                                        flow->mcam_id);
269                         if (rc)
270                                 return rc;
271
272                         flow->npc_action |=
273                                 ((uint64_t)(alg_idx & NIX_RSS_ACT_ALG_MASK) <<
274                                  NIX_RSS_ACT_ALG_OFFSET) |
275                                 ((uint64_t)(rss_grp & NIX_RSS_ACT_GRP_MASK) <<
276                                  NIX_RSS_ACT_GRP_OFFSET);
277                 }
278         }
279         return 0;
280 }
281
282 static int
283 flow_free_rss_action(struct rte_eth_dev *eth_dev,
284                      struct rte_flow *flow)
285 {
286         struct otx2_eth_dev *dev = eth_dev->data->dev_private;
287         struct otx2_npc_flow_info *npc = &dev->npc_flow;
288         uint32_t rss_grp;
289
290         if (flow->npc_action & NIX_RX_ACTIONOP_RSS) {
291                 rss_grp = (flow->npc_action >> NIX_RSS_ACT_GRP_OFFSET) &
292                         NIX_RSS_ACT_GRP_MASK;
293                 if (rss_grp == 0 || rss_grp >= npc->rss_grps)
294                         return -EINVAL;
295
296                 rte_bitmap_clear(npc->rss_grp_entries, rss_grp);
297         }
298
299         return 0;
300 }
301
302
303 static int
304 flow_parse_meta_items(__rte_unused struct otx2_parse_state *pst)
305 {
306         otx2_npc_dbg("Meta Item");
307         return 0;
308 }
309
310 /*
311  * Parse function of each layer:
312  *  - Consume one or more patterns that are relevant.
313  *  - Update parse_state
314  *  - Set parse_state.pattern = last item consumed
315  *  - Set appropriate error code/message when returning error.
316  */
317 typedef int (*flow_parse_stage_func_t)(struct otx2_parse_state *pst);
318
319 static int
320 flow_parse_pattern(struct rte_eth_dev *dev,
321                    const struct rte_flow_item pattern[],
322                    struct rte_flow_error *error,
323                    struct rte_flow *flow,
324                    struct otx2_parse_state *pst)
325 {
326         flow_parse_stage_func_t parse_stage_funcs[] = {
327                 flow_parse_meta_items,
328                 otx2_flow_parse_la,
329                 otx2_flow_parse_lb,
330                 otx2_flow_parse_lc,
331                 otx2_flow_parse_ld,
332                 otx2_flow_parse_le,
333                 otx2_flow_parse_lf,
334                 otx2_flow_parse_lg,
335                 otx2_flow_parse_lh,
336         };
337         struct otx2_eth_dev *hw = dev->data->dev_private;
338         uint8_t layer = 0;
339         int key_offset;
340         int rc;
341
342         if (pattern == NULL) {
343                 rte_flow_error_set(error, EINVAL,
344                                    RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
345                                    "pattern is NULL");
346                 return -EINVAL;
347         }
348
349         memset(pst, 0, sizeof(*pst));
350         pst->npc = &hw->npc_flow;
351         pst->error = error;
352         pst->flow = flow;
353
354         /* Use integral byte offset */
355         key_offset = pst->npc->keyx_len[flow->nix_intf];
356         key_offset = (key_offset + 7) / 8;
357
358         /* Location where LDATA would begin */
359         pst->mcam_data = (uint8_t *)flow->mcam_data;
360         pst->mcam_mask = (uint8_t *)flow->mcam_mask;
361
362         while (pattern->type != RTE_FLOW_ITEM_TYPE_END &&
363                layer < RTE_DIM(parse_stage_funcs)) {
364                 otx2_npc_dbg("Pattern type = %d", pattern->type);
365
366                 /* Skip place-holders */
367                 pattern = otx2_flow_skip_void_and_any_items(pattern);
368
369                 pst->pattern = pattern;
370                 otx2_npc_dbg("Is tunnel = %d, layer = %d", pst->tunnel, layer);
371                 rc = parse_stage_funcs[layer](pst);
372                 if (rc != 0)
373                         return -rte_errno;
374
375                 layer++;
376
377                 /*
378                  * Parse stage function sets pst->pattern to
379                  * 1 past the last item it consumed.
380                  */
381                 pattern = pst->pattern;
382
383                 if (pst->terminate)
384                         break;
385         }
386
387         /* Skip trailing place-holders */
388         pattern = otx2_flow_skip_void_and_any_items(pattern);
389
390         /* Are there more items than what we can handle? */
391         if (pattern->type != RTE_FLOW_ITEM_TYPE_END) {
392                 rte_flow_error_set(error, ENOTSUP,
393                                    RTE_FLOW_ERROR_TYPE_ITEM, pattern,
394                                    "unsupported item in the sequence");
395                 return -ENOTSUP;
396         }
397
398         return 0;
399 }
400
401 static int
402 flow_parse_rule(struct rte_eth_dev *dev,
403                 const struct rte_flow_attr *attr,
404                 const struct rte_flow_item pattern[],
405                 const struct rte_flow_action actions[],
406                 struct rte_flow_error *error,
407                 struct rte_flow *flow,
408                 struct otx2_parse_state *pst)
409 {
410         int err;
411
412         /* Check attributes */
413         err = flow_parse_attr(dev, attr, error, flow);
414         if (err)
415                 return err;
416
417         /* Check actions */
418         err = otx2_flow_parse_actions(dev, attr, actions, error, flow);
419         if (err)
420                 return err;
421
422         /* Check pattern */
423         err = flow_parse_pattern(dev, pattern, error, flow, pst);
424         if (err)
425                 return err;
426
427         /* Check for overlaps? */
428         return 0;
429 }
430
431 static int
432 otx2_flow_validate(struct rte_eth_dev *dev,
433                    const struct rte_flow_attr *attr,
434                    const struct rte_flow_item pattern[],
435                    const struct rte_flow_action actions[],
436                    struct rte_flow_error *error)
437 {
438         struct otx2_parse_state parse_state;
439         struct rte_flow flow;
440
441         memset(&flow, 0, sizeof(flow));
442         return flow_parse_rule(dev, attr, pattern, actions, error, &flow,
443                                &parse_state);
444 }
445
446 static struct rte_flow *
447 otx2_flow_create(struct rte_eth_dev *dev,
448                  const struct rte_flow_attr *attr,
449                  const struct rte_flow_item pattern[],
450                  const struct rte_flow_action actions[],
451                  struct rte_flow_error *error)
452 {
453         struct otx2_eth_dev *hw = dev->data->dev_private;
454         struct otx2_parse_state parse_state;
455         struct otx2_mbox *mbox = hw->mbox;
456         struct rte_flow *flow, *flow_iter;
457         struct otx2_flow_list *list;
458         int rc;
459
460         flow = rte_zmalloc("otx2_rte_flow", sizeof(*flow), 0);
461         if (flow == NULL) {
462                 rte_flow_error_set(error, ENOMEM,
463                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
464                                    NULL,
465                                    "Memory allocation failed");
466                 return NULL;
467         }
468         memset(flow, 0, sizeof(*flow));
469
470         rc = flow_parse_rule(dev, attr, pattern, actions, error, flow,
471                              &parse_state);
472         if (rc != 0)
473                 goto err_exit;
474
475         rc = flow_program_npc(&parse_state, mbox, &hw->npc_flow);
476         if (rc != 0) {
477                 rte_flow_error_set(error, EIO,
478                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
479                                    NULL,
480                                    "Failed to insert filter");
481                 goto err_exit;
482         }
483
484         rc = flow_program_rss_action(dev, actions, flow);
485         if (rc != 0) {
486                 rte_flow_error_set(error, EIO,
487                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
488                                    NULL,
489                                    "Failed to program rss action");
490                 goto err_exit;
491         }
492
493
494         list = &hw->npc_flow.flow_list[flow->priority];
495         /* List in ascending order of mcam entries */
496         TAILQ_FOREACH(flow_iter, list, next) {
497                 if (flow_iter->mcam_id > flow->mcam_id) {
498                         TAILQ_INSERT_BEFORE(flow_iter, flow, next);
499                         return flow;
500                 }
501         }
502
503         TAILQ_INSERT_TAIL(list, flow, next);
504         return flow;
505
506 err_exit:
507         rte_free(flow);
508         return NULL;
509 }
510
511 static int
512 otx2_flow_destroy(struct rte_eth_dev *dev,
513                   struct rte_flow *flow,
514                   struct rte_flow_error *error)
515 {
516         struct otx2_eth_dev *hw = dev->data->dev_private;
517         struct otx2_npc_flow_info *npc = &hw->npc_flow;
518         struct otx2_mbox *mbox = hw->mbox;
519         struct rte_bitmap *bmap;
520         uint16_t match_id;
521         int rc;
522
523         match_id = (flow->npc_action >> NIX_RX_ACT_MATCH_OFFSET) &
524                 NIX_RX_ACT_MATCH_MASK;
525
526         if (match_id && match_id < OTX2_FLOW_ACTION_FLAG_DEFAULT) {
527                 if (rte_atomic32_read(&npc->mark_actions) == 0)
528                         return -EINVAL;
529
530                 /* Clear mark offload flag if there are no more mark actions */
531                 if (rte_atomic32_sub_return(&npc->mark_actions, 1) == 0) {
532                         hw->rx_offload_flags &= ~NIX_RX_OFFLOAD_MARK_UPDATE_F;
533                         otx2_eth_set_rx_function(dev);
534                 }
535         }
536
537         rc = flow_free_rss_action(dev, flow);
538         if (rc != 0) {
539                 rte_flow_error_set(error, EIO,
540                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
541                                    NULL,
542                                    "Failed to free rss action");
543         }
544
545         rc = otx2_flow_mcam_free_entry(mbox, flow->mcam_id);
546         if (rc != 0) {
547                 rte_flow_error_set(error, EIO,
548                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
549                                    NULL,
550                                    "Failed to destroy filter");
551         }
552
553         TAILQ_REMOVE(&npc->flow_list[flow->priority], flow, next);
554
555         bmap = npc->live_entries[flow->priority];
556         rte_bitmap_clear(bmap, flow->mcam_id);
557
558         rte_free(flow);
559         return 0;
560 }
561
562 static int
563 otx2_flow_flush(struct rte_eth_dev *dev,
564                 struct rte_flow_error *error)
565 {
566         struct otx2_eth_dev *hw = dev->data->dev_private;
567         int rc;
568
569         rc = otx2_flow_free_all_resources(hw);
570         if (rc) {
571                 otx2_err("Error when deleting NPC MCAM entries "
572                                 ", counters");
573                 rte_flow_error_set(error, EIO,
574                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
575                                    NULL,
576                                    "Failed to flush filter");
577                 return -rte_errno;
578         }
579
580         return 0;
581 }
582
583 static int
584 otx2_flow_isolate(struct rte_eth_dev *dev __rte_unused,
585                   int enable __rte_unused,
586                   struct rte_flow_error *error)
587 {
588         /*
589          * If we support, we need to un-install the default mcam
590          * entry for this port.
591          */
592
593         rte_flow_error_set(error, ENOTSUP,
594                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
595                            NULL,
596                            "Flow isolation not supported");
597
598         return -rte_errno;
599 }
600
601 static int
602 otx2_flow_query(struct rte_eth_dev *dev,
603                 struct rte_flow *flow,
604                 const struct rte_flow_action *action,
605                 void *data,
606                 struct rte_flow_error *error)
607 {
608         struct otx2_eth_dev *hw = dev->data->dev_private;
609         struct rte_flow_query_count *query = data;
610         struct otx2_mbox *mbox = hw->mbox;
611         const char *errmsg = NULL;
612         int errcode = ENOTSUP;
613         int rc;
614
615         if (action->type != RTE_FLOW_ACTION_TYPE_COUNT) {
616                 errmsg = "Only COUNT is supported in query";
617                 goto err_exit;
618         }
619
620         if (flow->ctr_id == NPC_COUNTER_NONE) {
621                 errmsg = "Counter is not available";
622                 goto err_exit;
623         }
624
625         rc = otx2_flow_mcam_read_counter(mbox, flow->ctr_id, &query->hits);
626         if (rc != 0) {
627                 errcode = EIO;
628                 errmsg = "Error reading flow counter";
629                 goto err_exit;
630         }
631         query->hits_set = 1;
632         query->bytes_set = 0;
633
634         if (query->reset)
635                 rc = otx2_flow_mcam_clear_counter(mbox, flow->ctr_id);
636         if (rc != 0) {
637                 errcode = EIO;
638                 errmsg = "Error clearing flow counter";
639                 goto err_exit;
640         }
641
642         return 0;
643
644 err_exit:
645         rte_flow_error_set(error, errcode,
646                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
647                            NULL,
648                            errmsg);
649         return -rte_errno;
650 }
651
652 const struct rte_flow_ops otx2_flow_ops = {
653         .validate = otx2_flow_validate,
654         .create = otx2_flow_create,
655         .destroy = otx2_flow_destroy,
656         .flush = otx2_flow_flush,
657         .query = otx2_flow_query,
658         .isolate = otx2_flow_isolate,
659 };
660
661 static int
662 flow_supp_key_len(uint32_t supp_mask)
663 {
664         int nib_count = 0;
665         while (supp_mask) {
666                 nib_count++;
667                 supp_mask &= (supp_mask - 1);
668         }
669         return nib_count * 4;
670 }
671
672 /* Refer HRM register:
673  * NPC_AF_INTF(0..1)_LID(0..7)_LT(0..15)_LD(0..1)_CFG
674  * and
675  * NPC_AF_INTF(0..1)_LDATA(0..1)_FLAGS(0..15)_CFG
676  **/
677 #define BYTESM1_SHIFT   16
678 #define HDR_OFF_SHIFT   8
679 static void
680 flow_update_kex_info(struct npc_xtract_info *xtract_info,
681                      uint64_t val)
682 {
683         xtract_info->len = ((val >> BYTESM1_SHIFT) & 0xf) + 1;
684         xtract_info->hdr_off = (val >> HDR_OFF_SHIFT) & 0xff;
685         xtract_info->key_off = val & 0x3f;
686         xtract_info->enable = ((val >> 7) & 0x1);
687         xtract_info->flags_enable = ((val >> 6) & 0x1);
688 }
689
690 static void
691 flow_process_mkex_cfg(struct otx2_npc_flow_info *npc,
692                       struct npc_get_kex_cfg_rsp *kex_rsp)
693 {
694         volatile uint64_t (*q)[NPC_MAX_INTF][NPC_MAX_LID][NPC_MAX_LT]
695                 [NPC_MAX_LD];
696         struct npc_xtract_info *x_info = NULL;
697         int lid, lt, ld, fl, ix;
698         otx2_dxcfg_t *p;
699         uint64_t keyw;
700         uint64_t val;
701
702         npc->keyx_supp_nmask[NPC_MCAM_RX] =
703                 kex_rsp->rx_keyx_cfg & 0x7fffffffULL;
704         npc->keyx_supp_nmask[NPC_MCAM_TX] =
705                 kex_rsp->tx_keyx_cfg & 0x7fffffffULL;
706         npc->keyx_len[NPC_MCAM_RX] =
707                 flow_supp_key_len(npc->keyx_supp_nmask[NPC_MCAM_RX]);
708         npc->keyx_len[NPC_MCAM_TX] =
709                 flow_supp_key_len(npc->keyx_supp_nmask[NPC_MCAM_TX]);
710
711         keyw = (kex_rsp->rx_keyx_cfg >> 32) & 0x7ULL;
712         npc->keyw[NPC_MCAM_RX] = keyw;
713         keyw = (kex_rsp->tx_keyx_cfg >> 32) & 0x7ULL;
714         npc->keyw[NPC_MCAM_TX] = keyw;
715
716         /* Update KEX_LD_FLAG */
717         for (ix = 0; ix < NPC_MAX_INTF; ix++) {
718                 for (ld = 0; ld < NPC_MAX_LD; ld++) {
719                         for (fl = 0; fl < NPC_MAX_LFL; fl++) {
720                                 x_info =
721                                     &npc->prx_fxcfg[ix][ld][fl].xtract[0];
722                                 val = kex_rsp->intf_ld_flags[ix][ld][fl];
723                                 flow_update_kex_info(x_info, val);
724                         }
725                 }
726         }
727
728         /* Update LID, LT and LDATA cfg */
729         p = &npc->prx_dxcfg;
730         q = (volatile uint64_t (*)[][NPC_MAX_LID][NPC_MAX_LT][NPC_MAX_LD])
731                         (&kex_rsp->intf_lid_lt_ld);
732         for (ix = 0; ix < NPC_MAX_INTF; ix++) {
733                 for (lid = 0; lid < NPC_MAX_LID; lid++) {
734                         for (lt = 0; lt < NPC_MAX_LT; lt++) {
735                                 for (ld = 0; ld < NPC_MAX_LD; ld++) {
736                                         x_info = &(*p)[ix][lid][lt].xtract[ld];
737                                         val = (*q)[ix][lid][lt][ld];
738                                         flow_update_kex_info(x_info, val);
739                                 }
740                         }
741                 }
742         }
743         /* Update LDATA Flags cfg */
744         npc->prx_lfcfg[0].i = kex_rsp->kex_ld_flags[0];
745         npc->prx_lfcfg[1].i = kex_rsp->kex_ld_flags[1];
746 }
747
748 static struct otx2_idev_kex_cfg *
749 flow_intra_dev_kex_cfg(void)
750 {
751         static const char name[] = "octeontx2_intra_device_kex_conf";
752         struct otx2_idev_kex_cfg *idev;
753         const struct rte_memzone *mz;
754
755         mz = rte_memzone_lookup(name);
756         if (mz)
757                 return mz->addr;
758
759         /* Request for the first time */
760         mz = rte_memzone_reserve_aligned(name, sizeof(struct otx2_idev_kex_cfg),
761                                          SOCKET_ID_ANY, 0, OTX2_ALIGN);
762         if (mz) {
763                 idev = mz->addr;
764                 rte_atomic16_set(&idev->kex_refcnt, 0);
765                 return idev;
766         }
767         return NULL;
768 }
769
770 static int
771 flow_fetch_kex_cfg(struct otx2_eth_dev *dev)
772 {
773         struct otx2_npc_flow_info *npc = &dev->npc_flow;
774         struct npc_get_kex_cfg_rsp *kex_rsp;
775         struct otx2_mbox *mbox = dev->mbox;
776         char mkex_pfl_name[MKEX_NAME_LEN];
777         struct otx2_idev_kex_cfg *idev;
778         int rc = 0;
779
780         idev = flow_intra_dev_kex_cfg();
781         if (!idev)
782                 return -ENOMEM;
783
784         /* Is kex_cfg read by any another driver? */
785         if (rte_atomic16_add_return(&idev->kex_refcnt, 1) == 1) {
786                 /* Call mailbox to get key & data size */
787                 (void)otx2_mbox_alloc_msg_npc_get_kex_cfg(mbox);
788                 otx2_mbox_msg_send(mbox, 0);
789                 rc = otx2_mbox_get_rsp(mbox, 0, (void *)&kex_rsp);
790                 if (rc) {
791                         otx2_err("Failed to fetch NPC keyx config");
792                         goto done;
793                 }
794                 memcpy(&idev->kex_cfg, kex_rsp,
795                        sizeof(struct npc_get_kex_cfg_rsp));
796         }
797
798         otx2_mbox_memcpy(mkex_pfl_name,
799                          idev->kex_cfg.mkex_pfl_name, MKEX_NAME_LEN);
800
801         strlcpy((char *)dev->mkex_pfl_name,
802                 mkex_pfl_name, sizeof(dev->mkex_pfl_name));
803
804         flow_process_mkex_cfg(npc, &idev->kex_cfg);
805
806 done:
807         return rc;
808 }
809
810 int
811 otx2_flow_init(struct otx2_eth_dev *hw)
812 {
813         uint8_t *mem = NULL, *nix_mem = NULL, *npc_mem = NULL;
814         struct otx2_npc_flow_info *npc = &hw->npc_flow;
815         uint32_t bmap_sz;
816         int rc = 0, idx;
817
818         rc = flow_fetch_kex_cfg(hw);
819         if (rc) {
820                 otx2_err("Failed to fetch NPC keyx config from idev");
821                 return rc;
822         }
823
824         rte_atomic32_init(&npc->mark_actions);
825
826         npc->mcam_entries = NPC_MCAM_TOT_ENTRIES >> npc->keyw[NPC_MCAM_RX];
827         /* Free, free_rev, live and live_rev entries */
828         bmap_sz = rte_bitmap_get_memory_footprint(npc->mcam_entries);
829         mem = rte_zmalloc(NULL, 4 * bmap_sz * npc->flow_max_priority,
830                           RTE_CACHE_LINE_SIZE);
831         if (mem == NULL) {
832                 otx2_err("Bmap alloc failed");
833                 rc = -ENOMEM;
834                 return rc;
835         }
836
837         npc->flow_entry_info = rte_zmalloc(NULL, npc->flow_max_priority
838                                            * sizeof(struct otx2_mcam_ents_info),
839                                            0);
840         if (npc->flow_entry_info == NULL) {
841                 otx2_err("flow_entry_info alloc failed");
842                 rc = -ENOMEM;
843                 goto err;
844         }
845
846         npc->free_entries = rte_zmalloc(NULL, npc->flow_max_priority
847                                         * sizeof(struct rte_bitmap *),
848                                         0);
849         if (npc->free_entries == NULL) {
850                 otx2_err("free_entries alloc failed");
851                 rc = -ENOMEM;
852                 goto err;
853         }
854
855         npc->free_entries_rev = rte_zmalloc(NULL, npc->flow_max_priority
856                                         * sizeof(struct rte_bitmap *),
857                                         0);
858         if (npc->free_entries_rev == NULL) {
859                 otx2_err("free_entries_rev alloc failed");
860                 rc = -ENOMEM;
861                 goto err;
862         }
863
864         npc->live_entries = rte_zmalloc(NULL, npc->flow_max_priority
865                                         * sizeof(struct rte_bitmap *),
866                                         0);
867         if (npc->live_entries == NULL) {
868                 otx2_err("live_entries alloc failed");
869                 rc = -ENOMEM;
870                 goto err;
871         }
872
873         npc->live_entries_rev = rte_zmalloc(NULL, npc->flow_max_priority
874                                         * sizeof(struct rte_bitmap *),
875                                         0);
876         if (npc->live_entries_rev == NULL) {
877                 otx2_err("live_entries_rev alloc failed");
878                 rc = -ENOMEM;
879                 goto err;
880         }
881
882         npc->flow_list = rte_zmalloc(NULL, npc->flow_max_priority
883                                         * sizeof(struct otx2_flow_list),
884                                         0);
885         if (npc->flow_list == NULL) {
886                 otx2_err("flow_list alloc failed");
887                 rc = -ENOMEM;
888                 goto err;
889         }
890
891         npc_mem = mem;
892         for (idx = 0; idx < npc->flow_max_priority; idx++) {
893                 TAILQ_INIT(&npc->flow_list[idx]);
894
895                 npc->free_entries[idx] =
896                         rte_bitmap_init(npc->mcam_entries, mem, bmap_sz);
897                 mem += bmap_sz;
898
899                 npc->free_entries_rev[idx] =
900                         rte_bitmap_init(npc->mcam_entries, mem, bmap_sz);
901                 mem += bmap_sz;
902
903                 npc->live_entries[idx] =
904                         rte_bitmap_init(npc->mcam_entries, mem, bmap_sz);
905                 mem += bmap_sz;
906
907                 npc->live_entries_rev[idx] =
908                         rte_bitmap_init(npc->mcam_entries, mem, bmap_sz);
909                 mem += bmap_sz;
910
911                 npc->flow_entry_info[idx].free_ent = 0;
912                 npc->flow_entry_info[idx].live_ent = 0;
913                 npc->flow_entry_info[idx].max_id = 0;
914                 npc->flow_entry_info[idx].min_id = ~(0);
915         }
916
917         npc->rss_grps = NIX_RSS_GRPS;
918
919         bmap_sz = rte_bitmap_get_memory_footprint(npc->rss_grps);
920         nix_mem = rte_zmalloc(NULL, bmap_sz,  RTE_CACHE_LINE_SIZE);
921         if (nix_mem == NULL) {
922                 otx2_err("Bmap alloc failed");
923                 rc = -ENOMEM;
924                 goto err;
925         }
926
927         npc->rss_grp_entries = rte_bitmap_init(npc->rss_grps, nix_mem, bmap_sz);
928
929         /* Group 0 will be used for RSS,
930          * 1 -7 will be used for rte_flow RSS action
931          */
932         rte_bitmap_set(npc->rss_grp_entries, 0);
933
934         return 0;
935
936 err:
937         if (npc->flow_list)
938                 rte_free(npc->flow_list);
939         if (npc->live_entries_rev)
940                 rte_free(npc->live_entries_rev);
941         if (npc->live_entries)
942                 rte_free(npc->live_entries);
943         if (npc->free_entries_rev)
944                 rte_free(npc->free_entries_rev);
945         if (npc->free_entries)
946                 rte_free(npc->free_entries);
947         if (npc->flow_entry_info)
948                 rte_free(npc->flow_entry_info);
949         if (npc_mem)
950                 rte_free(npc_mem);
951         return rc;
952 }
953
954 int
955 otx2_flow_fini(struct otx2_eth_dev *hw)
956 {
957         struct otx2_npc_flow_info *npc = &hw->npc_flow;
958         int rc;
959
960         rc = otx2_flow_free_all_resources(hw);
961         if (rc) {
962                 otx2_err("Error when deleting NPC MCAM entries, counters");
963                 return rc;
964         }
965
966         if (npc->flow_list)
967                 rte_free(npc->flow_list);
968         if (npc->live_entries_rev)
969                 rte_free(npc->live_entries_rev);
970         if (npc->live_entries)
971                 rte_free(npc->live_entries);
972         if (npc->free_entries_rev)
973                 rte_free(npc->free_entries_rev);
974         if (npc->free_entries)
975                 rte_free(npc->free_entries);
976         if (npc->flow_entry_info)
977                 rte_free(npc->flow_entry_info);
978
979         return 0;
980 }