1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
5 #include "otx2_ethdev.h"
9 otx2_flow_free_all_resources(struct otx2_eth_dev *hw)
11 struct otx2_npc_flow_info *npc = &hw->npc_flow;
12 struct otx2_mbox *mbox = hw->mbox;
13 struct otx2_mcam_ents_info *info;
14 struct rte_bitmap *bmap;
15 struct rte_flow *flow;
19 for (idx = 0; idx < npc->flow_max_priority; idx++) {
20 info = &npc->flow_entry_info[idx];
21 entry_count += info->live_ent;
27 /* Free all MCAM entries allocated */
28 rc = otx2_flow_mcam_free_all_entries(mbox);
30 /* Free any MCAM counters and delete flow list */
31 for (idx = 0; idx < npc->flow_max_priority; idx++) {
32 while ((flow = TAILQ_FIRST(&npc->flow_list[idx])) != NULL) {
33 if (flow->ctr_id != NPC_COUNTER_NONE)
34 rc |= otx2_flow_mcam_free_counter(mbox,
37 TAILQ_REMOVE(&npc->flow_list[idx], flow, next);
39 bmap = npc->live_entries[flow->priority];
40 rte_bitmap_clear(bmap, flow->mcam_id);
42 info = &npc->flow_entry_info[idx];
51 flow_program_npc(struct otx2_parse_state *pst, struct otx2_mbox *mbox,
52 struct otx2_npc_flow_info *flow_info)
54 /* This is non-LDATA part in search key */
55 uint64_t key_data[2] = {0ULL, 0ULL};
56 uint64_t key_mask[2] = {0ULL, 0ULL};
57 int intf = pst->flow->nix_intf;
58 int key_len, bit = 0, index;
59 int off, idx, data_off = 0;
60 uint8_t lid, mask, data;
65 /* Skip till Layer A data start */
66 while (bit < NPC_PARSE_KEX_S_LA_OFFSET) {
67 if (flow_info->keyx_supp_nmask[intf] & (1 << bit))
72 /* Each bit represents 1 nibble */
76 for (lid = 0; lid < NPC_MAX_LID; lid++) {
78 off = NPC_PARSE_KEX_S_LID_OFFSET(lid);
79 lt = pst->lt[lid] & 0xf;
80 flags = pst->flags[lid] & 0xff;
83 layer_info = ((flow_info->keyx_supp_nmask[intf] >> off) & 0x7);
86 for (idx = 0; idx <= 2 ; idx++) {
87 if (layer_info & (1 << idx)) {
91 data = ((flags >> 4) & 0xf);
99 key_data[index] |= ((uint64_t)data <<
104 key_mask[index] |= ((uint64_t)mask <<
112 otx2_npc_dbg("Npc prog key data0: 0x%" PRIx64 ", data1: 0x%" PRIx64,
113 key_data[0], key_data[1]);
115 /* Copy this into mcam string */
116 key_len = (pst->npc->keyx_len[intf] + 7) / 8;
117 otx2_npc_dbg("Key_len = %d", key_len);
118 memcpy(pst->flow->mcam_data, key_data, key_len);
119 memcpy(pst->flow->mcam_mask, key_mask, key_len);
121 otx2_npc_dbg("Final flow data");
122 for (idx = 0; idx < OTX2_MAX_MCAM_WIDTH_DWORDS; idx++) {
123 otx2_npc_dbg("data[%d]: 0x%" PRIx64 ", mask[%d]: 0x%" PRIx64,
124 idx, pst->flow->mcam_data[idx],
125 idx, pst->flow->mcam_mask[idx]);
129 * Now we have mcam data and mask formatted as
130 * [Key_len/4 nibbles][0 or 1 nibble hole][data]
131 * hole is present if key_len is odd number of nibbles.
132 * mcam data must be split into 64 bits + 48 bits segments
133 * for each back W0, W1.
136 return otx2_flow_mcam_alloc_and_write(pst->flow, mbox, pst, flow_info);
140 flow_parse_attr(struct rte_eth_dev *eth_dev,
141 const struct rte_flow_attr *attr,
142 struct rte_flow_error *error,
143 struct rte_flow *flow)
145 struct otx2_eth_dev *dev = eth_dev->data->dev_private;
146 const char *errmsg = NULL;
149 errmsg = "Attribute can't be empty";
150 else if (attr->group)
151 errmsg = "Groups are not supported";
152 else if (attr->priority >= dev->npc_flow.flow_max_priority)
153 errmsg = "Priority should be with in specified range";
154 else if ((!attr->egress && !attr->ingress) ||
155 (attr->egress && attr->ingress))
156 errmsg = "Exactly one of ingress or egress must be set";
158 if (errmsg != NULL) {
159 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR,
165 flow->nix_intf = OTX2_INTF_RX;
167 flow->nix_intf = OTX2_INTF_TX;
169 flow->priority = attr->priority;
174 flow_get_free_rss_grp(struct rte_bitmap *bmap,
175 uint32_t size, uint32_t *pos)
177 for (*pos = 0; *pos < size; ++*pos) {
178 if (!rte_bitmap_get(bmap, *pos))
182 return *pos < size ? 0 : -1;
186 flow_configure_rss_action(struct otx2_eth_dev *dev,
187 const struct rte_flow_action_rss *rss,
188 uint8_t *alg_idx, uint32_t *rss_grp,
191 struct otx2_npc_flow_info *flow_info = &dev->npc_flow;
192 uint16_t reta[NIX_RSS_RETA_SIZE_MAX];
193 uint32_t flowkey_cfg, grp_aval, i;
194 uint16_t *ind_tbl = NULL;
195 uint8_t flowkey_algx;
198 rc = flow_get_free_rss_grp(flow_info->rss_grp_entries,
199 flow_info->rss_grps, &grp_aval);
200 /* RSS group :0 is not usable for flow rss action */
201 if (rc < 0 || grp_aval == 0)
206 otx2_nix_rss_set_key(dev, (uint8_t *)(uintptr_t)rss->key,
209 /* If queue count passed in the rss action is less than
210 * HW configured reta size, replicate rss action reta
211 * across HW reta table.
213 if (dev->rss_info.rss_size > rss->queue_num) {
216 for (i = 0; i < (dev->rss_info.rss_size / rss->queue_num); i++)
217 memcpy(reta + i * rss->queue_num, rss->queue,
218 sizeof(uint16_t) * rss->queue_num);
220 i = dev->rss_info.rss_size % rss->queue_num;
222 memcpy(&reta[dev->rss_info.rss_size] - i,
223 rss->queue, i * sizeof(uint16_t));
225 ind_tbl = (uint16_t *)(uintptr_t)rss->queue;
228 rc = otx2_nix_rss_tbl_init(dev, *rss_grp, ind_tbl);
230 otx2_err("Failed to init rss table rc = %d", rc);
234 flowkey_cfg = otx2_rss_ethdev_to_nix(dev, rss->types, rss->level);
236 rc = otx2_rss_set_hf(dev, flowkey_cfg, &flowkey_algx,
237 *rss_grp, mcam_index);
239 otx2_err("Failed to set rss hash function rc = %d", rc);
243 *alg_idx = flowkey_algx;
245 rte_bitmap_set(flow_info->rss_grp_entries, *rss_grp);
252 flow_program_rss_action(struct rte_eth_dev *eth_dev,
253 const struct rte_flow_action actions[],
254 struct rte_flow *flow)
256 struct otx2_eth_dev *dev = eth_dev->data->dev_private;
257 const struct rte_flow_action_rss *rss;
262 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
263 if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
264 rss = (const struct rte_flow_action_rss *)actions->conf;
266 rc = flow_configure_rss_action(dev,
267 rss, &alg_idx, &rss_grp,
273 ((uint64_t)(alg_idx & NIX_RSS_ACT_ALG_MASK) <<
274 NIX_RSS_ACT_ALG_OFFSET) |
275 ((uint64_t)(rss_grp & NIX_RSS_ACT_GRP_MASK) <<
276 NIX_RSS_ACT_GRP_OFFSET);
283 flow_free_rss_action(struct rte_eth_dev *eth_dev,
284 struct rte_flow *flow)
286 struct otx2_eth_dev *dev = eth_dev->data->dev_private;
287 struct otx2_npc_flow_info *npc = &dev->npc_flow;
290 if (flow->npc_action & NIX_RX_ACTIONOP_RSS) {
291 rss_grp = (flow->npc_action >> NIX_RSS_ACT_GRP_OFFSET) &
292 NIX_RSS_ACT_GRP_MASK;
293 if (rss_grp == 0 || rss_grp >= npc->rss_grps)
296 rte_bitmap_clear(npc->rss_grp_entries, rss_grp);
304 flow_parse_meta_items(__rte_unused struct otx2_parse_state *pst)
306 otx2_npc_dbg("Meta Item");
311 * Parse function of each layer:
312 * - Consume one or more patterns that are relevant.
313 * - Update parse_state
314 * - Set parse_state.pattern = last item consumed
315 * - Set appropriate error code/message when returning error.
317 typedef int (*flow_parse_stage_func_t)(struct otx2_parse_state *pst);
320 flow_parse_pattern(struct rte_eth_dev *dev,
321 const struct rte_flow_item pattern[],
322 struct rte_flow_error *error,
323 struct rte_flow *flow,
324 struct otx2_parse_state *pst)
326 flow_parse_stage_func_t parse_stage_funcs[] = {
327 flow_parse_meta_items,
337 struct otx2_eth_dev *hw = dev->data->dev_private;
342 if (pattern == NULL) {
343 rte_flow_error_set(error, EINVAL,
344 RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
349 memset(pst, 0, sizeof(*pst));
350 pst->npc = &hw->npc_flow;
354 /* Use integral byte offset */
355 key_offset = pst->npc->keyx_len[flow->nix_intf];
356 key_offset = (key_offset + 7) / 8;
358 /* Location where LDATA would begin */
359 pst->mcam_data = (uint8_t *)flow->mcam_data;
360 pst->mcam_mask = (uint8_t *)flow->mcam_mask;
362 while (pattern->type != RTE_FLOW_ITEM_TYPE_END &&
363 layer < RTE_DIM(parse_stage_funcs)) {
364 otx2_npc_dbg("Pattern type = %d", pattern->type);
366 /* Skip place-holders */
367 pattern = otx2_flow_skip_void_and_any_items(pattern);
369 pst->pattern = pattern;
370 otx2_npc_dbg("Is tunnel = %d, layer = %d", pst->tunnel, layer);
371 rc = parse_stage_funcs[layer](pst);
378 * Parse stage function sets pst->pattern to
379 * 1 past the last item it consumed.
381 pattern = pst->pattern;
387 /* Skip trailing place-holders */
388 pattern = otx2_flow_skip_void_and_any_items(pattern);
390 /* Are there more items than what we can handle? */
391 if (pattern->type != RTE_FLOW_ITEM_TYPE_END) {
392 rte_flow_error_set(error, ENOTSUP,
393 RTE_FLOW_ERROR_TYPE_ITEM, pattern,
394 "unsupported item in the sequence");
402 flow_parse_rule(struct rte_eth_dev *dev,
403 const struct rte_flow_attr *attr,
404 const struct rte_flow_item pattern[],
405 const struct rte_flow_action actions[],
406 struct rte_flow_error *error,
407 struct rte_flow *flow,
408 struct otx2_parse_state *pst)
412 /* Check attributes */
413 err = flow_parse_attr(dev, attr, error, flow);
418 err = otx2_flow_parse_actions(dev, attr, actions, error, flow);
423 err = flow_parse_pattern(dev, pattern, error, flow, pst);
427 /* Check for overlaps? */
432 otx2_flow_validate(struct rte_eth_dev *dev,
433 const struct rte_flow_attr *attr,
434 const struct rte_flow_item pattern[],
435 const struct rte_flow_action actions[],
436 struct rte_flow_error *error)
438 struct otx2_parse_state parse_state;
439 struct rte_flow flow;
441 memset(&flow, 0, sizeof(flow));
442 return flow_parse_rule(dev, attr, pattern, actions, error, &flow,
446 static struct rte_flow *
447 otx2_flow_create(struct rte_eth_dev *dev,
448 const struct rte_flow_attr *attr,
449 const struct rte_flow_item pattern[],
450 const struct rte_flow_action actions[],
451 struct rte_flow_error *error)
453 struct otx2_eth_dev *hw = dev->data->dev_private;
454 struct otx2_parse_state parse_state;
455 struct otx2_mbox *mbox = hw->mbox;
456 struct rte_flow *flow, *flow_iter;
457 struct otx2_flow_list *list;
460 flow = rte_zmalloc("otx2_rte_flow", sizeof(*flow), 0);
462 rte_flow_error_set(error, ENOMEM,
463 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
465 "Memory allocation failed");
468 memset(flow, 0, sizeof(*flow));
470 rc = flow_parse_rule(dev, attr, pattern, actions, error, flow,
475 rc = flow_program_npc(&parse_state, mbox, &hw->npc_flow);
477 rte_flow_error_set(error, EIO,
478 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
480 "Failed to insert filter");
484 rc = flow_program_rss_action(dev, actions, flow);
486 rte_flow_error_set(error, EIO,
487 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
489 "Failed to program rss action");
494 list = &hw->npc_flow.flow_list[flow->priority];
495 /* List in ascending order of mcam entries */
496 TAILQ_FOREACH(flow_iter, list, next) {
497 if (flow_iter->mcam_id > flow->mcam_id) {
498 TAILQ_INSERT_BEFORE(flow_iter, flow, next);
503 TAILQ_INSERT_TAIL(list, flow, next);
512 otx2_flow_destroy(struct rte_eth_dev *dev,
513 struct rte_flow *flow,
514 struct rte_flow_error *error)
516 struct otx2_eth_dev *hw = dev->data->dev_private;
517 struct otx2_npc_flow_info *npc = &hw->npc_flow;
518 struct otx2_mbox *mbox = hw->mbox;
519 struct rte_bitmap *bmap;
523 match_id = (flow->npc_action >> NIX_RX_ACT_MATCH_OFFSET) &
524 NIX_RX_ACT_MATCH_MASK;
526 if (match_id && match_id < OTX2_FLOW_ACTION_FLAG_DEFAULT) {
527 if (rte_atomic32_read(&npc->mark_actions) == 0)
530 /* Clear mark offload flag if there are no more mark actions */
531 if (rte_atomic32_sub_return(&npc->mark_actions, 1) == 0)
532 hw->rx_offload_flags &= ~NIX_RX_OFFLOAD_MARK_UPDATE_F;
535 rc = flow_free_rss_action(dev, flow);
537 rte_flow_error_set(error, EIO,
538 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
540 "Failed to free rss action");
543 rc = otx2_flow_mcam_free_entry(mbox, flow->mcam_id);
545 rte_flow_error_set(error, EIO,
546 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
548 "Failed to destroy filter");
551 TAILQ_REMOVE(&npc->flow_list[flow->priority], flow, next);
553 bmap = npc->live_entries[flow->priority];
554 rte_bitmap_clear(bmap, flow->mcam_id);
561 otx2_flow_flush(struct rte_eth_dev *dev,
562 struct rte_flow_error *error)
564 struct otx2_eth_dev *hw = dev->data->dev_private;
567 rc = otx2_flow_free_all_resources(hw);
569 otx2_err("Error when deleting NPC MCAM entries "
571 rte_flow_error_set(error, EIO,
572 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
574 "Failed to flush filter");
582 otx2_flow_isolate(struct rte_eth_dev *dev __rte_unused,
583 int enable __rte_unused,
584 struct rte_flow_error *error)
587 * If we support, we need to un-install the default mcam
588 * entry for this port.
591 rte_flow_error_set(error, ENOTSUP,
592 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
594 "Flow isolation not supported");
600 otx2_flow_query(struct rte_eth_dev *dev,
601 struct rte_flow *flow,
602 const struct rte_flow_action *action,
604 struct rte_flow_error *error)
606 struct otx2_eth_dev *hw = dev->data->dev_private;
607 struct rte_flow_query_count *query = data;
608 struct otx2_mbox *mbox = hw->mbox;
609 const char *errmsg = NULL;
610 int errcode = ENOTSUP;
613 if (action->type != RTE_FLOW_ACTION_TYPE_COUNT) {
614 errmsg = "Only COUNT is supported in query";
618 if (flow->ctr_id == NPC_COUNTER_NONE) {
619 errmsg = "Counter is not available";
623 rc = otx2_flow_mcam_read_counter(mbox, flow->ctr_id, &query->hits);
626 errmsg = "Error reading flow counter";
630 query->bytes_set = 0;
633 rc = otx2_flow_mcam_clear_counter(mbox, flow->ctr_id);
636 errmsg = "Error clearing flow counter";
643 rte_flow_error_set(error, errcode,
644 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
650 const struct rte_flow_ops otx2_flow_ops = {
651 .validate = otx2_flow_validate,
652 .create = otx2_flow_create,
653 .destroy = otx2_flow_destroy,
654 .flush = otx2_flow_flush,
655 .query = otx2_flow_query,
656 .isolate = otx2_flow_isolate,