1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Chelsio Communications.
5 #include "base/common.h"
6 #include "cxgbe_flow.h"
8 #define __CXGBE_FILL_FS(__v, __m, fs, elem, e) \
10 if ((fs)->mask.elem && ((fs)->val.elem != (__v))) \
11 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, \
12 NULL, "Redefined match item with" \
13 " different values found"); \
14 (fs)->val.elem = (__v); \
15 (fs)->mask.elem = (__m); \
18 #define __CXGBE_FILL_FS_MEMCPY(__v, __m, fs, elem) \
20 memcpy(&(fs)->val.elem, &(__v), sizeof(__v)); \
21 memcpy(&(fs)->mask.elem, &(__m), sizeof(__m)); \
24 #define CXGBE_FILL_FS(v, m, elem) \
25 __CXGBE_FILL_FS(v, m, fs, elem, e)
27 #define CXGBE_FILL_FS_MEMCPY(v, m, elem) \
28 __CXGBE_FILL_FS_MEMCPY(v, m, fs, elem)
31 cxgbe_validate_item(const struct rte_flow_item *i, struct rte_flow_error *e)
33 /* rte_flow specification does not allow it. */
34 if (!i->spec && (i->mask || i->last))
35 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
36 i, "last or mask given without spec");
38 * We don't support it.
39 * Although, we can support values in last as 0's or last == spec.
40 * But this will not provide user with any additional functionality
41 * and will only increase the complexity for us.
44 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
45 i, "last is not supported by chelsio pmd");
50 * Apart from the 4-tuple IPv4/IPv6 - TCP/UDP information,
51 * there's only 40-bits available to store match fields.
52 * So, to save space, optimize filter spec for some common
53 * known fields that hardware can parse against incoming
54 * packets automatically.
57 cxgbe_tweak_filter_spec(struct adapter *adap,
58 struct ch_filter_specification *fs)
60 /* Save 16-bit ethertype field space, by setting corresponding
61 * 1-bit flags in the filter spec for common known ethertypes.
62 * When hardware sees these flags, it automatically infers and
63 * matches incoming packets against the corresponding ethertype.
65 if (fs->mask.ethtype == 0xffff) {
66 switch (fs->val.ethtype) {
67 case RTE_ETHER_TYPE_IPV4:
68 if (adap->params.tp.ethertype_shift < 0) {
69 fs->type = FILTER_TYPE_IPV4;
74 case RTE_ETHER_TYPE_IPV6:
75 if (adap->params.tp.ethertype_shift < 0) {
76 fs->type = FILTER_TYPE_IPV6;
81 case RTE_ETHER_TYPE_VLAN:
82 if (adap->params.tp.ethertype_shift < 0 &&
83 adap->params.tp.vlan_shift >= 0) {
84 fs->val.ivlan_vld = 1;
85 fs->mask.ivlan_vld = 1;
90 case RTE_ETHER_TYPE_QINQ:
91 if (adap->params.tp.ethertype_shift < 0 &&
92 adap->params.tp.vnic_shift >= 0) {
93 fs->val.ovlan_vld = 1;
94 fs->mask.ovlan_vld = 1;
106 cxgbe_fill_filter_region(struct adapter *adap,
107 struct ch_filter_specification *fs)
109 struct tp_params *tp = &adap->params.tp;
110 u64 hash_filter_mask = tp->hash_filter_mask;
115 if (!is_hashfilter(adap))
119 uint8_t biton[16] = {0xff, 0xff, 0xff, 0xff,
120 0xff, 0xff, 0xff, 0xff,
121 0xff, 0xff, 0xff, 0xff,
122 0xff, 0xff, 0xff, 0xff};
123 uint8_t bitoff[16] = {0};
125 if (!memcmp(fs->val.lip, bitoff, sizeof(bitoff)) ||
126 !memcmp(fs->val.fip, bitoff, sizeof(bitoff)) ||
127 memcmp(fs->mask.lip, biton, sizeof(biton)) ||
128 memcmp(fs->mask.fip, biton, sizeof(biton)))
131 uint32_t biton = 0xffffffff;
132 uint32_t bitoff = 0x0U;
134 if (!memcmp(fs->val.lip, &bitoff, sizeof(bitoff)) ||
135 !memcmp(fs->val.fip, &bitoff, sizeof(bitoff)) ||
136 memcmp(fs->mask.lip, &biton, sizeof(biton)) ||
137 memcmp(fs->mask.fip, &biton, sizeof(biton)))
141 if (!fs->val.lport || fs->mask.lport != 0xffff)
143 if (!fs->val.fport || fs->mask.fport != 0xffff)
146 if (tp->protocol_shift >= 0)
147 ntuple_mask |= (u64)fs->mask.proto << tp->protocol_shift;
148 if (tp->ethertype_shift >= 0)
149 ntuple_mask |= (u64)fs->mask.ethtype << tp->ethertype_shift;
150 if (tp->port_shift >= 0)
151 ntuple_mask |= (u64)fs->mask.iport << tp->port_shift;
152 if (tp->macmatch_shift >= 0)
153 ntuple_mask |= (u64)fs->mask.macidx << tp->macmatch_shift;
154 if (tp->vlan_shift >= 0 && fs->mask.ivlan_vld)
155 ntuple_mask |= (u64)(F_FT_VLAN_VLD | fs->mask.ivlan) <<
157 if (tp->vnic_shift >= 0) {
158 if (fs->mask.ovlan_vld)
159 ntuple_mask |= (u64)(fs->val.ovlan_vld << 16 |
160 fs->mask.ovlan) << tp->vnic_shift;
161 else if (fs->mask.pfvf_vld)
162 ntuple_mask |= (u64)(fs->mask.pfvf_vld << 16 |
164 fs->mask.vf) << tp->vnic_shift;
166 if (tp->tos_shift >= 0)
167 ntuple_mask |= (u64)fs->mask.tos << tp->tos_shift;
169 if (ntuple_mask != hash_filter_mask)
172 fs->cap = 1; /* use hash region */
176 ch_rte_parsetype_eth(const void *dmask, const struct rte_flow_item *item,
177 struct ch_filter_specification *fs,
178 struct rte_flow_error *e)
180 const struct rte_flow_item_eth *spec = item->spec;
181 const struct rte_flow_item_eth *umask = item->mask;
182 const struct rte_flow_item_eth *mask;
184 /* If user has not given any mask, then use chelsio supported mask. */
185 mask = umask ? umask : (const struct rte_flow_item_eth *)dmask;
190 /* we don't support SRC_MAC filtering*/
191 if (!rte_is_zero_ether_addr(&mask->src))
192 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
194 "src mac filtering not supported");
196 if (!rte_is_zero_ether_addr(&mask->dst)) {
197 const u8 *addr = (const u8 *)&spec->dst.addr_bytes[0];
198 const u8 *m = (const u8 *)&mask->dst.addr_bytes[0];
199 struct rte_flow *flow = (struct rte_flow *)fs->private;
200 struct port_info *pi = (struct port_info *)
201 (flow->dev->data->dev_private);
204 idx = cxgbe_mpstcam_alloc(pi, addr, m);
206 return rte_flow_error_set(e, idx,
207 RTE_FLOW_ERROR_TYPE_ITEM,
208 NULL, "unable to allocate mac"
210 CXGBE_FILL_FS(idx, 0x1ff, macidx);
213 CXGBE_FILL_FS(be16_to_cpu(spec->type),
214 be16_to_cpu(mask->type), ethtype);
220 ch_rte_parsetype_port(const void *dmask, const struct rte_flow_item *item,
221 struct ch_filter_specification *fs,
222 struct rte_flow_error *e)
224 const struct rte_flow_item_phy_port *val = item->spec;
225 const struct rte_flow_item_phy_port *umask = item->mask;
226 const struct rte_flow_item_phy_port *mask;
228 mask = umask ? umask : (const struct rte_flow_item_phy_port *)dmask;
231 return 0; /* Wildcard, match all physical ports */
233 if (val->index > 0x7)
234 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
236 "port index upto 0x7 is supported");
238 CXGBE_FILL_FS(val->index, mask->index, iport);
244 ch_rte_parsetype_vlan(const void *dmask, const struct rte_flow_item *item,
245 struct ch_filter_specification *fs,
246 struct rte_flow_error *e)
248 const struct rte_flow_item_vlan *spec = item->spec;
249 const struct rte_flow_item_vlan *umask = item->mask;
250 const struct rte_flow_item_vlan *mask;
252 /* If user has not given any mask, then use chelsio supported mask. */
253 mask = umask ? umask : (const struct rte_flow_item_vlan *)dmask;
255 if (!fs->mask.ethtype)
256 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
258 "Can't parse VLAN item without knowing ethertype");
260 /* If ethertype is already set and is not VLAN (0x8100) or
261 * QINQ(0x88A8), then don't proceed further. Otherwise,
262 * reset the outer ethertype, so that it can be replaced by
263 * innermost ethertype. Note that hardware will automatically
264 * match against VLAN or QINQ packets, based on 'ivlan_vld' or
265 * 'ovlan_vld' bit set in Chelsio filter spec, respectively.
267 if (fs->mask.ethtype) {
268 if (fs->val.ethtype != RTE_ETHER_TYPE_VLAN &&
269 fs->val.ethtype != RTE_ETHER_TYPE_QINQ)
270 return rte_flow_error_set(e, EINVAL,
271 RTE_FLOW_ERROR_TYPE_ITEM,
273 "Ethertype must be 0x8100 or 0x88a8");
276 if (fs->val.ethtype == RTE_ETHER_TYPE_QINQ) {
277 CXGBE_FILL_FS(1, 1, ovlan_vld);
279 CXGBE_FILL_FS(be16_to_cpu(spec->tci),
280 be16_to_cpu(mask->tci), ovlan);
282 fs->mask.ethtype = 0;
285 } else if (fs->val.ethtype == RTE_ETHER_TYPE_VLAN) {
286 CXGBE_FILL_FS(1, 1, ivlan_vld);
288 CXGBE_FILL_FS(be16_to_cpu(spec->tci),
289 be16_to_cpu(mask->tci), ivlan);
291 fs->mask.ethtype = 0;
297 CXGBE_FILL_FS(be16_to_cpu(spec->inner_type),
298 be16_to_cpu(mask->inner_type), ethtype);
304 ch_rte_parsetype_pf(const void *dmask __rte_unused,
305 const struct rte_flow_item *item __rte_unused,
306 struct ch_filter_specification *fs,
307 struct rte_flow_error *e __rte_unused)
309 struct rte_flow *flow = (struct rte_flow *)fs->private;
310 struct rte_eth_dev *dev = flow->dev;
311 struct adapter *adap = ethdev2adap(dev);
313 CXGBE_FILL_FS(1, 1, pfvf_vld);
315 CXGBE_FILL_FS(adap->pf, 0x7, pf);
320 ch_rte_parsetype_vf(const void *dmask, const struct rte_flow_item *item,
321 struct ch_filter_specification *fs,
322 struct rte_flow_error *e)
324 const struct rte_flow_item_vf *umask = item->mask;
325 const struct rte_flow_item_vf *val = item->spec;
326 const struct rte_flow_item_vf *mask;
328 /* If user has not given any mask, then use chelsio supported mask. */
329 mask = umask ? umask : (const struct rte_flow_item_vf *)dmask;
331 CXGBE_FILL_FS(1, 1, pfvf_vld);
334 return 0; /* Wildcard, match all Vf */
336 if (val->id > UCHAR_MAX)
337 return rte_flow_error_set(e, EINVAL,
338 RTE_FLOW_ERROR_TYPE_ITEM,
342 CXGBE_FILL_FS(val->id, mask->id, vf);
348 ch_rte_parsetype_udp(const void *dmask, const struct rte_flow_item *item,
349 struct ch_filter_specification *fs,
350 struct rte_flow_error *e)
352 const struct rte_flow_item_udp *val = item->spec;
353 const struct rte_flow_item_udp *umask = item->mask;
354 const struct rte_flow_item_udp *mask;
356 mask = umask ? umask : (const struct rte_flow_item_udp *)dmask;
358 if (mask->hdr.dgram_len || mask->hdr.dgram_cksum)
359 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
361 "udp: only src/dst port supported");
363 CXGBE_FILL_FS(IPPROTO_UDP, 0xff, proto);
366 CXGBE_FILL_FS(be16_to_cpu(val->hdr.src_port),
367 be16_to_cpu(mask->hdr.src_port), fport);
368 CXGBE_FILL_FS(be16_to_cpu(val->hdr.dst_port),
369 be16_to_cpu(mask->hdr.dst_port), lport);
374 ch_rte_parsetype_tcp(const void *dmask, const struct rte_flow_item *item,
375 struct ch_filter_specification *fs,
376 struct rte_flow_error *e)
378 const struct rte_flow_item_tcp *val = item->spec;
379 const struct rte_flow_item_tcp *umask = item->mask;
380 const struct rte_flow_item_tcp *mask;
382 mask = umask ? umask : (const struct rte_flow_item_tcp *)dmask;
384 if (mask->hdr.sent_seq || mask->hdr.recv_ack || mask->hdr.data_off ||
385 mask->hdr.tcp_flags || mask->hdr.rx_win || mask->hdr.cksum ||
387 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
389 "tcp: only src/dst port supported");
391 CXGBE_FILL_FS(IPPROTO_TCP, 0xff, proto);
394 CXGBE_FILL_FS(be16_to_cpu(val->hdr.src_port),
395 be16_to_cpu(mask->hdr.src_port), fport);
396 CXGBE_FILL_FS(be16_to_cpu(val->hdr.dst_port),
397 be16_to_cpu(mask->hdr.dst_port), lport);
402 ch_rte_parsetype_ipv4(const void *dmask, const struct rte_flow_item *item,
403 struct ch_filter_specification *fs,
404 struct rte_flow_error *e)
406 const struct rte_flow_item_ipv4 *val = item->spec;
407 const struct rte_flow_item_ipv4 *umask = item->mask;
408 const struct rte_flow_item_ipv4 *mask;
410 mask = umask ? umask : (const struct rte_flow_item_ipv4 *)dmask;
412 if (mask->hdr.time_to_live)
413 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
414 item, "ttl is not supported");
416 if (fs->mask.ethtype &&
417 (fs->val.ethtype != RTE_ETHER_TYPE_IPV4))
418 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
420 "Couldn't find IPv4 ethertype");
421 fs->type = FILTER_TYPE_IPV4;
423 return 0; /* ipv4 wild card */
425 CXGBE_FILL_FS(val->hdr.next_proto_id, mask->hdr.next_proto_id, proto);
426 CXGBE_FILL_FS_MEMCPY(val->hdr.dst_addr, mask->hdr.dst_addr, lip);
427 CXGBE_FILL_FS_MEMCPY(val->hdr.src_addr, mask->hdr.src_addr, fip);
428 CXGBE_FILL_FS(val->hdr.type_of_service, mask->hdr.type_of_service, tos);
434 ch_rte_parsetype_ipv6(const void *dmask, const struct rte_flow_item *item,
435 struct ch_filter_specification *fs,
436 struct rte_flow_error *e)
438 const struct rte_flow_item_ipv6 *val = item->spec;
439 const struct rte_flow_item_ipv6 *umask = item->mask;
440 const struct rte_flow_item_ipv6 *mask;
441 u32 vtc_flow, vtc_flow_mask;
443 mask = umask ? umask : (const struct rte_flow_item_ipv6 *)dmask;
445 vtc_flow_mask = be32_to_cpu(mask->hdr.vtc_flow);
447 if (vtc_flow_mask & RTE_IPV6_HDR_FL_MASK ||
448 mask->hdr.payload_len || mask->hdr.hop_limits)
449 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
451 "flow/hop are not supported");
453 if (fs->mask.ethtype &&
454 (fs->val.ethtype != RTE_ETHER_TYPE_IPV6))
455 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
457 "Couldn't find IPv6 ethertype");
458 fs->type = FILTER_TYPE_IPV6;
460 return 0; /* ipv6 wild card */
462 CXGBE_FILL_FS(val->hdr.proto, mask->hdr.proto, proto);
464 vtc_flow = be32_to_cpu(val->hdr.vtc_flow);
465 CXGBE_FILL_FS((vtc_flow & RTE_IPV6_HDR_TC_MASK) >>
466 RTE_IPV6_HDR_TC_SHIFT,
467 (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
468 RTE_IPV6_HDR_TC_SHIFT,
471 CXGBE_FILL_FS_MEMCPY(val->hdr.dst_addr, mask->hdr.dst_addr, lip);
472 CXGBE_FILL_FS_MEMCPY(val->hdr.src_addr, mask->hdr.src_addr, fip);
478 cxgbe_rtef_parse_attr(struct rte_flow *flow, const struct rte_flow_attr *attr,
479 struct rte_flow_error *e)
482 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR,
483 attr, "attribute:<egress> is"
486 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR,
487 attr, "group parameter is"
490 flow->fidx = attr->priority ? attr->priority - 1 : FILTER_ID_MAX;
495 static inline int check_rxq(struct rte_eth_dev *dev, uint16_t rxq)
497 struct port_info *pi = ethdev2pinfo(dev);
499 if (rxq > pi->n_rx_qsets)
504 static int cxgbe_validate_fidxondel(struct filter_entry *f, unsigned int fidx)
506 struct adapter *adap = ethdev2adap(f->dev);
507 struct ch_filter_specification fs = f->fs;
510 if (fidx >= adap->tids.nftids) {
511 dev_err(adap, "invalid flow index %d.\n", fidx);
515 nentries = cxgbe_filter_slots(adap, fs.type);
516 if (!cxgbe_is_filter_set(&adap->tids, fidx, nentries)) {
517 dev_err(adap, "Already free fidx:%d f:%p\n", fidx, f);
525 cxgbe_validate_fidxonadd(struct ch_filter_specification *fs,
526 struct adapter *adap, unsigned int fidx)
530 nentries = cxgbe_filter_slots(adap, fs->type);
531 if (cxgbe_is_filter_set(&adap->tids, fidx, nentries)) {
532 dev_err(adap, "filter index: %d is busy.\n", fidx);
536 if (fidx >= adap->tids.nftids) {
537 dev_err(adap, "filter index (%u) >= max(%u)\n",
538 fidx, adap->tids.nftids);
546 cxgbe_verify_fidx(struct rte_flow *flow, unsigned int fidx, uint8_t del)
549 return 0; /* Hash filters */
550 return del ? cxgbe_validate_fidxondel(flow->f, fidx) :
551 cxgbe_validate_fidxonadd(&flow->fs,
552 ethdev2adap(flow->dev), fidx);
555 static int cxgbe_get_fidx(struct rte_flow *flow, unsigned int *fidx)
557 struct ch_filter_specification *fs = &flow->fs;
558 struct adapter *adap = ethdev2adap(flow->dev);
560 /* For tcam get the next available slot, if default value specified */
561 if (flow->fidx == FILTER_ID_MAX) {
565 nentries = cxgbe_filter_slots(adap, fs->type);
566 idx = cxgbe_alloc_ftid(adap, nentries);
568 dev_err(adap, "unable to get a filter index in tcam\n");
571 *fidx = (unsigned int)idx;
580 cxgbe_get_flow_item_index(const struct rte_flow_item items[], u32 type)
582 const struct rte_flow_item *i;
583 int j, index = -ENOENT;
585 for (i = items, j = 0; i->type != RTE_FLOW_ITEM_TYPE_END; i++, j++) {
586 if (i->type == type) {
596 ch_rte_parse_nat(uint8_t nmode, struct ch_filter_specification *fs)
599 * BIT_0 = [src_ip], BIT_1 = [dst_ip]
600 * BIT_2 = [src_port], BIT_3 = [dst_port]
602 * Only below cases are supported as per our spec.
606 fs->nat_mode = NAT_MODE_NONE;
609 fs->nat_mode = NAT_MODE_DIP;
612 fs->nat_mode = NAT_MODE_SIP_SP;
615 fs->nat_mode = NAT_MODE_DIP_SIP_SP;
618 fs->nat_mode = NAT_MODE_DIP_DP;
621 fs->nat_mode = NAT_MODE_DIP_DP_SIP;
624 fs->nat_mode = NAT_MODE_DIP_DP_SP;
627 fs->nat_mode = NAT_MODE_ALL;
637 ch_rte_parse_atype_switch(const struct rte_flow_action *a,
638 const struct rte_flow_item items[],
640 struct ch_filter_specification *fs,
641 struct rte_flow_error *e)
643 const struct rte_flow_action_of_set_vlan_vid *vlanid;
644 const struct rte_flow_action_of_set_vlan_pcp *vlanpcp;
645 const struct rte_flow_action_of_push_vlan *pushvlan;
646 const struct rte_flow_action_set_ipv4 *ipv4;
647 const struct rte_flow_action_set_ipv6 *ipv6;
648 const struct rte_flow_action_set_tp *tp_port;
649 const struct rte_flow_action_phy_port *port;
650 const struct rte_flow_action_set_mac *mac;
655 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
656 vlanid = (const struct rte_flow_action_of_set_vlan_vid *)
658 /* If explicitly asked to push a new VLAN header,
659 * then don't set rewrite mode. Otherwise, the
660 * incoming VLAN packets will get their VLAN fields
661 * rewritten, instead of adding an additional outer
664 if (fs->newvlan != VLAN_INSERT)
665 fs->newvlan = VLAN_REWRITE;
666 tmp_vlan = fs->vlan & 0xe000;
667 fs->vlan = (be16_to_cpu(vlanid->vlan_vid) & 0xfff) | tmp_vlan;
669 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
670 vlanpcp = (const struct rte_flow_action_of_set_vlan_pcp *)
672 /* If explicitly asked to push a new VLAN header,
673 * then don't set rewrite mode. Otherwise, the
674 * incoming VLAN packets will get their VLAN fields
675 * rewritten, instead of adding an additional outer
678 if (fs->newvlan != VLAN_INSERT)
679 fs->newvlan = VLAN_REWRITE;
680 tmp_vlan = fs->vlan & 0xfff;
681 fs->vlan = (vlanpcp->vlan_pcp << 13) | tmp_vlan;
683 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
684 pushvlan = (const struct rte_flow_action_of_push_vlan *)
686 if (be16_to_cpu(pushvlan->ethertype) != RTE_ETHER_TYPE_VLAN)
687 return rte_flow_error_set(e, EINVAL,
688 RTE_FLOW_ERROR_TYPE_ACTION, a,
689 "only ethertype 0x8100 "
690 "supported for push vlan.");
691 fs->newvlan = VLAN_INSERT;
693 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
694 fs->newvlan = VLAN_REMOVE;
696 case RTE_FLOW_ACTION_TYPE_PHY_PORT:
697 port = (const struct rte_flow_action_phy_port *)a->conf;
698 fs->eport = port->index;
700 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
701 item_index = cxgbe_get_flow_item_index(items,
702 RTE_FLOW_ITEM_TYPE_IPV4);
704 return rte_flow_error_set(e, EINVAL,
705 RTE_FLOW_ERROR_TYPE_ACTION, a,
706 "No RTE_FLOW_ITEM_TYPE_IPV4 "
709 ipv4 = (const struct rte_flow_action_set_ipv4 *)a->conf;
710 memcpy(fs->nat_fip, &ipv4->ipv4_addr, sizeof(ipv4->ipv4_addr));
713 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
714 item_index = cxgbe_get_flow_item_index(items,
715 RTE_FLOW_ITEM_TYPE_IPV4);
717 return rte_flow_error_set(e, EINVAL,
718 RTE_FLOW_ERROR_TYPE_ACTION, a,
719 "No RTE_FLOW_ITEM_TYPE_IPV4 "
722 ipv4 = (const struct rte_flow_action_set_ipv4 *)a->conf;
723 memcpy(fs->nat_lip, &ipv4->ipv4_addr, sizeof(ipv4->ipv4_addr));
726 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
727 item_index = cxgbe_get_flow_item_index(items,
728 RTE_FLOW_ITEM_TYPE_IPV6);
730 return rte_flow_error_set(e, EINVAL,
731 RTE_FLOW_ERROR_TYPE_ACTION, a,
732 "No RTE_FLOW_ITEM_TYPE_IPV6 "
735 ipv6 = (const struct rte_flow_action_set_ipv6 *)a->conf;
736 memcpy(fs->nat_fip, ipv6->ipv6_addr, sizeof(ipv6->ipv6_addr));
739 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
740 item_index = cxgbe_get_flow_item_index(items,
741 RTE_FLOW_ITEM_TYPE_IPV6);
743 return rte_flow_error_set(e, EINVAL,
744 RTE_FLOW_ERROR_TYPE_ACTION, a,
745 "No RTE_FLOW_ITEM_TYPE_IPV6 "
748 ipv6 = (const struct rte_flow_action_set_ipv6 *)a->conf;
749 memcpy(fs->nat_lip, ipv6->ipv6_addr, sizeof(ipv6->ipv6_addr));
752 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
753 item_index = cxgbe_get_flow_item_index(items,
754 RTE_FLOW_ITEM_TYPE_TCP);
755 if (item_index < 0) {
757 cxgbe_get_flow_item_index(items,
758 RTE_FLOW_ITEM_TYPE_UDP);
760 return rte_flow_error_set(e, EINVAL,
761 RTE_FLOW_ERROR_TYPE_ACTION, a,
762 "No RTE_FLOW_ITEM_TYPE_TCP or "
763 "RTE_FLOW_ITEM_TYPE_UDP found");
766 tp_port = (const struct rte_flow_action_set_tp *)a->conf;
767 fs->nat_fport = be16_to_cpu(tp_port->port);
770 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
771 item_index = cxgbe_get_flow_item_index(items,
772 RTE_FLOW_ITEM_TYPE_TCP);
773 if (item_index < 0) {
775 cxgbe_get_flow_item_index(items,
776 RTE_FLOW_ITEM_TYPE_UDP);
778 return rte_flow_error_set(e, EINVAL,
779 RTE_FLOW_ERROR_TYPE_ACTION, a,
780 "No RTE_FLOW_ITEM_TYPE_TCP or "
781 "RTE_FLOW_ITEM_TYPE_UDP found");
784 tp_port = (const struct rte_flow_action_set_tp *)a->conf;
785 fs->nat_lport = be16_to_cpu(tp_port->port);
788 case RTE_FLOW_ACTION_TYPE_MAC_SWAP:
789 item_index = cxgbe_get_flow_item_index(items,
790 RTE_FLOW_ITEM_TYPE_ETH);
792 return rte_flow_error_set(e, EINVAL,
793 RTE_FLOW_ERROR_TYPE_ACTION, a,
794 "No RTE_FLOW_ITEM_TYPE_ETH "
798 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
799 item_index = cxgbe_get_flow_item_index(items,
800 RTE_FLOW_ITEM_TYPE_ETH);
802 return rte_flow_error_set(e, EINVAL,
803 RTE_FLOW_ERROR_TYPE_ACTION, a,
804 "No RTE_FLOW_ITEM_TYPE_ETH found");
805 mac = (const struct rte_flow_action_set_mac *)a->conf;
808 memcpy(fs->dmac, mac->mac_addr, sizeof(fs->dmac));
811 /* We are not supposed to come here */
812 return rte_flow_error_set(e, EINVAL,
813 RTE_FLOW_ERROR_TYPE_ACTION, a,
814 "Action not supported");
821 cxgbe_rtef_parse_actions(struct rte_flow *flow,
822 const struct rte_flow_item items[],
823 const struct rte_flow_action action[],
824 struct rte_flow_error *e)
826 struct ch_filter_specification *fs = &flow->fs;
827 uint8_t nmode = 0, nat_ipv4 = 0, nat_ipv6 = 0;
828 uint8_t vlan_set_vid = 0, vlan_set_pcp = 0;
829 const struct rte_flow_action_queue *q;
830 const struct rte_flow_action *a;
834 for (a = action; a->type != RTE_FLOW_ACTION_TYPE_END; a++) {
836 case RTE_FLOW_ACTION_TYPE_VOID:
838 case RTE_FLOW_ACTION_TYPE_DROP:
840 return rte_flow_error_set(e, EINVAL,
841 RTE_FLOW_ERROR_TYPE_ACTION, a,
842 "specify only 1 pass/drop");
843 fs->action = FILTER_DROP;
845 case RTE_FLOW_ACTION_TYPE_QUEUE:
846 q = (const struct rte_flow_action_queue *)a->conf;
848 return rte_flow_error_set(e, EINVAL,
849 RTE_FLOW_ERROR_TYPE_ACTION, q,
850 "specify rx queue index");
851 if (check_rxq(flow->dev, q->index))
852 return rte_flow_error_set(e, EINVAL,
853 RTE_FLOW_ERROR_TYPE_ACTION, q,
856 return rte_flow_error_set(e, EINVAL,
857 RTE_FLOW_ERROR_TYPE_ACTION, a,
858 "specify only 1 pass/drop");
859 fs->action = FILTER_PASS;
863 case RTE_FLOW_ACTION_TYPE_COUNT:
866 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
869 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
872 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
873 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
874 case RTE_FLOW_ACTION_TYPE_PHY_PORT:
875 case RTE_FLOW_ACTION_TYPE_MAC_SWAP:
876 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
877 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
880 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
881 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
884 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
885 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
886 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
888 /* We allow multiple switch actions, but switch is
889 * not compatible with either queue or drop
891 if (abit++ && fs->action != FILTER_SWITCH)
892 return rte_flow_error_set(e, EINVAL,
893 RTE_FLOW_ERROR_TYPE_ACTION, a,
894 "overlapping action specified");
895 if (nat_ipv4 && nat_ipv6)
896 return rte_flow_error_set(e, EINVAL,
897 RTE_FLOW_ERROR_TYPE_ACTION, a,
898 "Can't have one address ipv4 and the"
901 ret = ch_rte_parse_atype_switch(a, items, &nmode, fs,
905 fs->action = FILTER_SWITCH;
908 /* Not supported action : return error */
909 return rte_flow_error_set(e, ENOTSUP,
910 RTE_FLOW_ERROR_TYPE_ACTION,
911 a, "Action not supported");
915 if (fs->newvlan == VLAN_REWRITE && (!vlan_set_vid || !vlan_set_pcp))
916 return rte_flow_error_set(e, EINVAL,
917 RTE_FLOW_ERROR_TYPE_ACTION, a,
918 "Both OF_SET_VLAN_VID and "
919 "OF_SET_VLAN_PCP must be specified");
921 if (ch_rte_parse_nat(nmode, fs))
922 return rte_flow_error_set(e, EINVAL,
923 RTE_FLOW_ERROR_TYPE_ACTION, a,
924 "invalid settings for swich action");
928 static struct chrte_fparse parseitem[] = {
929 [RTE_FLOW_ITEM_TYPE_ETH] = {
930 .fptr = ch_rte_parsetype_eth,
931 .dmask = &(const struct rte_flow_item_eth){
932 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
933 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
938 [RTE_FLOW_ITEM_TYPE_PHY_PORT] = {
939 .fptr = ch_rte_parsetype_port,
940 .dmask = &(const struct rte_flow_item_phy_port){
945 [RTE_FLOW_ITEM_TYPE_VLAN] = {
946 .fptr = ch_rte_parsetype_vlan,
947 .dmask = &(const struct rte_flow_item_vlan){
949 .inner_type = 0xffff,
953 [RTE_FLOW_ITEM_TYPE_IPV4] = {
954 .fptr = ch_rte_parsetype_ipv4,
955 .dmask = &(const struct rte_flow_item_ipv4) {
957 .src_addr = RTE_BE32(0xffffffff),
958 .dst_addr = RTE_BE32(0xffffffff),
959 .type_of_service = 0xff,
964 [RTE_FLOW_ITEM_TYPE_IPV6] = {
965 .fptr = ch_rte_parsetype_ipv6,
966 .dmask = &(const struct rte_flow_item_ipv6) {
969 "\xff\xff\xff\xff\xff\xff\xff\xff"
970 "\xff\xff\xff\xff\xff\xff\xff\xff",
972 "\xff\xff\xff\xff\xff\xff\xff\xff"
973 "\xff\xff\xff\xff\xff\xff\xff\xff",
974 .vtc_flow = RTE_BE32(0xff000000),
979 [RTE_FLOW_ITEM_TYPE_UDP] = {
980 .fptr = ch_rte_parsetype_udp,
981 .dmask = &rte_flow_item_udp_mask,
984 [RTE_FLOW_ITEM_TYPE_TCP] = {
985 .fptr = ch_rte_parsetype_tcp,
986 .dmask = &rte_flow_item_tcp_mask,
989 [RTE_FLOW_ITEM_TYPE_PF] = {
990 .fptr = ch_rte_parsetype_pf,
994 [RTE_FLOW_ITEM_TYPE_VF] = {
995 .fptr = ch_rte_parsetype_vf,
996 .dmask = &(const struct rte_flow_item_vf){
1003 cxgbe_rtef_parse_items(struct rte_flow *flow,
1004 const struct rte_flow_item items[],
1005 struct rte_flow_error *e)
1007 struct adapter *adap = ethdev2adap(flow->dev);
1008 const struct rte_flow_item *i;
1009 char repeat[ARRAY_SIZE(parseitem)] = {0};
1011 for (i = items; i->type != RTE_FLOW_ITEM_TYPE_END; i++) {
1012 struct chrte_fparse *idx;
1015 if (i->type >= ARRAY_SIZE(parseitem))
1016 return rte_flow_error_set(e, ENOTSUP,
1017 RTE_FLOW_ERROR_TYPE_ITEM,
1018 i, "Item not supported");
1021 case RTE_FLOW_ITEM_TYPE_VOID:
1024 /* check if item is repeated */
1025 if (repeat[i->type] &&
1026 i->type != RTE_FLOW_ITEM_TYPE_VLAN)
1027 return rte_flow_error_set(e, ENOTSUP,
1028 RTE_FLOW_ERROR_TYPE_ITEM, i,
1029 "parse items cannot be repeated(except void/vlan)");
1031 repeat[i->type] = 1;
1033 /* validate the item */
1034 ret = cxgbe_validate_item(i, e);
1038 idx = &flow->item_parser[i->type];
1039 if (!idx || !idx->fptr) {
1040 return rte_flow_error_set(e, ENOTSUP,
1041 RTE_FLOW_ERROR_TYPE_ITEM, i,
1042 "Item not supported");
1044 ret = idx->fptr(idx->dmask, i, &flow->fs, e);
1051 cxgbe_fill_filter_region(adap, &flow->fs);
1052 cxgbe_tweak_filter_spec(adap, &flow->fs);
1058 cxgbe_flow_parse(struct rte_flow *flow,
1059 const struct rte_flow_attr *attr,
1060 const struct rte_flow_item item[],
1061 const struct rte_flow_action action[],
1062 struct rte_flow_error *e)
1065 /* parse user request into ch_filter_specification */
1066 ret = cxgbe_rtef_parse_attr(flow, attr, e);
1069 ret = cxgbe_rtef_parse_items(flow, item, e);
1072 return cxgbe_rtef_parse_actions(flow, item, action, e);
1075 static int __cxgbe_flow_create(struct rte_eth_dev *dev, struct rte_flow *flow)
1077 struct ch_filter_specification *fs = &flow->fs;
1078 struct adapter *adap = ethdev2adap(dev);
1079 struct tid_info *t = &adap->tids;
1080 struct filter_ctx ctx;
1084 if (cxgbe_get_fidx(flow, &fidx))
1086 if (cxgbe_verify_fidx(flow, fidx, 0))
1089 t4_init_completion(&ctx.completion);
1090 /* go create the filter */
1091 err = cxgbe_set_filter(dev, fidx, fs, &ctx);
1093 dev_err(adap, "Error %d while creating filter.\n", err);
1097 /* Poll the FW for reply */
1098 err = cxgbe_poll_for_completion(&adap->sge.fw_evtq,
1100 CXGBE_FLOW_POLL_CNT,
1103 dev_err(adap, "Filter set operation timed out (%d)\n", err);
1107 dev_err(adap, "Hardware error %d while creating the filter.\n",
1112 if (fs->cap) { /* to destroy the filter */
1113 flow->fidx = ctx.tid;
1114 flow->f = lookup_tid(t, ctx.tid);
1117 flow->f = &adap->tids.ftid_tab[fidx];
1123 static struct rte_flow *
1124 cxgbe_flow_create(struct rte_eth_dev *dev,
1125 const struct rte_flow_attr *attr,
1126 const struct rte_flow_item item[],
1127 const struct rte_flow_action action[],
1128 struct rte_flow_error *e)
1130 struct adapter *adap = ethdev2adap(dev);
1131 struct rte_flow *flow;
1134 flow = t4_os_alloc(sizeof(struct rte_flow));
1136 rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1137 NULL, "Unable to allocate memory for"
1142 flow->item_parser = parseitem;
1144 flow->fs.private = (void *)flow;
1146 if (cxgbe_flow_parse(flow, attr, item, action, e)) {
1151 t4_os_lock(&adap->flow_lock);
1152 /* go, interact with cxgbe_filter */
1153 ret = __cxgbe_flow_create(dev, flow);
1154 t4_os_unlock(&adap->flow_lock);
1156 rte_flow_error_set(e, ret, RTE_FLOW_ERROR_TYPE_HANDLE,
1157 NULL, "Unable to create flow rule");
1162 flow->f->private = flow; /* Will be used during flush */
1167 static int __cxgbe_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1169 struct adapter *adap = ethdev2adap(dev);
1170 struct filter_entry *f = flow->f;
1171 struct ch_filter_specification *fs;
1172 struct filter_ctx ctx;
1176 if (cxgbe_verify_fidx(flow, flow->fidx, 1))
1179 t4_init_completion(&ctx.completion);
1180 err = cxgbe_del_filter(dev, flow->fidx, fs, &ctx);
1182 dev_err(adap, "Error %d while deleting filter.\n", err);
1186 /* Poll the FW for reply */
1187 err = cxgbe_poll_for_completion(&adap->sge.fw_evtq,
1189 CXGBE_FLOW_POLL_CNT,
1192 dev_err(adap, "Filter delete operation timed out (%d)\n", err);
1196 dev_err(adap, "Hardware error %d while deleting the filter.\n",
1202 if (fs->mask.macidx) {
1203 struct port_info *pi = (struct port_info *)
1204 (dev->data->dev_private);
1207 ret = cxgbe_mpstcam_remove(pi, fs->val.macidx);
1216 cxgbe_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
1217 struct rte_flow_error *e)
1219 struct adapter *adap = ethdev2adap(dev);
1222 t4_os_lock(&adap->flow_lock);
1223 ret = __cxgbe_flow_destroy(dev, flow);
1224 t4_os_unlock(&adap->flow_lock);
1226 return rte_flow_error_set(e, ret, RTE_FLOW_ERROR_TYPE_HANDLE,
1227 flow, "error destroying filter.");
1232 static int __cxgbe_flow_query(struct rte_flow *flow, u64 *count,
1235 struct adapter *adap = ethdev2adap(flow->dev);
1236 struct ch_filter_specification fs = flow->f->fs;
1237 unsigned int fidx = flow->fidx;
1240 ret = cxgbe_get_filter_count(adap, fidx, count, fs.cap, 0);
1243 return cxgbe_get_filter_count(adap, fidx, byte_count, fs.cap, 1);
1247 cxgbe_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
1248 const struct rte_flow_action *action, void *data,
1249 struct rte_flow_error *e)
1251 struct adapter *adap = ethdev2adap(flow->dev);
1252 struct ch_filter_specification fs;
1253 struct rte_flow_query_count *c;
1254 struct filter_entry *f;
1262 if (action->type != RTE_FLOW_ACTION_TYPE_COUNT)
1263 return rte_flow_error_set(e, ENOTSUP,
1264 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1265 "only count supported for query");
1268 * This is a valid operation, Since we are allowed to do chelsio
1269 * specific operations in rte side of our code but not vise-versa
1271 * So, fs can be queried/modified here BUT rte_flow_query_count
1272 * cannot be worked on by the lower layer since we want to maintain
1273 * it as rte_flow agnostic.
1276 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
1277 &fs, "filter hit counters were not"
1278 " enabled during filter creation");
1280 c = (struct rte_flow_query_count *)data;
1282 t4_os_lock(&adap->flow_lock);
1283 ret = __cxgbe_flow_query(flow, &c->hits, &c->bytes);
1285 rte_flow_error_set(e, -ret, RTE_FLOW_ERROR_TYPE_ACTION,
1286 f, "cxgbe pmd failed to perform query");
1290 /* Query was successful */
1294 cxgbe_clear_filter_count(adap, flow->fidx, f->fs.cap, true);
1297 t4_os_unlock(&adap->flow_lock);
1302 cxgbe_flow_validate(struct rte_eth_dev *dev,
1303 const struct rte_flow_attr *attr,
1304 const struct rte_flow_item item[],
1305 const struct rte_flow_action action[],
1306 struct rte_flow_error *e)
1308 struct adapter *adap = ethdev2adap(dev);
1309 struct rte_flow *flow;
1313 flow = t4_os_alloc(sizeof(struct rte_flow));
1315 return rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1317 "Unable to allocate memory for filter_entry");
1319 flow->item_parser = parseitem;
1322 ret = cxgbe_flow_parse(flow, attr, item, action, e);
1328 if (cxgbe_validate_filter(adap, &flow->fs)) {
1330 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
1332 "validation failed. Check f/w config file.");
1335 t4_os_lock(&adap->flow_lock);
1336 if (cxgbe_get_fidx(flow, &fidx)) {
1337 ret = rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1338 NULL, "no memory in tcam.");
1342 if (cxgbe_verify_fidx(flow, fidx, 0)) {
1343 ret = rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
1344 NULL, "validation failed");
1349 t4_os_unlock(&adap->flow_lock);
1355 * @ret : > 0 filter destroyed succsesfully
1356 * < 0 error destroying filter
1357 * == 1 filter not active / not found
1360 cxgbe_check_n_destroy(struct filter_entry *f, struct rte_eth_dev *dev)
1362 if (f && (f->valid || f->pending) &&
1363 f->dev == dev && /* Only if user has asked for this port */
1364 f->private) /* We (rte_flow) created this filter */
1365 return __cxgbe_flow_destroy(dev, (struct rte_flow *)f->private);
1369 static int cxgbe_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *e)
1371 struct adapter *adap = ethdev2adap(dev);
1375 t4_os_lock(&adap->flow_lock);
1376 if (adap->tids.ftid_tab) {
1377 struct filter_entry *f = &adap->tids.ftid_tab[0];
1379 for (i = 0; i < adap->tids.nftids; i++, f++) {
1380 ret = cxgbe_check_n_destroy(f, dev);
1382 rte_flow_error_set(e, ret,
1383 RTE_FLOW_ERROR_TYPE_HANDLE,
1385 "error destroying TCAM "
1392 if (is_hashfilter(adap) && adap->tids.tid_tab) {
1393 struct filter_entry *f;
1395 for (i = adap->tids.hash_base; i <= adap->tids.ntids; i++) {
1396 f = (struct filter_entry *)adap->tids.tid_tab[i];
1398 ret = cxgbe_check_n_destroy(f, dev);
1400 rte_flow_error_set(e, ret,
1401 RTE_FLOW_ERROR_TYPE_HANDLE,
1403 "error destroying HASH "
1411 t4_os_unlock(&adap->flow_lock);
1412 return ret >= 0 ? 0 : ret;
1415 static const struct rte_flow_ops cxgbe_flow_ops = {
1416 .validate = cxgbe_flow_validate,
1417 .create = cxgbe_flow_create,
1418 .destroy = cxgbe_flow_destroy,
1419 .flush = cxgbe_flow_flush,
1420 .query = cxgbe_flow_query,
1425 cxgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
1426 enum rte_filter_type filter_type,
1427 enum rte_filter_op filter_op,
1433 switch (filter_type) {
1434 case RTE_ETH_FILTER_GENERIC:
1435 if (filter_op != RTE_ETH_FILTER_GET)
1437 *(const void **)arg = &cxgbe_flow_ops;