net/sfc: do not use RSS context if it is not required
[dpdk.git] / drivers / net / sfc / sfc_flow.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  *
3  * Copyright (c) 2017-2018 Solarflare Communications Inc.
4  * All rights reserved.
5  *
6  * This software was jointly developed between OKTET Labs (under contract
7  * for Solarflare) and Solarflare Communications, Inc.
8  */
9
10 #include <rte_byteorder.h>
11 #include <rte_tailq.h>
12 #include <rte_common.h>
13 #include <rte_ethdev_driver.h>
14 #include <rte_eth_ctrl.h>
15 #include <rte_ether.h>
16 #include <rte_flow.h>
17 #include <rte_flow_driver.h>
18
19 #include "efx.h"
20
21 #include "sfc.h"
22 #include "sfc_rx.h"
23 #include "sfc_filter.h"
24 #include "sfc_flow.h"
25 #include "sfc_log.h"
26
27 /*
28  * At now flow API is implemented in such a manner that each
29  * flow rule is converted to one or more hardware filters.
30  * All elements of flow rule (attributes, pattern items, actions)
31  * correspond to one or more fields in the efx_filter_spec_s structure
32  * that is responsible for the hardware filter.
33  * If some required field is unset in the flow rule, then a handful
34  * of filter copies will be created to cover all possible values
35  * of such a field.
36  */
37
38 enum sfc_flow_item_layers {
39         SFC_FLOW_ITEM_ANY_LAYER,
40         SFC_FLOW_ITEM_START_LAYER,
41         SFC_FLOW_ITEM_L2,
42         SFC_FLOW_ITEM_L3,
43         SFC_FLOW_ITEM_L4,
44 };
45
46 typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item,
47                                   efx_filter_spec_t *spec,
48                                   struct rte_flow_error *error);
49
50 struct sfc_flow_item {
51         enum rte_flow_item_type type;           /* Type of item */
52         enum sfc_flow_item_layers layer;        /* Layer of item */
53         enum sfc_flow_item_layers prev_layer;   /* Previous layer of item */
54         sfc_flow_item_parse *parse;             /* Parsing function */
55 };
56
57 static sfc_flow_item_parse sfc_flow_parse_void;
58 static sfc_flow_item_parse sfc_flow_parse_eth;
59 static sfc_flow_item_parse sfc_flow_parse_vlan;
60 static sfc_flow_item_parse sfc_flow_parse_ipv4;
61 static sfc_flow_item_parse sfc_flow_parse_ipv6;
62 static sfc_flow_item_parse sfc_flow_parse_tcp;
63 static sfc_flow_item_parse sfc_flow_parse_udp;
64 static sfc_flow_item_parse sfc_flow_parse_vxlan;
65 static sfc_flow_item_parse sfc_flow_parse_geneve;
66 static sfc_flow_item_parse sfc_flow_parse_nvgre;
67
68 typedef int (sfc_flow_spec_set_vals)(struct sfc_flow_spec *spec,
69                                      unsigned int filters_count_for_one_val,
70                                      struct rte_flow_error *error);
71
72 typedef boolean_t (sfc_flow_spec_check)(efx_filter_match_flags_t match,
73                                         efx_filter_spec_t *spec,
74                                         struct sfc_filter *filter);
75
76 struct sfc_flow_copy_flag {
77         /* EFX filter specification match flag */
78         efx_filter_match_flags_t flag;
79         /* Number of values of corresponding field */
80         unsigned int vals_count;
81         /* Function to set values in specifications */
82         sfc_flow_spec_set_vals *set_vals;
83         /*
84          * Function to check that the specification is suitable
85          * for adding this match flag
86          */
87         sfc_flow_spec_check *spec_check;
88 };
89
90 static sfc_flow_spec_set_vals sfc_flow_set_unknown_dst_flags;
91 static sfc_flow_spec_check sfc_flow_check_unknown_dst_flags;
92 static sfc_flow_spec_set_vals sfc_flow_set_ethertypes;
93 static sfc_flow_spec_set_vals sfc_flow_set_ifrm_unknown_dst_flags;
94 static sfc_flow_spec_check sfc_flow_check_ifrm_unknown_dst_flags;
95
96 static boolean_t
97 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
98 {
99         uint8_t sum = 0;
100         unsigned int i;
101
102         for (i = 0; i < size; i++)
103                 sum |= buf[i];
104
105         return (sum == 0) ? B_TRUE : B_FALSE;
106 }
107
108 /*
109  * Validate item and prepare structures spec and mask for parsing
110  */
111 static int
112 sfc_flow_parse_init(const struct rte_flow_item *item,
113                     const void **spec_ptr,
114                     const void **mask_ptr,
115                     const void *supp_mask,
116                     const void *def_mask,
117                     unsigned int size,
118                     struct rte_flow_error *error)
119 {
120         const uint8_t *spec;
121         const uint8_t *mask;
122         const uint8_t *last;
123         uint8_t supp;
124         unsigned int i;
125
126         if (item == NULL) {
127                 rte_flow_error_set(error, EINVAL,
128                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
129                                    "NULL item");
130                 return -rte_errno;
131         }
132
133         if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) {
134                 rte_flow_error_set(error, EINVAL,
135                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
136                                    "Mask or last is set without spec");
137                 return -rte_errno;
138         }
139
140         /*
141          * If "mask" is not set, default mask is used,
142          * but if default mask is NULL, "mask" should be set
143          */
144         if (item->mask == NULL) {
145                 if (def_mask == NULL) {
146                         rte_flow_error_set(error, EINVAL,
147                                 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
148                                 "Mask should be specified");
149                         return -rte_errno;
150                 }
151
152                 mask = def_mask;
153         } else {
154                 mask = item->mask;
155         }
156
157         spec = item->spec;
158         last = item->last;
159
160         if (spec == NULL)
161                 goto exit;
162
163         /*
164          * If field values in "last" are either 0 or equal to the corresponding
165          * values in "spec" then they are ignored
166          */
167         if (last != NULL &&
168             !sfc_flow_is_zero(last, size) &&
169             memcmp(last, spec, size) != 0) {
170                 rte_flow_error_set(error, ENOTSUP,
171                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
172                                    "Ranging is not supported");
173                 return -rte_errno;
174         }
175
176         if (supp_mask == NULL) {
177                 rte_flow_error_set(error, EINVAL,
178                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
179                         "Supported mask for item should be specified");
180                 return -rte_errno;
181         }
182
183         /* Check that mask does not ask for more match than supp_mask */
184         for (i = 0; i < size; i++) {
185                 supp = ((const uint8_t *)supp_mask)[i];
186
187                 if (~supp & mask[i]) {
188                         rte_flow_error_set(error, ENOTSUP,
189                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
190                                            "Item's field is not supported");
191                         return -rte_errno;
192                 }
193         }
194
195 exit:
196         *spec_ptr = spec;
197         *mask_ptr = mask;
198         return 0;
199 }
200
201 /*
202  * Protocol parsers.
203  * Masking is not supported, so masks in items should be either
204  * full or empty (zeroed) and set only for supported fields which
205  * are specified in the supp_mask.
206  */
207
208 static int
209 sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item,
210                     __rte_unused efx_filter_spec_t *efx_spec,
211                     __rte_unused struct rte_flow_error *error)
212 {
213         return 0;
214 }
215
216 /**
217  * Convert Ethernet item to EFX filter specification.
218  *
219  * @param item[in]
220  *   Item specification. Outer frame specification may only comprise
221  *   source/destination addresses and Ethertype field.
222  *   Inner frame specification may contain destination address only.
223  *   There is support for individual/group mask as well as for empty and full.
224  *   If the mask is NULL, default mask will be used. Ranging is not supported.
225  * @param efx_spec[in, out]
226  *   EFX filter specification to update.
227  * @param[out] error
228  *   Perform verbose error reporting if not NULL.
229  */
230 static int
231 sfc_flow_parse_eth(const struct rte_flow_item *item,
232                    efx_filter_spec_t *efx_spec,
233                    struct rte_flow_error *error)
234 {
235         int rc;
236         const struct rte_flow_item_eth *spec = NULL;
237         const struct rte_flow_item_eth *mask = NULL;
238         const struct rte_flow_item_eth supp_mask = {
239                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
240                 .src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
241                 .type = 0xffff,
242         };
243         const struct rte_flow_item_eth ifrm_supp_mask = {
244                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
245         };
246         const uint8_t ig_mask[EFX_MAC_ADDR_LEN] = {
247                 0x01, 0x00, 0x00, 0x00, 0x00, 0x00
248         };
249         const struct rte_flow_item_eth *supp_mask_p;
250         const struct rte_flow_item_eth *def_mask_p;
251         uint8_t *loc_mac = NULL;
252         boolean_t is_ifrm = (efx_spec->efs_encap_type !=
253                 EFX_TUNNEL_PROTOCOL_NONE);
254
255         if (is_ifrm) {
256                 supp_mask_p = &ifrm_supp_mask;
257                 def_mask_p = &ifrm_supp_mask;
258                 loc_mac = efx_spec->efs_ifrm_loc_mac;
259         } else {
260                 supp_mask_p = &supp_mask;
261                 def_mask_p = &rte_flow_item_eth_mask;
262                 loc_mac = efx_spec->efs_loc_mac;
263         }
264
265         rc = sfc_flow_parse_init(item,
266                                  (const void **)&spec,
267                                  (const void **)&mask,
268                                  supp_mask_p, def_mask_p,
269                                  sizeof(struct rte_flow_item_eth),
270                                  error);
271         if (rc != 0)
272                 return rc;
273
274         /* If "spec" is not set, could be any Ethernet */
275         if (spec == NULL)
276                 return 0;
277
278         if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) {
279                 efx_spec->efs_match_flags |= is_ifrm ?
280                         EFX_FILTER_MATCH_IFRM_LOC_MAC :
281                         EFX_FILTER_MATCH_LOC_MAC;
282                 rte_memcpy(loc_mac, spec->dst.addr_bytes,
283                            EFX_MAC_ADDR_LEN);
284         } else if (memcmp(mask->dst.addr_bytes, ig_mask,
285                           EFX_MAC_ADDR_LEN) == 0) {
286                 if (is_unicast_ether_addr(&spec->dst))
287                         efx_spec->efs_match_flags |= is_ifrm ?
288                                 EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST :
289                                 EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
290                 else
291                         efx_spec->efs_match_flags |= is_ifrm ?
292                                 EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST :
293                                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
294         } else if (!is_zero_ether_addr(&mask->dst)) {
295                 goto fail_bad_mask;
296         }
297
298         /*
299          * ifrm_supp_mask ensures that the source address and
300          * ethertype masks are equal to zero in inner frame,
301          * so these fields are filled in only for the outer frame
302          */
303         if (is_same_ether_addr(&mask->src, &supp_mask.src)) {
304                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC;
305                 rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes,
306                            EFX_MAC_ADDR_LEN);
307         } else if (!is_zero_ether_addr(&mask->src)) {
308                 goto fail_bad_mask;
309         }
310
311         /*
312          * Ether type is in big-endian byte order in item and
313          * in little-endian in efx_spec, so byte swap is used
314          */
315         if (mask->type == supp_mask.type) {
316                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
317                 efx_spec->efs_ether_type = rte_bswap16(spec->type);
318         } else if (mask->type != 0) {
319                 goto fail_bad_mask;
320         }
321
322         return 0;
323
324 fail_bad_mask:
325         rte_flow_error_set(error, EINVAL,
326                            RTE_FLOW_ERROR_TYPE_ITEM, item,
327                            "Bad mask in the ETH pattern item");
328         return -rte_errno;
329 }
330
331 /**
332  * Convert VLAN item to EFX filter specification.
333  *
334  * @param item[in]
335  *   Item specification. Only VID field is supported.
336  *   The mask can not be NULL. Ranging is not supported.
337  * @param efx_spec[in, out]
338  *   EFX filter specification to update.
339  * @param[out] error
340  *   Perform verbose error reporting if not NULL.
341  */
342 static int
343 sfc_flow_parse_vlan(const struct rte_flow_item *item,
344                     efx_filter_spec_t *efx_spec,
345                     struct rte_flow_error *error)
346 {
347         int rc;
348         uint16_t vid;
349         const struct rte_flow_item_vlan *spec = NULL;
350         const struct rte_flow_item_vlan *mask = NULL;
351         const struct rte_flow_item_vlan supp_mask = {
352                 .tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX),
353                 .inner_type = RTE_BE16(0xffff),
354         };
355
356         rc = sfc_flow_parse_init(item,
357                                  (const void **)&spec,
358                                  (const void **)&mask,
359                                  &supp_mask,
360                                  NULL,
361                                  sizeof(struct rte_flow_item_vlan),
362                                  error);
363         if (rc != 0)
364                 return rc;
365
366         /*
367          * VID is in big-endian byte order in item and
368          * in little-endian in efx_spec, so byte swap is used.
369          * If two VLAN items are included, the first matches
370          * the outer tag and the next matches the inner tag.
371          */
372         if (mask->tci == supp_mask.tci) {
373                 vid = rte_bswap16(spec->tci);
374
375                 if (!(efx_spec->efs_match_flags &
376                       EFX_FILTER_MATCH_OUTER_VID)) {
377                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
378                         efx_spec->efs_outer_vid = vid;
379                 } else if (!(efx_spec->efs_match_flags &
380                              EFX_FILTER_MATCH_INNER_VID)) {
381                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID;
382                         efx_spec->efs_inner_vid = vid;
383                 } else {
384                         rte_flow_error_set(error, EINVAL,
385                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
386                                            "More than two VLAN items");
387                         return -rte_errno;
388                 }
389         } else {
390                 rte_flow_error_set(error, EINVAL,
391                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
392                                    "VLAN ID in TCI match is required");
393                 return -rte_errno;
394         }
395
396         if (efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE) {
397                 rte_flow_error_set(error, EINVAL,
398                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
399                                    "VLAN TPID matching is not supported");
400                 return -rte_errno;
401         }
402         if (mask->inner_type == supp_mask.inner_type) {
403                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
404                 efx_spec->efs_ether_type = rte_bswap16(spec->inner_type);
405         } else if (mask->inner_type) {
406                 rte_flow_error_set(error, EINVAL,
407                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
408                                    "Bad mask for VLAN inner_type");
409                 return -rte_errno;
410         }
411
412         return 0;
413 }
414
415 /**
416  * Convert IPv4 item to EFX filter specification.
417  *
418  * @param item[in]
419  *   Item specification. Only source and destination addresses and
420  *   protocol fields are supported. If the mask is NULL, default
421  *   mask will be used. Ranging is not supported.
422  * @param efx_spec[in, out]
423  *   EFX filter specification to update.
424  * @param[out] error
425  *   Perform verbose error reporting if not NULL.
426  */
427 static int
428 sfc_flow_parse_ipv4(const struct rte_flow_item *item,
429                     efx_filter_spec_t *efx_spec,
430                     struct rte_flow_error *error)
431 {
432         int rc;
433         const struct rte_flow_item_ipv4 *spec = NULL;
434         const struct rte_flow_item_ipv4 *mask = NULL;
435         const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4);
436         const struct rte_flow_item_ipv4 supp_mask = {
437                 .hdr = {
438                         .src_addr = 0xffffffff,
439                         .dst_addr = 0xffffffff,
440                         .next_proto_id = 0xff,
441                 }
442         };
443
444         rc = sfc_flow_parse_init(item,
445                                  (const void **)&spec,
446                                  (const void **)&mask,
447                                  &supp_mask,
448                                  &rte_flow_item_ipv4_mask,
449                                  sizeof(struct rte_flow_item_ipv4),
450                                  error);
451         if (rc != 0)
452                 return rc;
453
454         /*
455          * Filtering by IPv4 source and destination addresses requires
456          * the appropriate ETHER_TYPE in hardware filters
457          */
458         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
459                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
460                 efx_spec->efs_ether_type = ether_type_ipv4;
461         } else if (efx_spec->efs_ether_type != ether_type_ipv4) {
462                 rte_flow_error_set(error, EINVAL,
463                         RTE_FLOW_ERROR_TYPE_ITEM, item,
464                         "Ethertype in pattern with IPV4 item should be appropriate");
465                 return -rte_errno;
466         }
467
468         if (spec == NULL)
469                 return 0;
470
471         /*
472          * IPv4 addresses are in big-endian byte order in item and in
473          * efx_spec
474          */
475         if (mask->hdr.src_addr == supp_mask.hdr.src_addr) {
476                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
477                 efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr;
478         } else if (mask->hdr.src_addr != 0) {
479                 goto fail_bad_mask;
480         }
481
482         if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) {
483                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
484                 efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr;
485         } else if (mask->hdr.dst_addr != 0) {
486                 goto fail_bad_mask;
487         }
488
489         if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) {
490                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
491                 efx_spec->efs_ip_proto = spec->hdr.next_proto_id;
492         } else if (mask->hdr.next_proto_id != 0) {
493                 goto fail_bad_mask;
494         }
495
496         return 0;
497
498 fail_bad_mask:
499         rte_flow_error_set(error, EINVAL,
500                            RTE_FLOW_ERROR_TYPE_ITEM, item,
501                            "Bad mask in the IPV4 pattern item");
502         return -rte_errno;
503 }
504
505 /**
506  * Convert IPv6 item to EFX filter specification.
507  *
508  * @param item[in]
509  *   Item specification. Only source and destination addresses and
510  *   next header fields are supported. If the mask is NULL, default
511  *   mask will be used. Ranging is not supported.
512  * @param efx_spec[in, out]
513  *   EFX filter specification to update.
514  * @param[out] error
515  *   Perform verbose error reporting if not NULL.
516  */
517 static int
518 sfc_flow_parse_ipv6(const struct rte_flow_item *item,
519                     efx_filter_spec_t *efx_spec,
520                     struct rte_flow_error *error)
521 {
522         int rc;
523         const struct rte_flow_item_ipv6 *spec = NULL;
524         const struct rte_flow_item_ipv6 *mask = NULL;
525         const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6);
526         const struct rte_flow_item_ipv6 supp_mask = {
527                 .hdr = {
528                         .src_addr = { 0xff, 0xff, 0xff, 0xff,
529                                       0xff, 0xff, 0xff, 0xff,
530                                       0xff, 0xff, 0xff, 0xff,
531                                       0xff, 0xff, 0xff, 0xff },
532                         .dst_addr = { 0xff, 0xff, 0xff, 0xff,
533                                       0xff, 0xff, 0xff, 0xff,
534                                       0xff, 0xff, 0xff, 0xff,
535                                       0xff, 0xff, 0xff, 0xff },
536                         .proto = 0xff,
537                 }
538         };
539
540         rc = sfc_flow_parse_init(item,
541                                  (const void **)&spec,
542                                  (const void **)&mask,
543                                  &supp_mask,
544                                  &rte_flow_item_ipv6_mask,
545                                  sizeof(struct rte_flow_item_ipv6),
546                                  error);
547         if (rc != 0)
548                 return rc;
549
550         /*
551          * Filtering by IPv6 source and destination addresses requires
552          * the appropriate ETHER_TYPE in hardware filters
553          */
554         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
555                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
556                 efx_spec->efs_ether_type = ether_type_ipv6;
557         } else if (efx_spec->efs_ether_type != ether_type_ipv6) {
558                 rte_flow_error_set(error, EINVAL,
559                         RTE_FLOW_ERROR_TYPE_ITEM, item,
560                         "Ethertype in pattern with IPV6 item should be appropriate");
561                 return -rte_errno;
562         }
563
564         if (spec == NULL)
565                 return 0;
566
567         /*
568          * IPv6 addresses are in big-endian byte order in item and in
569          * efx_spec
570          */
571         if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr,
572                    sizeof(mask->hdr.src_addr)) == 0) {
573                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
574
575                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) !=
576                                  sizeof(spec->hdr.src_addr));
577                 rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr,
578                            sizeof(efx_spec->efs_rem_host));
579         } else if (!sfc_flow_is_zero(mask->hdr.src_addr,
580                                      sizeof(mask->hdr.src_addr))) {
581                 goto fail_bad_mask;
582         }
583
584         if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr,
585                    sizeof(mask->hdr.dst_addr)) == 0) {
586                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
587
588                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) !=
589                                  sizeof(spec->hdr.dst_addr));
590                 rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr,
591                            sizeof(efx_spec->efs_loc_host));
592         } else if (!sfc_flow_is_zero(mask->hdr.dst_addr,
593                                      sizeof(mask->hdr.dst_addr))) {
594                 goto fail_bad_mask;
595         }
596
597         if (mask->hdr.proto == supp_mask.hdr.proto) {
598                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
599                 efx_spec->efs_ip_proto = spec->hdr.proto;
600         } else if (mask->hdr.proto != 0) {
601                 goto fail_bad_mask;
602         }
603
604         return 0;
605
606 fail_bad_mask:
607         rte_flow_error_set(error, EINVAL,
608                            RTE_FLOW_ERROR_TYPE_ITEM, item,
609                            "Bad mask in the IPV6 pattern item");
610         return -rte_errno;
611 }
612
613 /**
614  * Convert TCP item to EFX filter specification.
615  *
616  * @param item[in]
617  *   Item specification. Only source and destination ports fields
618  *   are supported. If the mask is NULL, default mask will be used.
619  *   Ranging is not supported.
620  * @param efx_spec[in, out]
621  *   EFX filter specification to update.
622  * @param[out] error
623  *   Perform verbose error reporting if not NULL.
624  */
625 static int
626 sfc_flow_parse_tcp(const struct rte_flow_item *item,
627                    efx_filter_spec_t *efx_spec,
628                    struct rte_flow_error *error)
629 {
630         int rc;
631         const struct rte_flow_item_tcp *spec = NULL;
632         const struct rte_flow_item_tcp *mask = NULL;
633         const struct rte_flow_item_tcp supp_mask = {
634                 .hdr = {
635                         .src_port = 0xffff,
636                         .dst_port = 0xffff,
637                 }
638         };
639
640         rc = sfc_flow_parse_init(item,
641                                  (const void **)&spec,
642                                  (const void **)&mask,
643                                  &supp_mask,
644                                  &rte_flow_item_tcp_mask,
645                                  sizeof(struct rte_flow_item_tcp),
646                                  error);
647         if (rc != 0)
648                 return rc;
649
650         /*
651          * Filtering by TCP source and destination ports requires
652          * the appropriate IP_PROTO in hardware filters
653          */
654         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
655                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
656                 efx_spec->efs_ip_proto = EFX_IPPROTO_TCP;
657         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) {
658                 rte_flow_error_set(error, EINVAL,
659                         RTE_FLOW_ERROR_TYPE_ITEM, item,
660                         "IP proto in pattern with TCP item should be appropriate");
661                 return -rte_errno;
662         }
663
664         if (spec == NULL)
665                 return 0;
666
667         /*
668          * Source and destination ports are in big-endian byte order in item and
669          * in little-endian in efx_spec, so byte swap is used
670          */
671         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
672                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
673                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
674         } else if (mask->hdr.src_port != 0) {
675                 goto fail_bad_mask;
676         }
677
678         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
679                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
680                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
681         } else if (mask->hdr.dst_port != 0) {
682                 goto fail_bad_mask;
683         }
684
685         return 0;
686
687 fail_bad_mask:
688         rte_flow_error_set(error, EINVAL,
689                            RTE_FLOW_ERROR_TYPE_ITEM, item,
690                            "Bad mask in the TCP pattern item");
691         return -rte_errno;
692 }
693
694 /**
695  * Convert UDP item to EFX filter specification.
696  *
697  * @param item[in]
698  *   Item specification. Only source and destination ports fields
699  *   are supported. If the mask is NULL, default mask will be used.
700  *   Ranging is not supported.
701  * @param efx_spec[in, out]
702  *   EFX filter specification to update.
703  * @param[out] error
704  *   Perform verbose error reporting if not NULL.
705  */
706 static int
707 sfc_flow_parse_udp(const struct rte_flow_item *item,
708                    efx_filter_spec_t *efx_spec,
709                    struct rte_flow_error *error)
710 {
711         int rc;
712         const struct rte_flow_item_udp *spec = NULL;
713         const struct rte_flow_item_udp *mask = NULL;
714         const struct rte_flow_item_udp supp_mask = {
715                 .hdr = {
716                         .src_port = 0xffff,
717                         .dst_port = 0xffff,
718                 }
719         };
720
721         rc = sfc_flow_parse_init(item,
722                                  (const void **)&spec,
723                                  (const void **)&mask,
724                                  &supp_mask,
725                                  &rte_flow_item_udp_mask,
726                                  sizeof(struct rte_flow_item_udp),
727                                  error);
728         if (rc != 0)
729                 return rc;
730
731         /*
732          * Filtering by UDP source and destination ports requires
733          * the appropriate IP_PROTO in hardware filters
734          */
735         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
736                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
737                 efx_spec->efs_ip_proto = EFX_IPPROTO_UDP;
738         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) {
739                 rte_flow_error_set(error, EINVAL,
740                         RTE_FLOW_ERROR_TYPE_ITEM, item,
741                         "IP proto in pattern with UDP item should be appropriate");
742                 return -rte_errno;
743         }
744
745         if (spec == NULL)
746                 return 0;
747
748         /*
749          * Source and destination ports are in big-endian byte order in item and
750          * in little-endian in efx_spec, so byte swap is used
751          */
752         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
753                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
754                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
755         } else if (mask->hdr.src_port != 0) {
756                 goto fail_bad_mask;
757         }
758
759         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
760                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
761                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
762         } else if (mask->hdr.dst_port != 0) {
763                 goto fail_bad_mask;
764         }
765
766         return 0;
767
768 fail_bad_mask:
769         rte_flow_error_set(error, EINVAL,
770                            RTE_FLOW_ERROR_TYPE_ITEM, item,
771                            "Bad mask in the UDP pattern item");
772         return -rte_errno;
773 }
774
775 /*
776  * Filters for encapsulated packets match based on the EtherType and IP
777  * protocol in the outer frame.
778  */
779 static int
780 sfc_flow_set_match_flags_for_encap_pkts(const struct rte_flow_item *item,
781                                         efx_filter_spec_t *efx_spec,
782                                         uint8_t ip_proto,
783                                         struct rte_flow_error *error)
784 {
785         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
786                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
787                 efx_spec->efs_ip_proto = ip_proto;
788         } else if (efx_spec->efs_ip_proto != ip_proto) {
789                 switch (ip_proto) {
790                 case EFX_IPPROTO_UDP:
791                         rte_flow_error_set(error, EINVAL,
792                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
793                                 "Outer IP header protocol must be UDP "
794                                 "in VxLAN/GENEVE pattern");
795                         return -rte_errno;
796
797                 case EFX_IPPROTO_GRE:
798                         rte_flow_error_set(error, EINVAL,
799                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
800                                 "Outer IP header protocol must be GRE "
801                                 "in NVGRE pattern");
802                         return -rte_errno;
803
804                 default:
805                         rte_flow_error_set(error, EINVAL,
806                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
807                                 "Only VxLAN/GENEVE/NVGRE tunneling patterns "
808                                 "are supported");
809                         return -rte_errno;
810                 }
811         }
812
813         if (efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE &&
814             efx_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4 &&
815             efx_spec->efs_ether_type != EFX_ETHER_TYPE_IPV6) {
816                 rte_flow_error_set(error, EINVAL,
817                         RTE_FLOW_ERROR_TYPE_ITEM, item,
818                         "Outer frame EtherType in pattern with tunneling "
819                         "must be IPv4 or IPv6");
820                 return -rte_errno;
821         }
822
823         return 0;
824 }
825
826 static int
827 sfc_flow_set_efx_spec_vni_or_vsid(efx_filter_spec_t *efx_spec,
828                                   const uint8_t *vni_or_vsid_val,
829                                   const uint8_t *vni_or_vsid_mask,
830                                   const struct rte_flow_item *item,
831                                   struct rte_flow_error *error)
832 {
833         const uint8_t vni_or_vsid_full_mask[EFX_VNI_OR_VSID_LEN] = {
834                 0xff, 0xff, 0xff
835         };
836
837         if (memcmp(vni_or_vsid_mask, vni_or_vsid_full_mask,
838                    EFX_VNI_OR_VSID_LEN) == 0) {
839                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
840                 rte_memcpy(efx_spec->efs_vni_or_vsid, vni_or_vsid_val,
841                            EFX_VNI_OR_VSID_LEN);
842         } else if (!sfc_flow_is_zero(vni_or_vsid_mask, EFX_VNI_OR_VSID_LEN)) {
843                 rte_flow_error_set(error, EINVAL,
844                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
845                                    "Unsupported VNI/VSID mask");
846                 return -rte_errno;
847         }
848
849         return 0;
850 }
851
852 /**
853  * Convert VXLAN item to EFX filter specification.
854  *
855  * @param item[in]
856  *   Item specification. Only VXLAN network identifier field is supported.
857  *   If the mask is NULL, default mask will be used.
858  *   Ranging is not supported.
859  * @param efx_spec[in, out]
860  *   EFX filter specification to update.
861  * @param[out] error
862  *   Perform verbose error reporting if not NULL.
863  */
864 static int
865 sfc_flow_parse_vxlan(const struct rte_flow_item *item,
866                      efx_filter_spec_t *efx_spec,
867                      struct rte_flow_error *error)
868 {
869         int rc;
870         const struct rte_flow_item_vxlan *spec = NULL;
871         const struct rte_flow_item_vxlan *mask = NULL;
872         const struct rte_flow_item_vxlan supp_mask = {
873                 .vni = { 0xff, 0xff, 0xff }
874         };
875
876         rc = sfc_flow_parse_init(item,
877                                  (const void **)&spec,
878                                  (const void **)&mask,
879                                  &supp_mask,
880                                  &rte_flow_item_vxlan_mask,
881                                  sizeof(struct rte_flow_item_vxlan),
882                                  error);
883         if (rc != 0)
884                 return rc;
885
886         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
887                                                      EFX_IPPROTO_UDP, error);
888         if (rc != 0)
889                 return rc;
890
891         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_VXLAN;
892         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
893
894         if (spec == NULL)
895                 return 0;
896
897         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->vni,
898                                                mask->vni, item, error);
899
900         return rc;
901 }
902
903 /**
904  * Convert GENEVE item to EFX filter specification.
905  *
906  * @param item[in]
907  *   Item specification. Only Virtual Network Identifier and protocol type
908  *   fields are supported. But protocol type can be only Ethernet (0x6558).
909  *   If the mask is NULL, default mask will be used.
910  *   Ranging is not supported.
911  * @param efx_spec[in, out]
912  *   EFX filter specification to update.
913  * @param[out] error
914  *   Perform verbose error reporting if not NULL.
915  */
916 static int
917 sfc_flow_parse_geneve(const struct rte_flow_item *item,
918                       efx_filter_spec_t *efx_spec,
919                       struct rte_flow_error *error)
920 {
921         int rc;
922         const struct rte_flow_item_geneve *spec = NULL;
923         const struct rte_flow_item_geneve *mask = NULL;
924         const struct rte_flow_item_geneve supp_mask = {
925                 .protocol = RTE_BE16(0xffff),
926                 .vni = { 0xff, 0xff, 0xff }
927         };
928
929         rc = sfc_flow_parse_init(item,
930                                  (const void **)&spec,
931                                  (const void **)&mask,
932                                  &supp_mask,
933                                  &rte_flow_item_geneve_mask,
934                                  sizeof(struct rte_flow_item_geneve),
935                                  error);
936         if (rc != 0)
937                 return rc;
938
939         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
940                                                      EFX_IPPROTO_UDP, error);
941         if (rc != 0)
942                 return rc;
943
944         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_GENEVE;
945         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
946
947         if (spec == NULL)
948                 return 0;
949
950         if (mask->protocol == supp_mask.protocol) {
951                 if (spec->protocol != rte_cpu_to_be_16(ETHER_TYPE_TEB)) {
952                         rte_flow_error_set(error, EINVAL,
953                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
954                                 "GENEVE encap. protocol must be Ethernet "
955                                 "(0x6558) in the GENEVE pattern item");
956                         return -rte_errno;
957                 }
958         } else if (mask->protocol != 0) {
959                 rte_flow_error_set(error, EINVAL,
960                         RTE_FLOW_ERROR_TYPE_ITEM, item,
961                         "Unsupported mask for GENEVE encap. protocol");
962                 return -rte_errno;
963         }
964
965         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->vni,
966                                                mask->vni, item, error);
967
968         return rc;
969 }
970
971 /**
972  * Convert NVGRE item to EFX filter specification.
973  *
974  * @param item[in]
975  *   Item specification. Only virtual subnet ID field is supported.
976  *   If the mask is NULL, default mask will be used.
977  *   Ranging is not supported.
978  * @param efx_spec[in, out]
979  *   EFX filter specification to update.
980  * @param[out] error
981  *   Perform verbose error reporting if not NULL.
982  */
983 static int
984 sfc_flow_parse_nvgre(const struct rte_flow_item *item,
985                      efx_filter_spec_t *efx_spec,
986                      struct rte_flow_error *error)
987 {
988         int rc;
989         const struct rte_flow_item_nvgre *spec = NULL;
990         const struct rte_flow_item_nvgre *mask = NULL;
991         const struct rte_flow_item_nvgre supp_mask = {
992                 .tni = { 0xff, 0xff, 0xff }
993         };
994
995         rc = sfc_flow_parse_init(item,
996                                  (const void **)&spec,
997                                  (const void **)&mask,
998                                  &supp_mask,
999                                  &rte_flow_item_nvgre_mask,
1000                                  sizeof(struct rte_flow_item_nvgre),
1001                                  error);
1002         if (rc != 0)
1003                 return rc;
1004
1005         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
1006                                                      EFX_IPPROTO_GRE, error);
1007         if (rc != 0)
1008                 return rc;
1009
1010         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_NVGRE;
1011         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
1012
1013         if (spec == NULL)
1014                 return 0;
1015
1016         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->tni,
1017                                                mask->tni, item, error);
1018
1019         return rc;
1020 }
1021
1022 static const struct sfc_flow_item sfc_flow_items[] = {
1023         {
1024                 .type = RTE_FLOW_ITEM_TYPE_VOID,
1025                 .prev_layer = SFC_FLOW_ITEM_ANY_LAYER,
1026                 .layer = SFC_FLOW_ITEM_ANY_LAYER,
1027                 .parse = sfc_flow_parse_void,
1028         },
1029         {
1030                 .type = RTE_FLOW_ITEM_TYPE_ETH,
1031                 .prev_layer = SFC_FLOW_ITEM_START_LAYER,
1032                 .layer = SFC_FLOW_ITEM_L2,
1033                 .parse = sfc_flow_parse_eth,
1034         },
1035         {
1036                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
1037                 .prev_layer = SFC_FLOW_ITEM_L2,
1038                 .layer = SFC_FLOW_ITEM_L2,
1039                 .parse = sfc_flow_parse_vlan,
1040         },
1041         {
1042                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
1043                 .prev_layer = SFC_FLOW_ITEM_L2,
1044                 .layer = SFC_FLOW_ITEM_L3,
1045                 .parse = sfc_flow_parse_ipv4,
1046         },
1047         {
1048                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
1049                 .prev_layer = SFC_FLOW_ITEM_L2,
1050                 .layer = SFC_FLOW_ITEM_L3,
1051                 .parse = sfc_flow_parse_ipv6,
1052         },
1053         {
1054                 .type = RTE_FLOW_ITEM_TYPE_TCP,
1055                 .prev_layer = SFC_FLOW_ITEM_L3,
1056                 .layer = SFC_FLOW_ITEM_L4,
1057                 .parse = sfc_flow_parse_tcp,
1058         },
1059         {
1060                 .type = RTE_FLOW_ITEM_TYPE_UDP,
1061                 .prev_layer = SFC_FLOW_ITEM_L3,
1062                 .layer = SFC_FLOW_ITEM_L4,
1063                 .parse = sfc_flow_parse_udp,
1064         },
1065         {
1066                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
1067                 .prev_layer = SFC_FLOW_ITEM_L4,
1068                 .layer = SFC_FLOW_ITEM_START_LAYER,
1069                 .parse = sfc_flow_parse_vxlan,
1070         },
1071         {
1072                 .type = RTE_FLOW_ITEM_TYPE_GENEVE,
1073                 .prev_layer = SFC_FLOW_ITEM_L4,
1074                 .layer = SFC_FLOW_ITEM_START_LAYER,
1075                 .parse = sfc_flow_parse_geneve,
1076         },
1077         {
1078                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
1079                 .prev_layer = SFC_FLOW_ITEM_L3,
1080                 .layer = SFC_FLOW_ITEM_START_LAYER,
1081                 .parse = sfc_flow_parse_nvgre,
1082         },
1083 };
1084
1085 /*
1086  * Protocol-independent flow API support
1087  */
1088 static int
1089 sfc_flow_parse_attr(const struct rte_flow_attr *attr,
1090                     struct rte_flow *flow,
1091                     struct rte_flow_error *error)
1092 {
1093         if (attr == NULL) {
1094                 rte_flow_error_set(error, EINVAL,
1095                                    RTE_FLOW_ERROR_TYPE_ATTR, NULL,
1096                                    "NULL attribute");
1097                 return -rte_errno;
1098         }
1099         if (attr->group != 0) {
1100                 rte_flow_error_set(error, ENOTSUP,
1101                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1102                                    "Groups are not supported");
1103                 return -rte_errno;
1104         }
1105         if (attr->priority != 0) {
1106                 rte_flow_error_set(error, ENOTSUP,
1107                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
1108                                    "Priorities are not supported");
1109                 return -rte_errno;
1110         }
1111         if (attr->egress != 0) {
1112                 rte_flow_error_set(error, ENOTSUP,
1113                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
1114                                    "Egress is not supported");
1115                 return -rte_errno;
1116         }
1117         if (attr->transfer != 0) {
1118                 rte_flow_error_set(error, ENOTSUP,
1119                                    RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, attr,
1120                                    "Transfer is not supported");
1121                 return -rte_errno;
1122         }
1123         if (attr->ingress == 0) {
1124                 rte_flow_error_set(error, ENOTSUP,
1125                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
1126                                    "Only ingress is supported");
1127                 return -rte_errno;
1128         }
1129
1130         flow->spec.template.efs_flags |= EFX_FILTER_FLAG_RX;
1131         flow->spec.template.efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
1132
1133         return 0;
1134 }
1135
1136 /* Get item from array sfc_flow_items */
1137 static const struct sfc_flow_item *
1138 sfc_flow_get_item(enum rte_flow_item_type type)
1139 {
1140         unsigned int i;
1141
1142         for (i = 0; i < RTE_DIM(sfc_flow_items); i++)
1143                 if (sfc_flow_items[i].type == type)
1144                         return &sfc_flow_items[i];
1145
1146         return NULL;
1147 }
1148
1149 static int
1150 sfc_flow_parse_pattern(const struct rte_flow_item pattern[],
1151                        struct rte_flow *flow,
1152                        struct rte_flow_error *error)
1153 {
1154         int rc;
1155         unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER;
1156         boolean_t is_ifrm = B_FALSE;
1157         const struct sfc_flow_item *item;
1158
1159         if (pattern == NULL) {
1160                 rte_flow_error_set(error, EINVAL,
1161                                    RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
1162                                    "NULL pattern");
1163                 return -rte_errno;
1164         }
1165
1166         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
1167                 item = sfc_flow_get_item(pattern->type);
1168                 if (item == NULL) {
1169                         rte_flow_error_set(error, ENOTSUP,
1170                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
1171                                            "Unsupported pattern item");
1172                         return -rte_errno;
1173                 }
1174
1175                 /*
1176                  * Omitting one or several protocol layers at the beginning
1177                  * of pattern is supported
1178                  */
1179                 if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
1180                     prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
1181                     item->prev_layer != prev_layer) {
1182                         rte_flow_error_set(error, ENOTSUP,
1183                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
1184                                            "Unexpected sequence of pattern items");
1185                         return -rte_errno;
1186                 }
1187
1188                 /*
1189                  * Allow only VOID and ETH pattern items in the inner frame.
1190                  * Also check that there is only one tunneling protocol.
1191                  */
1192                 switch (item->type) {
1193                 case RTE_FLOW_ITEM_TYPE_VOID:
1194                 case RTE_FLOW_ITEM_TYPE_ETH:
1195                         break;
1196
1197                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1198                 case RTE_FLOW_ITEM_TYPE_GENEVE:
1199                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1200                         if (is_ifrm) {
1201                                 rte_flow_error_set(error, EINVAL,
1202                                         RTE_FLOW_ERROR_TYPE_ITEM,
1203                                         pattern,
1204                                         "More than one tunneling protocol");
1205                                 return -rte_errno;
1206                         }
1207                         is_ifrm = B_TRUE;
1208                         break;
1209
1210                 default:
1211                         if (is_ifrm) {
1212                                 rte_flow_error_set(error, EINVAL,
1213                                         RTE_FLOW_ERROR_TYPE_ITEM,
1214                                         pattern,
1215                                         "There is an unsupported pattern item "
1216                                         "in the inner frame");
1217                                 return -rte_errno;
1218                         }
1219                         break;
1220                 }
1221
1222                 rc = item->parse(pattern, &flow->spec.template, error);
1223                 if (rc != 0)
1224                         return rc;
1225
1226                 if (item->layer != SFC_FLOW_ITEM_ANY_LAYER)
1227                         prev_layer = item->layer;
1228         }
1229
1230         return 0;
1231 }
1232
1233 static int
1234 sfc_flow_parse_queue(struct sfc_adapter *sa,
1235                      const struct rte_flow_action_queue *queue,
1236                      struct rte_flow *flow)
1237 {
1238         struct sfc_rxq *rxq;
1239
1240         if (queue->index >= sa->rxq_count)
1241                 return -EINVAL;
1242
1243         rxq = sa->rxq_info[queue->index].rxq;
1244         flow->spec.template.efs_dmaq_id = (uint16_t)rxq->hw_index;
1245
1246         return 0;
1247 }
1248
1249 static int
1250 sfc_flow_parse_rss(struct sfc_adapter *sa,
1251                    const struct rte_flow_action_rss *action_rss,
1252                    struct rte_flow *flow)
1253 {
1254         struct sfc_rss *rss = &sa->rss;
1255         unsigned int rxq_sw_index;
1256         struct sfc_rxq *rxq;
1257         unsigned int rxq_hw_index_min;
1258         unsigned int rxq_hw_index_max;
1259         efx_rx_hash_type_t efx_hash_types;
1260         const uint8_t *rss_key;
1261         struct sfc_flow_rss *sfc_rss_conf = &flow->rss_conf;
1262         unsigned int i;
1263
1264         if (action_rss->queue_num == 0)
1265                 return -EINVAL;
1266
1267         rxq_sw_index = sa->rxq_count - 1;
1268         rxq = sa->rxq_info[rxq_sw_index].rxq;
1269         rxq_hw_index_min = rxq->hw_index;
1270         rxq_hw_index_max = 0;
1271
1272         for (i = 0; i < action_rss->queue_num; ++i) {
1273                 rxq_sw_index = action_rss->queue[i];
1274
1275                 if (rxq_sw_index >= sa->rxq_count)
1276                         return -EINVAL;
1277
1278                 rxq = sa->rxq_info[rxq_sw_index].rxq;
1279
1280                 if (rxq->hw_index < rxq_hw_index_min)
1281                         rxq_hw_index_min = rxq->hw_index;
1282
1283                 if (rxq->hw_index > rxq_hw_index_max)
1284                         rxq_hw_index_max = rxq->hw_index;
1285         }
1286
1287         switch (action_rss->func) {
1288         case RTE_ETH_HASH_FUNCTION_DEFAULT:
1289         case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
1290                 break;
1291         default:
1292                 return -EINVAL;
1293         }
1294
1295         if (action_rss->level)
1296                 return -EINVAL;
1297
1298         /*
1299          * Dummy RSS action with only one queue and no specific settings
1300          * for hash types and key does not require dedicated RSS context
1301          * and may be simplified to single queue action.
1302          */
1303         if (action_rss->queue_num == 1 && action_rss->types == 0 &&
1304             action_rss->key_len == 0) {
1305                 flow->spec.template.efs_dmaq_id = rxq_hw_index_min;
1306                 return 0;
1307         }
1308
1309         if (action_rss->types) {
1310                 int rc;
1311
1312                 rc = sfc_rx_hf_rte_to_efx(sa, action_rss->types,
1313                                           &efx_hash_types);
1314                 if (rc != 0)
1315                         return -rc;
1316         } else {
1317                 unsigned int i;
1318
1319                 efx_hash_types = 0;
1320                 for (i = 0; i < rss->hf_map_nb_entries; ++i)
1321                         efx_hash_types |= rss->hf_map[i].efx;
1322         }
1323
1324         if (action_rss->key_len) {
1325                 if (action_rss->key_len != sizeof(rss->key))
1326                         return -EINVAL;
1327
1328                 rss_key = action_rss->key;
1329         } else {
1330                 rss_key = rss->key;
1331         }
1332
1333         flow->rss = B_TRUE;
1334
1335         sfc_rss_conf->rxq_hw_index_min = rxq_hw_index_min;
1336         sfc_rss_conf->rxq_hw_index_max = rxq_hw_index_max;
1337         sfc_rss_conf->rss_hash_types = efx_hash_types;
1338         rte_memcpy(sfc_rss_conf->rss_key, rss_key, sizeof(rss->key));
1339
1340         for (i = 0; i < RTE_DIM(sfc_rss_conf->rss_tbl); ++i) {
1341                 unsigned int nb_queues = action_rss->queue_num;
1342                 unsigned int rxq_sw_index = action_rss->queue[i % nb_queues];
1343                 struct sfc_rxq *rxq = sa->rxq_info[rxq_sw_index].rxq;
1344
1345                 sfc_rss_conf->rss_tbl[i] = rxq->hw_index - rxq_hw_index_min;
1346         }
1347
1348         return 0;
1349 }
1350
1351 static int
1352 sfc_flow_spec_flush(struct sfc_adapter *sa, struct sfc_flow_spec *spec,
1353                     unsigned int filters_count)
1354 {
1355         unsigned int i;
1356         int ret = 0;
1357
1358         for (i = 0; i < filters_count; i++) {
1359                 int rc;
1360
1361                 rc = efx_filter_remove(sa->nic, &spec->filters[i]);
1362                 if (ret == 0 && rc != 0) {
1363                         sfc_err(sa, "failed to remove filter specification "
1364                                 "(rc = %d)", rc);
1365                         ret = rc;
1366                 }
1367         }
1368
1369         return ret;
1370 }
1371
1372 static int
1373 sfc_flow_spec_insert(struct sfc_adapter *sa, struct sfc_flow_spec *spec)
1374 {
1375         unsigned int i;
1376         int rc = 0;
1377
1378         for (i = 0; i < spec->count; i++) {
1379                 rc = efx_filter_insert(sa->nic, &spec->filters[i]);
1380                 if (rc != 0) {
1381                         sfc_flow_spec_flush(sa, spec, i);
1382                         break;
1383                 }
1384         }
1385
1386         return rc;
1387 }
1388
1389 static int
1390 sfc_flow_spec_remove(struct sfc_adapter *sa, struct sfc_flow_spec *spec)
1391 {
1392         return sfc_flow_spec_flush(sa, spec, spec->count);
1393 }
1394
1395 static int
1396 sfc_flow_filter_insert(struct sfc_adapter *sa,
1397                        struct rte_flow *flow)
1398 {
1399         struct sfc_rss *rss = &sa->rss;
1400         struct sfc_flow_rss *flow_rss = &flow->rss_conf;
1401         uint32_t efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
1402         unsigned int i;
1403         int rc = 0;
1404
1405         if (flow->rss) {
1406                 unsigned int rss_spread = MIN(flow_rss->rxq_hw_index_max -
1407                                               flow_rss->rxq_hw_index_min + 1,
1408                                               EFX_MAXRSS);
1409
1410                 rc = efx_rx_scale_context_alloc(sa->nic,
1411                                                 EFX_RX_SCALE_EXCLUSIVE,
1412                                                 rss_spread,
1413                                                 &efs_rss_context);
1414                 if (rc != 0)
1415                         goto fail_scale_context_alloc;
1416
1417                 rc = efx_rx_scale_mode_set(sa->nic, efs_rss_context,
1418                                            rss->hash_alg,
1419                                            flow_rss->rss_hash_types, B_TRUE);
1420                 if (rc != 0)
1421                         goto fail_scale_mode_set;
1422
1423                 rc = efx_rx_scale_key_set(sa->nic, efs_rss_context,
1424                                           flow_rss->rss_key,
1425                                           sizeof(rss->key));
1426                 if (rc != 0)
1427                         goto fail_scale_key_set;
1428
1429                 /*
1430                  * At this point, fully elaborated filter specifications
1431                  * have been produced from the template. To make sure that
1432                  * RSS behaviour is consistent between them, set the same
1433                  * RSS context value everywhere.
1434                  */
1435                 for (i = 0; i < flow->spec.count; i++) {
1436                         efx_filter_spec_t *spec = &flow->spec.filters[i];
1437
1438                         spec->efs_rss_context = efs_rss_context;
1439                         spec->efs_dmaq_id = flow_rss->rxq_hw_index_min;
1440                         spec->efs_flags |= EFX_FILTER_FLAG_RX_RSS;
1441                 }
1442         }
1443
1444         rc = sfc_flow_spec_insert(sa, &flow->spec);
1445         if (rc != 0)
1446                 goto fail_filter_insert;
1447
1448         if (flow->rss) {
1449                 /*
1450                  * Scale table is set after filter insertion because
1451                  * the table entries are relative to the base RxQ ID
1452                  * and the latter is submitted to the HW by means of
1453                  * inserting a filter, so by the time of the request
1454                  * the HW knows all the information needed to verify
1455                  * the table entries, and the operation will succeed
1456                  */
1457                 rc = efx_rx_scale_tbl_set(sa->nic, efs_rss_context,
1458                                           flow_rss->rss_tbl,
1459                                           RTE_DIM(flow_rss->rss_tbl));
1460                 if (rc != 0)
1461                         goto fail_scale_tbl_set;
1462         }
1463
1464         return 0;
1465
1466 fail_scale_tbl_set:
1467         sfc_flow_spec_remove(sa, &flow->spec);
1468
1469 fail_filter_insert:
1470 fail_scale_key_set:
1471 fail_scale_mode_set:
1472         if (efs_rss_context != EFX_RSS_CONTEXT_DEFAULT)
1473                 efx_rx_scale_context_free(sa->nic, efs_rss_context);
1474
1475 fail_scale_context_alloc:
1476         return rc;
1477 }
1478
1479 static int
1480 sfc_flow_filter_remove(struct sfc_adapter *sa,
1481                        struct rte_flow *flow)
1482 {
1483         int rc = 0;
1484
1485         rc = sfc_flow_spec_remove(sa, &flow->spec);
1486         if (rc != 0)
1487                 return rc;
1488
1489         if (flow->rss) {
1490                 /*
1491                  * All specifications for a given flow rule have the same RSS
1492                  * context, so that RSS context value is taken from the first
1493                  * filter specification
1494                  */
1495                 efx_filter_spec_t *spec = &flow->spec.filters[0];
1496
1497                 rc = efx_rx_scale_context_free(sa->nic, spec->efs_rss_context);
1498         }
1499
1500         return rc;
1501 }
1502
1503 static int
1504 sfc_flow_parse_actions(struct sfc_adapter *sa,
1505                        const struct rte_flow_action actions[],
1506                        struct rte_flow *flow,
1507                        struct rte_flow_error *error)
1508 {
1509         int rc;
1510         boolean_t is_specified = B_FALSE;
1511
1512         if (actions == NULL) {
1513                 rte_flow_error_set(error, EINVAL,
1514                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
1515                                    "NULL actions");
1516                 return -rte_errno;
1517         }
1518
1519         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1520                 /* This one may appear anywhere multiple times. */
1521                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID)
1522                         continue;
1523                 /* Fate-deciding actions may appear exactly once. */
1524                 if (is_specified) {
1525                         rte_flow_error_set
1526                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1527                                  actions,
1528                                  "Cannot combine several fate-deciding actions,"
1529                                  "choose between QUEUE, RSS or DROP");
1530                         return -rte_errno;
1531                 }
1532                 switch (actions->type) {
1533                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1534                         rc = sfc_flow_parse_queue(sa, actions->conf, flow);
1535                         if (rc != 0) {
1536                                 rte_flow_error_set(error, EINVAL,
1537                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1538                                         "Bad QUEUE action");
1539                                 return -rte_errno;
1540                         }
1541
1542                         is_specified = B_TRUE;
1543                         break;
1544
1545                 case RTE_FLOW_ACTION_TYPE_RSS:
1546                         rc = sfc_flow_parse_rss(sa, actions->conf, flow);
1547                         if (rc != 0) {
1548                                 rte_flow_error_set(error, rc,
1549                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1550                                         "Bad RSS action");
1551                                 return -rte_errno;
1552                         }
1553
1554                         is_specified = B_TRUE;
1555                         break;
1556
1557                 case RTE_FLOW_ACTION_TYPE_DROP:
1558                         flow->spec.template.efs_dmaq_id =
1559                                 EFX_FILTER_SPEC_RX_DMAQ_ID_DROP;
1560
1561                         is_specified = B_TRUE;
1562                         break;
1563
1564                 default:
1565                         rte_flow_error_set(error, ENOTSUP,
1566                                            RTE_FLOW_ERROR_TYPE_ACTION, actions,
1567                                            "Action is not supported");
1568                         return -rte_errno;
1569                 }
1570         }
1571
1572         /* When fate is unknown, drop traffic. */
1573         if (!is_specified) {
1574                 flow->spec.template.efs_dmaq_id =
1575                         EFX_FILTER_SPEC_RX_DMAQ_ID_DROP;
1576         }
1577
1578         return 0;
1579 }
1580
1581 /**
1582  * Set the EFX_FILTER_MATCH_UNKNOWN_UCAST_DST
1583  * and EFX_FILTER_MATCH_UNKNOWN_MCAST_DST match flags in the same
1584  * specifications after copying.
1585  *
1586  * @param spec[in, out]
1587  *   SFC flow specification to update.
1588  * @param filters_count_for_one_val[in]
1589  *   How many specifications should have the same match flag, what is the
1590  *   number of specifications before copying.
1591  * @param error[out]
1592  *   Perform verbose error reporting if not NULL.
1593  */
1594 static int
1595 sfc_flow_set_unknown_dst_flags(struct sfc_flow_spec *spec,
1596                                unsigned int filters_count_for_one_val,
1597                                struct rte_flow_error *error)
1598 {
1599         unsigned int i;
1600         static const efx_filter_match_flags_t vals[] = {
1601                 EFX_FILTER_MATCH_UNKNOWN_UCAST_DST,
1602                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST
1603         };
1604
1605         if (filters_count_for_one_val * RTE_DIM(vals) != spec->count) {
1606                 rte_flow_error_set(error, EINVAL,
1607                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1608                         "Number of specifications is incorrect while copying "
1609                         "by unknown destination flags");
1610                 return -rte_errno;
1611         }
1612
1613         for (i = 0; i < spec->count; i++) {
1614                 /* The check above ensures that divisor can't be zero here */
1615                 spec->filters[i].efs_match_flags |=
1616                         vals[i / filters_count_for_one_val];
1617         }
1618
1619         return 0;
1620 }
1621
1622 /**
1623  * Check that the following conditions are met:
1624  * - the list of supported filters has a filter
1625  *   with EFX_FILTER_MATCH_UNKNOWN_MCAST_DST flag instead of
1626  *   EFX_FILTER_MATCH_UNKNOWN_UCAST_DST, since this filter will also
1627  *   be inserted.
1628  *
1629  * @param match[in]
1630  *   The match flags of filter.
1631  * @param spec[in]
1632  *   Specification to be supplemented.
1633  * @param filter[in]
1634  *   SFC filter with list of supported filters.
1635  */
1636 static boolean_t
1637 sfc_flow_check_unknown_dst_flags(efx_filter_match_flags_t match,
1638                                  __rte_unused efx_filter_spec_t *spec,
1639                                  struct sfc_filter *filter)
1640 {
1641         unsigned int i;
1642         efx_filter_match_flags_t match_mcast_dst;
1643
1644         match_mcast_dst =
1645                 (match & ~EFX_FILTER_MATCH_UNKNOWN_UCAST_DST) |
1646                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
1647         for (i = 0; i < filter->supported_match_num; i++) {
1648                 if (match_mcast_dst == filter->supported_match[i])
1649                         return B_TRUE;
1650         }
1651
1652         return B_FALSE;
1653 }
1654
1655 /**
1656  * Set the EFX_FILTER_MATCH_ETHER_TYPE match flag and EFX_ETHER_TYPE_IPV4 and
1657  * EFX_ETHER_TYPE_IPV6 values of the corresponding field in the same
1658  * specifications after copying.
1659  *
1660  * @param spec[in, out]
1661  *   SFC flow specification to update.
1662  * @param filters_count_for_one_val[in]
1663  *   How many specifications should have the same EtherType value, what is the
1664  *   number of specifications before copying.
1665  * @param error[out]
1666  *   Perform verbose error reporting if not NULL.
1667  */
1668 static int
1669 sfc_flow_set_ethertypes(struct sfc_flow_spec *spec,
1670                         unsigned int filters_count_for_one_val,
1671                         struct rte_flow_error *error)
1672 {
1673         unsigned int i;
1674         static const uint16_t vals[] = {
1675                 EFX_ETHER_TYPE_IPV4, EFX_ETHER_TYPE_IPV6
1676         };
1677
1678         if (filters_count_for_one_val * RTE_DIM(vals) != spec->count) {
1679                 rte_flow_error_set(error, EINVAL,
1680                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1681                         "Number of specifications is incorrect "
1682                         "while copying by Ethertype");
1683                 return -rte_errno;
1684         }
1685
1686         for (i = 0; i < spec->count; i++) {
1687                 spec->filters[i].efs_match_flags |=
1688                         EFX_FILTER_MATCH_ETHER_TYPE;
1689
1690                 /*
1691                  * The check above ensures that
1692                  * filters_count_for_one_val is not 0
1693                  */
1694                 spec->filters[i].efs_ether_type =
1695                         vals[i / filters_count_for_one_val];
1696         }
1697
1698         return 0;
1699 }
1700
1701 /**
1702  * Set the EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST and
1703  * EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST match flags in the same
1704  * specifications after copying.
1705  *
1706  * @param spec[in, out]
1707  *   SFC flow specification to update.
1708  * @param filters_count_for_one_val[in]
1709  *   How many specifications should have the same match flag, what is the
1710  *   number of specifications before copying.
1711  * @param error[out]
1712  *   Perform verbose error reporting if not NULL.
1713  */
1714 static int
1715 sfc_flow_set_ifrm_unknown_dst_flags(struct sfc_flow_spec *spec,
1716                                     unsigned int filters_count_for_one_val,
1717                                     struct rte_flow_error *error)
1718 {
1719         unsigned int i;
1720         static const efx_filter_match_flags_t vals[] = {
1721                 EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST,
1722                 EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST
1723         };
1724
1725         if (filters_count_for_one_val * RTE_DIM(vals) != spec->count) {
1726                 rte_flow_error_set(error, EINVAL,
1727                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1728                         "Number of specifications is incorrect while copying "
1729                         "by inner frame unknown destination flags");
1730                 return -rte_errno;
1731         }
1732
1733         for (i = 0; i < spec->count; i++) {
1734                 /* The check above ensures that divisor can't be zero here */
1735                 spec->filters[i].efs_match_flags |=
1736                         vals[i / filters_count_for_one_val];
1737         }
1738
1739         return 0;
1740 }
1741
1742 /**
1743  * Check that the following conditions are met:
1744  * - the specification corresponds to a filter for encapsulated traffic
1745  * - the list of supported filters has a filter
1746  *   with EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST flag instead of
1747  *   EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST, since this filter will also
1748  *   be inserted.
1749  *
1750  * @param match[in]
1751  *   The match flags of filter.
1752  * @param spec[in]
1753  *   Specification to be supplemented.
1754  * @param filter[in]
1755  *   SFC filter with list of supported filters.
1756  */
1757 static boolean_t
1758 sfc_flow_check_ifrm_unknown_dst_flags(efx_filter_match_flags_t match,
1759                                       efx_filter_spec_t *spec,
1760                                       struct sfc_filter *filter)
1761 {
1762         unsigned int i;
1763         efx_tunnel_protocol_t encap_type = spec->efs_encap_type;
1764         efx_filter_match_flags_t match_mcast_dst;
1765
1766         if (encap_type == EFX_TUNNEL_PROTOCOL_NONE)
1767                 return B_FALSE;
1768
1769         match_mcast_dst =
1770                 (match & ~EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST) |
1771                 EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
1772         for (i = 0; i < filter->supported_match_num; i++) {
1773                 if (match_mcast_dst == filter->supported_match[i])
1774                         return B_TRUE;
1775         }
1776
1777         return B_FALSE;
1778 }
1779
1780 /*
1781  * Match flags that can be automatically added to filters.
1782  * Selecting the last minimum when searching for the copy flag ensures that the
1783  * EFX_FILTER_MATCH_UNKNOWN_UCAST_DST flag has a higher priority than
1784  * EFX_FILTER_MATCH_ETHER_TYPE. This is because the filter
1785  * EFX_FILTER_MATCH_UNKNOWN_UCAST_DST is at the end of the list of supported
1786  * filters.
1787  */
1788 static const struct sfc_flow_copy_flag sfc_flow_copy_flags[] = {
1789         {
1790                 .flag = EFX_FILTER_MATCH_UNKNOWN_UCAST_DST,
1791                 .vals_count = 2,
1792                 .set_vals = sfc_flow_set_unknown_dst_flags,
1793                 .spec_check = sfc_flow_check_unknown_dst_flags,
1794         },
1795         {
1796                 .flag = EFX_FILTER_MATCH_ETHER_TYPE,
1797                 .vals_count = 2,
1798                 .set_vals = sfc_flow_set_ethertypes,
1799                 .spec_check = NULL,
1800         },
1801         {
1802                 .flag = EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST,
1803                 .vals_count = 2,
1804                 .set_vals = sfc_flow_set_ifrm_unknown_dst_flags,
1805                 .spec_check = sfc_flow_check_ifrm_unknown_dst_flags,
1806         },
1807 };
1808
1809 /* Get item from array sfc_flow_copy_flags */
1810 static const struct sfc_flow_copy_flag *
1811 sfc_flow_get_copy_flag(efx_filter_match_flags_t flag)
1812 {
1813         unsigned int i;
1814
1815         for (i = 0; i < RTE_DIM(sfc_flow_copy_flags); i++) {
1816                 if (sfc_flow_copy_flags[i].flag == flag)
1817                         return &sfc_flow_copy_flags[i];
1818         }
1819
1820         return NULL;
1821 }
1822
1823 /**
1824  * Make copies of the specifications, set match flag and values
1825  * of the field that corresponds to it.
1826  *
1827  * @param spec[in, out]
1828  *   SFC flow specification to update.
1829  * @param flag[in]
1830  *   The match flag to add.
1831  * @param error[out]
1832  *   Perform verbose error reporting if not NULL.
1833  */
1834 static int
1835 sfc_flow_spec_add_match_flag(struct sfc_flow_spec *spec,
1836                              efx_filter_match_flags_t flag,
1837                              struct rte_flow_error *error)
1838 {
1839         unsigned int i;
1840         unsigned int new_filters_count;
1841         unsigned int filters_count_for_one_val;
1842         const struct sfc_flow_copy_flag *copy_flag;
1843         int rc;
1844
1845         copy_flag = sfc_flow_get_copy_flag(flag);
1846         if (copy_flag == NULL) {
1847                 rte_flow_error_set(error, ENOTSUP,
1848                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1849                                    "Unsupported spec field for copying");
1850                 return -rte_errno;
1851         }
1852
1853         new_filters_count = spec->count * copy_flag->vals_count;
1854         if (new_filters_count > SF_FLOW_SPEC_NB_FILTERS_MAX) {
1855                 rte_flow_error_set(error, EINVAL,
1856                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1857                         "Too much EFX specifications in the flow rule");
1858                 return -rte_errno;
1859         }
1860
1861         /* Copy filters specifications */
1862         for (i = spec->count; i < new_filters_count; i++)
1863                 spec->filters[i] = spec->filters[i - spec->count];
1864
1865         filters_count_for_one_val = spec->count;
1866         spec->count = new_filters_count;
1867
1868         rc = copy_flag->set_vals(spec, filters_count_for_one_val, error);
1869         if (rc != 0)
1870                 return rc;
1871
1872         return 0;
1873 }
1874
1875 /**
1876  * Check that the given set of match flags missing in the original filter spec
1877  * could be covered by adding spec copies which specify the corresponding
1878  * flags and packet field values to match.
1879  *
1880  * @param miss_flags[in]
1881  *   Flags that are missing until the supported filter.
1882  * @param spec[in]
1883  *   Specification to be supplemented.
1884  * @param filter[in]
1885  *   SFC filter.
1886  *
1887  * @return
1888  *   Number of specifications after copy or 0, if the flags can not be added.
1889  */
1890 static unsigned int
1891 sfc_flow_check_missing_flags(efx_filter_match_flags_t miss_flags,
1892                              efx_filter_spec_t *spec,
1893                              struct sfc_filter *filter)
1894 {
1895         unsigned int i;
1896         efx_filter_match_flags_t copy_flags = 0;
1897         efx_filter_match_flags_t flag;
1898         efx_filter_match_flags_t match = spec->efs_match_flags | miss_flags;
1899         sfc_flow_spec_check *check;
1900         unsigned int multiplier = 1;
1901
1902         for (i = 0; i < RTE_DIM(sfc_flow_copy_flags); i++) {
1903                 flag = sfc_flow_copy_flags[i].flag;
1904                 check = sfc_flow_copy_flags[i].spec_check;
1905                 if ((flag & miss_flags) == flag) {
1906                         if (check != NULL && (!check(match, spec, filter)))
1907                                 continue;
1908
1909                         copy_flags |= flag;
1910                         multiplier *= sfc_flow_copy_flags[i].vals_count;
1911                 }
1912         }
1913
1914         if (copy_flags == miss_flags)
1915                 return multiplier;
1916
1917         return 0;
1918 }
1919
1920 /**
1921  * Attempt to supplement the specification template to the minimally
1922  * supported set of match flags. To do this, it is necessary to copy
1923  * the specifications, filling them with the values of fields that
1924  * correspond to the missing flags.
1925  * The necessary and sufficient filter is built from the fewest number
1926  * of copies which could be made to cover the minimally required set
1927  * of flags.
1928  *
1929  * @param sa[in]
1930  *   SFC adapter.
1931  * @param spec[in, out]
1932  *   SFC flow specification to update.
1933  * @param error[out]
1934  *   Perform verbose error reporting if not NULL.
1935  */
1936 static int
1937 sfc_flow_spec_filters_complete(struct sfc_adapter *sa,
1938                                struct sfc_flow_spec *spec,
1939                                struct rte_flow_error *error)
1940 {
1941         struct sfc_filter *filter = &sa->filter;
1942         efx_filter_match_flags_t miss_flags;
1943         efx_filter_match_flags_t min_miss_flags = 0;
1944         efx_filter_match_flags_t match;
1945         unsigned int min_multiplier = UINT_MAX;
1946         unsigned int multiplier;
1947         unsigned int i;
1948         int rc;
1949
1950         match = spec->template.efs_match_flags;
1951         for (i = 0; i < filter->supported_match_num; i++) {
1952                 if ((match & filter->supported_match[i]) == match) {
1953                         miss_flags = filter->supported_match[i] & (~match);
1954                         multiplier = sfc_flow_check_missing_flags(miss_flags,
1955                                 &spec->template, filter);
1956                         if (multiplier > 0) {
1957                                 if (multiplier <= min_multiplier) {
1958                                         min_multiplier = multiplier;
1959                                         min_miss_flags = miss_flags;
1960                                 }
1961                         }
1962                 }
1963         }
1964
1965         if (min_multiplier == UINT_MAX) {
1966                 rte_flow_error_set(error, ENOTSUP,
1967                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1968                                    "Flow rule pattern is not supported");
1969                 return -rte_errno;
1970         }
1971
1972         for (i = 0; i < RTE_DIM(sfc_flow_copy_flags); i++) {
1973                 efx_filter_match_flags_t flag = sfc_flow_copy_flags[i].flag;
1974
1975                 if ((flag & min_miss_flags) == flag) {
1976                         rc = sfc_flow_spec_add_match_flag(spec, flag, error);
1977                         if (rc != 0)
1978                                 return rc;
1979                 }
1980         }
1981
1982         return 0;
1983 }
1984
1985 /**
1986  * Check that set of match flags is referred to by a filter. Filter is
1987  * described by match flags with the ability to add OUTER_VID and INNER_VID
1988  * flags.
1989  *
1990  * @param match_flags[in]
1991  *   Set of match flags.
1992  * @param flags_pattern[in]
1993  *   Pattern of filter match flags.
1994  */
1995 static boolean_t
1996 sfc_flow_is_match_with_vids(efx_filter_match_flags_t match_flags,
1997                             efx_filter_match_flags_t flags_pattern)
1998 {
1999         if ((match_flags & flags_pattern) != flags_pattern)
2000                 return B_FALSE;
2001
2002         switch (match_flags & ~flags_pattern) {
2003         case 0:
2004         case EFX_FILTER_MATCH_OUTER_VID:
2005         case EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_INNER_VID:
2006                 return B_TRUE;
2007         default:
2008                 return B_FALSE;
2009         }
2010 }
2011
2012 /**
2013  * Check whether the spec maps to a hardware filter which is known to be
2014  * ineffective despite being valid.
2015  *
2016  * @param spec[in]
2017  *   SFC flow specification.
2018  */
2019 static boolean_t
2020 sfc_flow_is_match_flags_exception(struct sfc_flow_spec *spec)
2021 {
2022         unsigned int i;
2023         uint16_t ether_type;
2024         uint8_t ip_proto;
2025         efx_filter_match_flags_t match_flags;
2026
2027         for (i = 0; i < spec->count; i++) {
2028                 match_flags = spec->filters[i].efs_match_flags;
2029
2030                 if (sfc_flow_is_match_with_vids(match_flags,
2031                                                 EFX_FILTER_MATCH_ETHER_TYPE) ||
2032                     sfc_flow_is_match_with_vids(match_flags,
2033                                                 EFX_FILTER_MATCH_ETHER_TYPE |
2034                                                 EFX_FILTER_MATCH_LOC_MAC)) {
2035                         ether_type = spec->filters[i].efs_ether_type;
2036                         if (ether_type == EFX_ETHER_TYPE_IPV4 ||
2037                             ether_type == EFX_ETHER_TYPE_IPV6)
2038                                 return B_TRUE;
2039                 } else if (sfc_flow_is_match_with_vids(match_flags,
2040                                 EFX_FILTER_MATCH_ETHER_TYPE |
2041                                 EFX_FILTER_MATCH_IP_PROTO) ||
2042                            sfc_flow_is_match_with_vids(match_flags,
2043                                 EFX_FILTER_MATCH_ETHER_TYPE |
2044                                 EFX_FILTER_MATCH_IP_PROTO |
2045                                 EFX_FILTER_MATCH_LOC_MAC)) {
2046                         ip_proto = spec->filters[i].efs_ip_proto;
2047                         if (ip_proto == EFX_IPPROTO_TCP ||
2048                             ip_proto == EFX_IPPROTO_UDP)
2049                                 return B_TRUE;
2050                 }
2051         }
2052
2053         return B_FALSE;
2054 }
2055
2056 static int
2057 sfc_flow_validate_match_flags(struct sfc_adapter *sa,
2058                               struct rte_flow *flow,
2059                               struct rte_flow_error *error)
2060 {
2061         efx_filter_spec_t *spec_tmpl = &flow->spec.template;
2062         efx_filter_match_flags_t match_flags = spec_tmpl->efs_match_flags;
2063         int rc;
2064
2065         /* Initialize the first filter spec with template */
2066         flow->spec.filters[0] = *spec_tmpl;
2067         flow->spec.count = 1;
2068
2069         if (!sfc_filter_is_match_supported(sa, match_flags)) {
2070                 rc = sfc_flow_spec_filters_complete(sa, &flow->spec, error);
2071                 if (rc != 0)
2072                         return rc;
2073         }
2074
2075         if (sfc_flow_is_match_flags_exception(&flow->spec)) {
2076                 rte_flow_error_set(error, ENOTSUP,
2077                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2078                         "The flow rule pattern is unsupported");
2079                 return -rte_errno;
2080         }
2081
2082         return 0;
2083 }
2084
2085 static int
2086 sfc_flow_parse(struct rte_eth_dev *dev,
2087                const struct rte_flow_attr *attr,
2088                const struct rte_flow_item pattern[],
2089                const struct rte_flow_action actions[],
2090                struct rte_flow *flow,
2091                struct rte_flow_error *error)
2092 {
2093         struct sfc_adapter *sa = dev->data->dev_private;
2094         int rc;
2095
2096         rc = sfc_flow_parse_attr(attr, flow, error);
2097         if (rc != 0)
2098                 goto fail_bad_value;
2099
2100         rc = sfc_flow_parse_pattern(pattern, flow, error);
2101         if (rc != 0)
2102                 goto fail_bad_value;
2103
2104         rc = sfc_flow_parse_actions(sa, actions, flow, error);
2105         if (rc != 0)
2106                 goto fail_bad_value;
2107
2108         rc = sfc_flow_validate_match_flags(sa, flow, error);
2109         if (rc != 0)
2110                 goto fail_bad_value;
2111
2112         return 0;
2113
2114 fail_bad_value:
2115         return rc;
2116 }
2117
2118 static int
2119 sfc_flow_validate(struct rte_eth_dev *dev,
2120                   const struct rte_flow_attr *attr,
2121                   const struct rte_flow_item pattern[],
2122                   const struct rte_flow_action actions[],
2123                   struct rte_flow_error *error)
2124 {
2125         struct rte_flow flow;
2126
2127         memset(&flow, 0, sizeof(flow));
2128
2129         return sfc_flow_parse(dev, attr, pattern, actions, &flow, error);
2130 }
2131
2132 static struct rte_flow *
2133 sfc_flow_create(struct rte_eth_dev *dev,
2134                 const struct rte_flow_attr *attr,
2135                 const struct rte_flow_item pattern[],
2136                 const struct rte_flow_action actions[],
2137                 struct rte_flow_error *error)
2138 {
2139         struct sfc_adapter *sa = dev->data->dev_private;
2140         struct rte_flow *flow = NULL;
2141         int rc;
2142
2143         flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0);
2144         if (flow == NULL) {
2145                 rte_flow_error_set(error, ENOMEM,
2146                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2147                                    "Failed to allocate memory");
2148                 goto fail_no_mem;
2149         }
2150
2151         rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error);
2152         if (rc != 0)
2153                 goto fail_bad_value;
2154
2155         TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries);
2156
2157         sfc_adapter_lock(sa);
2158
2159         if (sa->state == SFC_ADAPTER_STARTED) {
2160                 rc = sfc_flow_filter_insert(sa, flow);
2161                 if (rc != 0) {
2162                         rte_flow_error_set(error, rc,
2163                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2164                                 "Failed to insert filter");
2165                         goto fail_filter_insert;
2166                 }
2167         }
2168
2169         sfc_adapter_unlock(sa);
2170
2171         return flow;
2172
2173 fail_filter_insert:
2174         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
2175
2176 fail_bad_value:
2177         rte_free(flow);
2178         sfc_adapter_unlock(sa);
2179
2180 fail_no_mem:
2181         return NULL;
2182 }
2183
2184 static int
2185 sfc_flow_remove(struct sfc_adapter *sa,
2186                 struct rte_flow *flow,
2187                 struct rte_flow_error *error)
2188 {
2189         int rc = 0;
2190
2191         SFC_ASSERT(sfc_adapter_is_locked(sa));
2192
2193         if (sa->state == SFC_ADAPTER_STARTED) {
2194                 rc = sfc_flow_filter_remove(sa, flow);
2195                 if (rc != 0)
2196                         rte_flow_error_set(error, rc,
2197                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2198                                 "Failed to destroy flow rule");
2199         }
2200
2201         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
2202         rte_free(flow);
2203
2204         return rc;
2205 }
2206
2207 static int
2208 sfc_flow_destroy(struct rte_eth_dev *dev,
2209                  struct rte_flow *flow,
2210                  struct rte_flow_error *error)
2211 {
2212         struct sfc_adapter *sa = dev->data->dev_private;
2213         struct rte_flow *flow_ptr;
2214         int rc = EINVAL;
2215
2216         sfc_adapter_lock(sa);
2217
2218         TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) {
2219                 if (flow_ptr == flow)
2220                         rc = 0;
2221         }
2222         if (rc != 0) {
2223                 rte_flow_error_set(error, rc,
2224                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2225                                    "Failed to find flow rule to destroy");
2226                 goto fail_bad_value;
2227         }
2228
2229         rc = sfc_flow_remove(sa, flow, error);
2230
2231 fail_bad_value:
2232         sfc_adapter_unlock(sa);
2233
2234         return -rc;
2235 }
2236
2237 static int
2238 sfc_flow_flush(struct rte_eth_dev *dev,
2239                struct rte_flow_error *error)
2240 {
2241         struct sfc_adapter *sa = dev->data->dev_private;
2242         struct rte_flow *flow;
2243         int rc = 0;
2244         int ret = 0;
2245
2246         sfc_adapter_lock(sa);
2247
2248         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
2249                 rc = sfc_flow_remove(sa, flow, error);
2250                 if (rc != 0)
2251                         ret = rc;
2252         }
2253
2254         sfc_adapter_unlock(sa);
2255
2256         return -ret;
2257 }
2258
2259 static int
2260 sfc_flow_isolate(struct rte_eth_dev *dev, int enable,
2261                  struct rte_flow_error *error)
2262 {
2263         struct sfc_adapter *sa = dev->data->dev_private;
2264         struct sfc_port *port = &sa->port;
2265         int ret = 0;
2266
2267         sfc_adapter_lock(sa);
2268         if (sa->state != SFC_ADAPTER_INITIALIZED) {
2269                 rte_flow_error_set(error, EBUSY,
2270                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2271                                    NULL, "please close the port first");
2272                 ret = -rte_errno;
2273         } else {
2274                 port->isolated = (enable) ? B_TRUE : B_FALSE;
2275         }
2276         sfc_adapter_unlock(sa);
2277
2278         return ret;
2279 }
2280
2281 const struct rte_flow_ops sfc_flow_ops = {
2282         .validate = sfc_flow_validate,
2283         .create = sfc_flow_create,
2284         .destroy = sfc_flow_destroy,
2285         .flush = sfc_flow_flush,
2286         .query = NULL,
2287         .isolate = sfc_flow_isolate,
2288 };
2289
2290 void
2291 sfc_flow_init(struct sfc_adapter *sa)
2292 {
2293         SFC_ASSERT(sfc_adapter_is_locked(sa));
2294
2295         TAILQ_INIT(&sa->filter.flow_list);
2296 }
2297
2298 void
2299 sfc_flow_fini(struct sfc_adapter *sa)
2300 {
2301         struct rte_flow *flow;
2302
2303         SFC_ASSERT(sfc_adapter_is_locked(sa));
2304
2305         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
2306                 TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
2307                 rte_free(flow);
2308         }
2309 }
2310
2311 void
2312 sfc_flow_stop(struct sfc_adapter *sa)
2313 {
2314         struct rte_flow *flow;
2315
2316         SFC_ASSERT(sfc_adapter_is_locked(sa));
2317
2318         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries)
2319                 sfc_flow_filter_remove(sa, flow);
2320 }
2321
2322 int
2323 sfc_flow_start(struct sfc_adapter *sa)
2324 {
2325         struct rte_flow *flow;
2326         int rc = 0;
2327
2328         sfc_log_init(sa, "entry");
2329
2330         SFC_ASSERT(sfc_adapter_is_locked(sa));
2331
2332         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) {
2333                 rc = sfc_flow_filter_insert(sa, flow);
2334                 if (rc != 0)
2335                         goto fail_bad_flow;
2336         }
2337
2338         sfc_log_init(sa, "done");
2339
2340 fail_bad_flow:
2341         return rc;
2342 }