drivers/net/sfc/sfc_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2017-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * This software was jointly developed between OKTET Labs (under contract
   7  * for Solarflare) and Solarflare Communications, Inc.
   8  */
   9
  10 #include <rte_byteorder.h>
  11 #include <rte_tailq.h>
  12 #include <rte_common.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_eth_ctrl.h>
  15 #include <rte_ether.h>
  16 #include <rte_flow.h>
  17 #include <rte_flow_driver.h>
  18
  19 #include "efx.h"
  20
  21 #include "sfc.h"
  22 #include "sfc_rx.h"
  23 #include "sfc_filter.h"
  24 #include "sfc_flow.h"
  25 #include "sfc_log.h"
  26 #include "sfc_dp_rx.h"
  27
  28 /*
  29  * At now flow API is implemented in such a manner that each
  30  * flow rule is converted to one or more hardware filters.
  31  * All elements of flow rule (attributes, pattern items, actions)
  32  * correspond to one or more fields in the efx_filter_spec_s structure
  33  * that is responsible for the hardware filter.
  34  * If some required field is unset in the flow rule, then a handful
  35  * of filter copies will be created to cover all possible values
  36  * of such a field.
  37  */
  38
  39 enum sfc_flow_item_layers {
  40         SFC_FLOW_ITEM_ANY_LAYER,
  41         SFC_FLOW_ITEM_START_LAYER,
  42         SFC_FLOW_ITEM_L2,
  43         SFC_FLOW_ITEM_L3,
  44         SFC_FLOW_ITEM_L4,
  45 };
  46
  47 typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item,
  48                                   efx_filter_spec_t *spec,
  49                                   struct rte_flow_error *error);
  50
  51 struct sfc_flow_item {
  52         enum rte_flow_item_type type;           /* Type of item */
  53         enum sfc_flow_item_layers layer;        /* Layer of item */
  54         enum sfc_flow_item_layers prev_layer;   /* Previous layer of item */
  55         sfc_flow_item_parse *parse;             /* Parsing function */
  56 };
  57
  58 static sfc_flow_item_parse sfc_flow_parse_void;
  59 static sfc_flow_item_parse sfc_flow_parse_eth;
  60 static sfc_flow_item_parse sfc_flow_parse_vlan;
  61 static sfc_flow_item_parse sfc_flow_parse_ipv4;
  62 static sfc_flow_item_parse sfc_flow_parse_ipv6;
  63 static sfc_flow_item_parse sfc_flow_parse_tcp;
  64 static sfc_flow_item_parse sfc_flow_parse_udp;
  65 static sfc_flow_item_parse sfc_flow_parse_vxlan;
  66 static sfc_flow_item_parse sfc_flow_parse_geneve;
  67 static sfc_flow_item_parse sfc_flow_parse_nvgre;
  68
  69 typedef int (sfc_flow_spec_set_vals)(struct sfc_flow_spec *spec,
  70                                      unsigned int filters_count_for_one_val,
  71                                      struct rte_flow_error *error);
  72
  73 typedef boolean_t (sfc_flow_spec_check)(efx_filter_match_flags_t match,
  74                                         efx_filter_spec_t *spec,
  75                                         struct sfc_filter *filter);
  76
  77 struct sfc_flow_copy_flag {
  78         /* EFX filter specification match flag */
  79         efx_filter_match_flags_t flag;
  80         /* Number of values of corresponding field */
  81         unsigned int vals_count;
  82         /* Function to set values in specifications */
  83         sfc_flow_spec_set_vals *set_vals;
  84         /*
  85          * Function to check that the specification is suitable
  86          * for adding this match flag
  87          */
  88         sfc_flow_spec_check *spec_check;
  89 };
  90
  91 static sfc_flow_spec_set_vals sfc_flow_set_unknown_dst_flags;
  92 static sfc_flow_spec_check sfc_flow_check_unknown_dst_flags;
  93 static sfc_flow_spec_set_vals sfc_flow_set_ethertypes;
  94 static sfc_flow_spec_set_vals sfc_flow_set_ifrm_unknown_dst_flags;
  95 static sfc_flow_spec_check sfc_flow_check_ifrm_unknown_dst_flags;
  96
  97 static boolean_t
  98 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
  99 {
 100         uint8_t sum = 0;
 101         unsigned int i;
 102
 103         for (i = 0; i < size; i++)
 104                 sum |= buf[i];
 105
 106         return (sum == 0) ? B_TRUE : B_FALSE;
 107 }
 108
 109 /*
 110  * Validate item and prepare structures spec and mask for parsing
 111  */
 112 static int
 113 sfc_flow_parse_init(const struct rte_flow_item *item,
 114                     const void **spec_ptr,
 115                     const void **mask_ptr,
 116                     const void *supp_mask,
 117                     const void *def_mask,
 118                     unsigned int size,
 119                     struct rte_flow_error *error)
 120 {
 121         const uint8_t *spec;
 122         const uint8_t *mask;
 123         const uint8_t *last;
 124         uint8_t supp;
 125         unsigned int i;
 126
 127         if (item == NULL) {
 128                 rte_flow_error_set(error, EINVAL,
 129                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 130                                    "NULL item");
 131                 return -rte_errno;
 132         }
 133
 134         if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) {
 135                 rte_flow_error_set(error, EINVAL,
 136                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 137                                    "Mask or last is set without spec");
 138                 return -rte_errno;
 139         }
 140
 141         /*
 142          * If "mask" is not set, default mask is used,
 143          * but if default mask is NULL, "mask" should be set
 144          */
 145         if (item->mask == NULL) {
 146                 if (def_mask == NULL) {
 147                         rte_flow_error_set(error, EINVAL,
 148                                 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 149                                 "Mask should be specified");
 150                         return -rte_errno;
 151                 }
 152
 153                 mask = def_mask;
 154         } else {
 155                 mask = item->mask;
 156         }
 157
 158         spec = item->spec;
 159         last = item->last;
 160
 161         if (spec == NULL)
 162                 goto exit;
 163
 164         /*
 165          * If field values in "last" are either 0 or equal to the corresponding
 166          * values in "spec" then they are ignored
 167          */
 168         if (last != NULL &&
 169             !sfc_flow_is_zero(last, size) &&
 170             memcmp(last, spec, size) != 0) {
 171                 rte_flow_error_set(error, ENOTSUP,
 172                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 173                                    "Ranging is not supported");
 174                 return -rte_errno;
 175         }
 176
 177         if (supp_mask == NULL) {
 178                 rte_flow_error_set(error, EINVAL,
 179                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 180                         "Supported mask for item should be specified");
 181                 return -rte_errno;
 182         }
 183
 184         /* Check that mask does not ask for more match than supp_mask */
 185         for (i = 0; i < size; i++) {
 186                 supp = ((const uint8_t *)supp_mask)[i];
 187
 188                 if (~supp & mask[i]) {
 189                         rte_flow_error_set(error, ENOTSUP,
 190                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 191                                            "Item's field is not supported");
 192                         return -rte_errno;
 193                 }
 194         }
 195
 196 exit:
 197         *spec_ptr = spec;
 198         *mask_ptr = mask;
 199         return 0;
 200 }
 201
 202 /*
 203  * Protocol parsers.
 204  * Masking is not supported, so masks in items should be either
 205  * full or empty (zeroed) and set only for supported fields which
 206  * are specified in the supp_mask.
 207  */
 208
 209 static int
 210 sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item,
 211                     __rte_unused efx_filter_spec_t *efx_spec,
 212                     __rte_unused struct rte_flow_error *error)
 213 {
 214         return 0;
 215 }
 216
 217 /**
 218  * Convert Ethernet item to EFX filter specification.
 219  *
 220  * @param item[in]
 221  *   Item specification. Outer frame specification may only comprise
 222  *   source/destination addresses and Ethertype field.
 223  *   Inner frame specification may contain destination address only.
 224  *   There is support for individual/group mask as well as for empty and full.
 225  *   If the mask is NULL, default mask will be used. Ranging is not supported.
 226  * @param efx_spec[in, out]
 227  *   EFX filter specification to update.
 228  * @param[out] error
 229  *   Perform verbose error reporting if not NULL.
 230  */
 231 static int
 232 sfc_flow_parse_eth(const struct rte_flow_item *item,
 233                    efx_filter_spec_t *efx_spec,
 234                    struct rte_flow_error *error)
 235 {
 236         int rc;
 237         const struct rte_flow_item_eth *spec = NULL;
 238         const struct rte_flow_item_eth *mask = NULL;
 239         const struct rte_flow_item_eth supp_mask = {
 240                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 241                 .src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 242                 .type = 0xffff,
 243         };
 244         const struct rte_flow_item_eth ifrm_supp_mask = {
 245                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 246         };
 247         const uint8_t ig_mask[EFX_MAC_ADDR_LEN] = {
 248                 0x01, 0x00, 0x00, 0x00, 0x00, 0x00
 249         };
 250         const struct rte_flow_item_eth *supp_mask_p;
 251         const struct rte_flow_item_eth *def_mask_p;
 252         uint8_t *loc_mac = NULL;
 253         boolean_t is_ifrm = (efx_spec->efs_encap_type !=
 254                 EFX_TUNNEL_PROTOCOL_NONE);
 255
 256         if (is_ifrm) {
 257                 supp_mask_p = &ifrm_supp_mask;
 258                 def_mask_p = &ifrm_supp_mask;
 259                 loc_mac = efx_spec->efs_ifrm_loc_mac;
 260         } else {
 261                 supp_mask_p = &supp_mask;
 262                 def_mask_p = &rte_flow_item_eth_mask;
 263                 loc_mac = efx_spec->efs_loc_mac;
 264         }
 265
 266         rc = sfc_flow_parse_init(item,
 267                                  (const void **)&spec,
 268                                  (const void **)&mask,
 269                                  supp_mask_p, def_mask_p,
 270                                  sizeof(struct rte_flow_item_eth),
 271                                  error);
 272         if (rc != 0)
 273                 return rc;
 274
 275         /* If "spec" is not set, could be any Ethernet */
 276         if (spec == NULL)
 277                 return 0;
 278
 279         if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) {
 280                 efx_spec->efs_match_flags |= is_ifrm ?
 281                         EFX_FILTER_MATCH_IFRM_LOC_MAC :
 282                         EFX_FILTER_MATCH_LOC_MAC;
 283                 rte_memcpy(loc_mac, spec->dst.addr_bytes,
 284                            EFX_MAC_ADDR_LEN);
 285         } else if (memcmp(mask->dst.addr_bytes, ig_mask,
 286                           EFX_MAC_ADDR_LEN) == 0) {
 287                 if (is_unicast_ether_addr(&spec->dst))
 288                         efx_spec->efs_match_flags |= is_ifrm ?
 289                                 EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST :
 290                                 EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 291                 else
 292                         efx_spec->efs_match_flags |= is_ifrm ?
 293                                 EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST :
 294                                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 295         } else if (!is_zero_ether_addr(&mask->dst)) {
 296                 goto fail_bad_mask;
 297         }
 298
 299         /*
 300          * ifrm_supp_mask ensures that the source address and
 301          * ethertype masks are equal to zero in inner frame,
 302          * so these fields are filled in only for the outer frame
 303          */
 304         if (is_same_ether_addr(&mask->src, &supp_mask.src)) {
 305                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC;
 306                 rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes,
 307                            EFX_MAC_ADDR_LEN);
 308         } else if (!is_zero_ether_addr(&mask->src)) {
 309                 goto fail_bad_mask;
 310         }
 311
 312         /*
 313          * Ether type is in big-endian byte order in item and
 314          * in little-endian in efx_spec, so byte swap is used
 315          */
 316         if (mask->type == supp_mask.type) {
 317                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 318                 efx_spec->efs_ether_type = rte_bswap16(spec->type);
 319         } else if (mask->type != 0) {
 320                 goto fail_bad_mask;
 321         }
 322
 323         return 0;
 324
 325 fail_bad_mask:
 326         rte_flow_error_set(error, EINVAL,
 327                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 328                            "Bad mask in the ETH pattern item");
 329         return -rte_errno;
 330 }
 331
 332 /**
 333  * Convert VLAN item to EFX filter specification.
 334  *
 335  * @param item[in]
 336  *   Item specification. Only VID field is supported.
 337  *   The mask can not be NULL. Ranging is not supported.
 338  * @param efx_spec[in, out]
 339  *   EFX filter specification to update.
 340  * @param[out] error
 341  *   Perform verbose error reporting if not NULL.
 342  */
 343 static int
 344 sfc_flow_parse_vlan(const struct rte_flow_item *item,
 345                     efx_filter_spec_t *efx_spec,
 346                     struct rte_flow_error *error)
 347 {
 348         int rc;
 349         uint16_t vid;
 350         const struct rte_flow_item_vlan *spec = NULL;
 351         const struct rte_flow_item_vlan *mask = NULL;
 352         const struct rte_flow_item_vlan supp_mask = {
 353                 .tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX),
 354                 .inner_type = RTE_BE16(0xffff),
 355         };
 356
 357         rc = sfc_flow_parse_init(item,
 358                                  (const void **)&spec,
 359                                  (const void **)&mask,
 360                                  &supp_mask,
 361                                  NULL,
 362                                  sizeof(struct rte_flow_item_vlan),
 363                                  error);
 364         if (rc != 0)
 365                 return rc;
 366
 367         /*
 368          * VID is in big-endian byte order in item and
 369          * in little-endian in efx_spec, so byte swap is used.
 370          * If two VLAN items are included, the first matches
 371          * the outer tag and the next matches the inner tag.
 372          */
 373         if (mask->tci == supp_mask.tci) {
 374                 vid = rte_bswap16(spec->tci);
 375
 376                 if (!(efx_spec->efs_match_flags &
 377                       EFX_FILTER_MATCH_OUTER_VID)) {
 378                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 379                         efx_spec->efs_outer_vid = vid;
 380                 } else if (!(efx_spec->efs_match_flags &
 381                              EFX_FILTER_MATCH_INNER_VID)) {
 382                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID;
 383                         efx_spec->efs_inner_vid = vid;
 384                 } else {
 385                         rte_flow_error_set(error, EINVAL,
 386                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 387                                            "More than two VLAN items");
 388                         return -rte_errno;
 389                 }
 390         } else {
 391                 rte_flow_error_set(error, EINVAL,
 392                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 393                                    "VLAN ID in TCI match is required");
 394                 return -rte_errno;
 395         }
 396
 397         if (efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE) {
 398                 rte_flow_error_set(error, EINVAL,
 399                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 400                                    "VLAN TPID matching is not supported");
 401                 return -rte_errno;
 402         }
 403         if (mask->inner_type == supp_mask.inner_type) {
 404                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 405                 efx_spec->efs_ether_type = rte_bswap16(spec->inner_type);
 406         } else if (mask->inner_type) {
 407                 rte_flow_error_set(error, EINVAL,
 408                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 409                                    "Bad mask for VLAN inner_type");
 410                 return -rte_errno;
 411         }
 412
 413         return 0;
 414 }
 415
 416 /**
 417  * Convert IPv4 item to EFX filter specification.
 418  *
 419  * @param item[in]
 420  *   Item specification. Only source and destination addresses and
 421  *   protocol fields are supported. If the mask is NULL, default
 422  *   mask will be used. Ranging is not supported.
 423  * @param efx_spec[in, out]
 424  *   EFX filter specification to update.
 425  * @param[out] error
 426  *   Perform verbose error reporting if not NULL.
 427  */
 428 static int
 429 sfc_flow_parse_ipv4(const struct rte_flow_item *item,
 430                     efx_filter_spec_t *efx_spec,
 431                     struct rte_flow_error *error)
 432 {
 433         int rc;
 434         const struct rte_flow_item_ipv4 *spec = NULL;
 435         const struct rte_flow_item_ipv4 *mask = NULL;
 436         const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4);
 437         const struct rte_flow_item_ipv4 supp_mask = {
 438                 .hdr = {
 439                         .src_addr = 0xffffffff,
 440                         .dst_addr = 0xffffffff,
 441                         .next_proto_id = 0xff,
 442                 }
 443         };
 444
 445         rc = sfc_flow_parse_init(item,
 446                                  (const void **)&spec,
 447                                  (const void **)&mask,
 448                                  &supp_mask,
 449                                  &rte_flow_item_ipv4_mask,
 450                                  sizeof(struct rte_flow_item_ipv4),
 451                                  error);
 452         if (rc != 0)
 453                 return rc;
 454
 455         /*
 456          * Filtering by IPv4 source and destination addresses requires
 457          * the appropriate ETHER_TYPE in hardware filters
 458          */
 459         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 460                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 461                 efx_spec->efs_ether_type = ether_type_ipv4;
 462         } else if (efx_spec->efs_ether_type != ether_type_ipv4) {
 463                 rte_flow_error_set(error, EINVAL,
 464                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 465                         "Ethertype in pattern with IPV4 item should be appropriate");
 466                 return -rte_errno;
 467         }
 468
 469         if (spec == NULL)
 470                 return 0;
 471
 472         /*
 473          * IPv4 addresses are in big-endian byte order in item and in
 474          * efx_spec
 475          */
 476         if (mask->hdr.src_addr == supp_mask.hdr.src_addr) {
 477                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 478                 efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr;
 479         } else if (mask->hdr.src_addr != 0) {
 480                 goto fail_bad_mask;
 481         }
 482
 483         if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) {
 484                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 485                 efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr;
 486         } else if (mask->hdr.dst_addr != 0) {
 487                 goto fail_bad_mask;
 488         }
 489
 490         if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) {
 491                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 492                 efx_spec->efs_ip_proto = spec->hdr.next_proto_id;
 493         } else if (mask->hdr.next_proto_id != 0) {
 494                 goto fail_bad_mask;
 495         }
 496
 497         return 0;
 498
 499 fail_bad_mask:
 500         rte_flow_error_set(error, EINVAL,
 501                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 502                            "Bad mask in the IPV4 pattern item");
 503         return -rte_errno;
 504 }
 505
 506 /**
 507  * Convert IPv6 item to EFX filter specification.
 508  *
 509  * @param item[in]
 510  *   Item specification. Only source and destination addresses and
 511  *   next header fields are supported. If the mask is NULL, default
 512  *   mask will be used. Ranging is not supported.
 513  * @param efx_spec[in, out]
 514  *   EFX filter specification to update.
 515  * @param[out] error
 516  *   Perform verbose error reporting if not NULL.
 517  */
 518 static int
 519 sfc_flow_parse_ipv6(const struct rte_flow_item *item,
 520                     efx_filter_spec_t *efx_spec,
 521                     struct rte_flow_error *error)
 522 {
 523         int rc;
 524         const struct rte_flow_item_ipv6 *spec = NULL;
 525         const struct rte_flow_item_ipv6 *mask = NULL;
 526         const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6);
 527         const struct rte_flow_item_ipv6 supp_mask = {
 528                 .hdr = {
 529                         .src_addr = { 0xff, 0xff, 0xff, 0xff,
 530                                       0xff, 0xff, 0xff, 0xff,
 531                                       0xff, 0xff, 0xff, 0xff,
 532                                       0xff, 0xff, 0xff, 0xff },
 533                         .dst_addr = { 0xff, 0xff, 0xff, 0xff,
 534                                       0xff, 0xff, 0xff, 0xff,
 535                                       0xff, 0xff, 0xff, 0xff,
 536                                       0xff, 0xff, 0xff, 0xff },
 537                         .proto = 0xff,
 538                 }
 539         };
 540
 541         rc = sfc_flow_parse_init(item,
 542                                  (const void **)&spec,
 543                                  (const void **)&mask,
 544                                  &supp_mask,
 545                                  &rte_flow_item_ipv6_mask,
 546                                  sizeof(struct rte_flow_item_ipv6),
 547                                  error);
 548         if (rc != 0)
 549                 return rc;
 550
 551         /*
 552          * Filtering by IPv6 source and destination addresses requires
 553          * the appropriate ETHER_TYPE in hardware filters
 554          */
 555         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 556                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 557                 efx_spec->efs_ether_type = ether_type_ipv6;
 558         } else if (efx_spec->efs_ether_type != ether_type_ipv6) {
 559                 rte_flow_error_set(error, EINVAL,
 560                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 561                         "Ethertype in pattern with IPV6 item should be appropriate");
 562                 return -rte_errno;
 563         }
 564
 565         if (spec == NULL)
 566                 return 0;
 567
 568         /*
 569          * IPv6 addresses are in big-endian byte order in item and in
 570          * efx_spec
 571          */
 572         if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr,
 573                    sizeof(mask->hdr.src_addr)) == 0) {
 574                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 575
 576                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) !=
 577                                  sizeof(spec->hdr.src_addr));
 578                 rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr,
 579                            sizeof(efx_spec->efs_rem_host));
 580         } else if (!sfc_flow_is_zero(mask->hdr.src_addr,
 581                                      sizeof(mask->hdr.src_addr))) {
 582                 goto fail_bad_mask;
 583         }
 584
 585         if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr,
 586                    sizeof(mask->hdr.dst_addr)) == 0) {
 587                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 588
 589                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) !=
 590                                  sizeof(spec->hdr.dst_addr));
 591                 rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr,
 592                            sizeof(efx_spec->efs_loc_host));
 593         } else if (!sfc_flow_is_zero(mask->hdr.dst_addr,
 594                                      sizeof(mask->hdr.dst_addr))) {
 595                 goto fail_bad_mask;
 596         }
 597
 598         if (mask->hdr.proto == supp_mask.hdr.proto) {
 599                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 600                 efx_spec->efs_ip_proto = spec->hdr.proto;
 601         } else if (mask->hdr.proto != 0) {
 602                 goto fail_bad_mask;
 603         }
 604
 605         return 0;
 606
 607 fail_bad_mask:
 608         rte_flow_error_set(error, EINVAL,
 609                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 610                            "Bad mask in the IPV6 pattern item");
 611         return -rte_errno;
 612 }
 613
 614 /**
 615  * Convert TCP item to EFX filter specification.
 616  *
 617  * @param item[in]
 618  *   Item specification. Only source and destination ports fields
 619  *   are supported. If the mask is NULL, default mask will be used.
 620  *   Ranging is not supported.
 621  * @param efx_spec[in, out]
 622  *   EFX filter specification to update.
 623  * @param[out] error
 624  *   Perform verbose error reporting if not NULL.
 625  */
 626 static int
 627 sfc_flow_parse_tcp(const struct rte_flow_item *item,
 628                    efx_filter_spec_t *efx_spec,
 629                    struct rte_flow_error *error)
 630 {
 631         int rc;
 632         const struct rte_flow_item_tcp *spec = NULL;
 633         const struct rte_flow_item_tcp *mask = NULL;
 634         const struct rte_flow_item_tcp supp_mask = {
 635                 .hdr = {
 636                         .src_port = 0xffff,
 637                         .dst_port = 0xffff,
 638                 }
 639         };
 640
 641         rc = sfc_flow_parse_init(item,
 642                                  (const void **)&spec,
 643                                  (const void **)&mask,
 644                                  &supp_mask,
 645                                  &rte_flow_item_tcp_mask,
 646                                  sizeof(struct rte_flow_item_tcp),
 647                                  error);
 648         if (rc != 0)
 649                 return rc;
 650
 651         /*
 652          * Filtering by TCP source and destination ports requires
 653          * the appropriate IP_PROTO in hardware filters
 654          */
 655         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 656                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 657                 efx_spec->efs_ip_proto = EFX_IPPROTO_TCP;
 658         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) {
 659                 rte_flow_error_set(error, EINVAL,
 660                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 661                         "IP proto in pattern with TCP item should be appropriate");
 662                 return -rte_errno;
 663         }
 664
 665         if (spec == NULL)
 666                 return 0;
 667
 668         /*
 669          * Source and destination ports are in big-endian byte order in item and
 670          * in little-endian in efx_spec, so byte swap is used
 671          */
 672         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 673                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 674                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 675         } else if (mask->hdr.src_port != 0) {
 676                 goto fail_bad_mask;
 677         }
 678
 679         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 680                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 681                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 682         } else if (mask->hdr.dst_port != 0) {
 683                 goto fail_bad_mask;
 684         }
 685
 686         return 0;
 687
 688 fail_bad_mask:
 689         rte_flow_error_set(error, EINVAL,
 690                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 691                            "Bad mask in the TCP pattern item");
 692         return -rte_errno;
 693 }
 694
 695 /**
 696  * Convert UDP item to EFX filter specification.
 697  *
 698  * @param item[in]
 699  *   Item specification. Only source and destination ports fields
 700  *   are supported. If the mask is NULL, default mask will be used.
 701  *   Ranging is not supported.
 702  * @param efx_spec[in, out]
 703  *   EFX filter specification to update.
 704  * @param[out] error
 705  *   Perform verbose error reporting if not NULL.
 706  */
 707 static int
 708 sfc_flow_parse_udp(const struct rte_flow_item *item,
 709                    efx_filter_spec_t *efx_spec,
 710                    struct rte_flow_error *error)
 711 {
 712         int rc;
 713         const struct rte_flow_item_udp *spec = NULL;
 714         const struct rte_flow_item_udp *mask = NULL;
 715         const struct rte_flow_item_udp supp_mask = {
 716                 .hdr = {
 717                         .src_port = 0xffff,
 718                         .dst_port = 0xffff,
 719                 }
 720         };
 721
 722         rc = sfc_flow_parse_init(item,
 723                                  (const void **)&spec,
 724                                  (const void **)&mask,
 725                                  &supp_mask,
 726                                  &rte_flow_item_udp_mask,
 727                                  sizeof(struct rte_flow_item_udp),
 728                                  error);
 729         if (rc != 0)
 730                 return rc;
 731
 732         /*
 733          * Filtering by UDP source and destination ports requires
 734          * the appropriate IP_PROTO in hardware filters
 735          */
 736         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 737                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 738                 efx_spec->efs_ip_proto = EFX_IPPROTO_UDP;
 739         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) {
 740                 rte_flow_error_set(error, EINVAL,
 741                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 742                         "IP proto in pattern with UDP item should be appropriate");
 743                 return -rte_errno;
 744         }
 745
 746         if (spec == NULL)
 747                 return 0;
 748
 749         /*
 750          * Source and destination ports are in big-endian byte order in item and
 751          * in little-endian in efx_spec, so byte swap is used
 752          */
 753         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 754                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 755                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 756         } else if (mask->hdr.src_port != 0) {
 757                 goto fail_bad_mask;
 758         }
 759
 760         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 761                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 762                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 763         } else if (mask->hdr.dst_port != 0) {
 764                 goto fail_bad_mask;
 765         }
 766
 767         return 0;
 768
 769 fail_bad_mask:
 770         rte_flow_error_set(error, EINVAL,
 771                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 772                            "Bad mask in the UDP pattern item");
 773         return -rte_errno;
 774 }
 775
 776 /*
 777  * Filters for encapsulated packets match based on the EtherType and IP
 778  * protocol in the outer frame.
 779  */
 780 static int
 781 sfc_flow_set_match_flags_for_encap_pkts(const struct rte_flow_item *item,
 782                                         efx_filter_spec_t *efx_spec,
 783                                         uint8_t ip_proto,
 784                                         struct rte_flow_error *error)
 785 {
 786         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 787                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 788                 efx_spec->efs_ip_proto = ip_proto;
 789         } else if (efx_spec->efs_ip_proto != ip_proto) {
 790                 switch (ip_proto) {
 791                 case EFX_IPPROTO_UDP:
 792                         rte_flow_error_set(error, EINVAL,
 793                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 794                                 "Outer IP header protocol must be UDP "
 795                                 "in VxLAN/GENEVE pattern");
 796                         return -rte_errno;
 797
 798                 case EFX_IPPROTO_GRE:
 799                         rte_flow_error_set(error, EINVAL,
 800                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 801                                 "Outer IP header protocol must be GRE "
 802                                 "in NVGRE pattern");
 803                         return -rte_errno;
 804
 805                 default:
 806                         rte_flow_error_set(error, EINVAL,
 807                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 808                                 "Only VxLAN/GENEVE/NVGRE tunneling patterns "
 809                                 "are supported");
 810                         return -rte_errno;
 811                 }
 812         }
 813
 814         if (efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE &&
 815             efx_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4 &&
 816             efx_spec->efs_ether_type != EFX_ETHER_TYPE_IPV6) {
 817                 rte_flow_error_set(error, EINVAL,
 818                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 819                         "Outer frame EtherType in pattern with tunneling "
 820                         "must be IPv4 or IPv6");
 821                 return -rte_errno;
 822         }
 823
 824         return 0;
 825 }
 826
 827 static int
 828 sfc_flow_set_efx_spec_vni_or_vsid(efx_filter_spec_t *efx_spec,
 829                                   const uint8_t *vni_or_vsid_val,
 830                                   const uint8_t *vni_or_vsid_mask,
 831                                   const struct rte_flow_item *item,
 832                                   struct rte_flow_error *error)
 833 {
 834         const uint8_t vni_or_vsid_full_mask[EFX_VNI_OR_VSID_LEN] = {
 835                 0xff, 0xff, 0xff
 836         };
 837
 838         if (memcmp(vni_or_vsid_mask, vni_or_vsid_full_mask,
 839                    EFX_VNI_OR_VSID_LEN) == 0) {
 840                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 841                 rte_memcpy(efx_spec->efs_vni_or_vsid, vni_or_vsid_val,
 842                            EFX_VNI_OR_VSID_LEN);
 843         } else if (!sfc_flow_is_zero(vni_or_vsid_mask, EFX_VNI_OR_VSID_LEN)) {
 844                 rte_flow_error_set(error, EINVAL,
 845                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 846                                    "Unsupported VNI/VSID mask");
 847                 return -rte_errno;
 848         }
 849
 850         return 0;
 851 }
 852
 853 /**
 854  * Convert VXLAN item to EFX filter specification.
 855  *
 856  * @param item[in]
 857  *   Item specification. Only VXLAN network identifier field is supported.
 858  *   If the mask is NULL, default mask will be used.
 859  *   Ranging is not supported.
 860  * @param efx_spec[in, out]
 861  *   EFX filter specification to update.
 862  * @param[out] error
 863  *   Perform verbose error reporting if not NULL.
 864  */
 865 static int
 866 sfc_flow_parse_vxlan(const struct rte_flow_item *item,
 867                      efx_filter_spec_t *efx_spec,
 868                      struct rte_flow_error *error)
 869 {
 870         int rc;
 871         const struct rte_flow_item_vxlan *spec = NULL;
 872         const struct rte_flow_item_vxlan *mask = NULL;
 873         const struct rte_flow_item_vxlan supp_mask = {
 874                 .vni = { 0xff, 0xff, 0xff }
 875         };
 876
 877         rc = sfc_flow_parse_init(item,
 878                                  (const void **)&spec,
 879                                  (const void **)&mask,
 880                                  &supp_mask,
 881                                  &rte_flow_item_vxlan_mask,
 882                                  sizeof(struct rte_flow_item_vxlan),
 883                                  error);
 884         if (rc != 0)
 885                 return rc;
 886
 887         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
 888                                                      EFX_IPPROTO_UDP, error);
 889         if (rc != 0)
 890                 return rc;
 891
 892         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_VXLAN;
 893         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 894
 895         if (spec == NULL)
 896                 return 0;
 897
 898         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->vni,
 899                                                mask->vni, item, error);
 900
 901         return rc;
 902 }
 903
 904 /**
 905  * Convert GENEVE item to EFX filter specification.
 906  *
 907  * @param item[in]
 908  *   Item specification. Only Virtual Network Identifier and protocol type
 909  *   fields are supported. But protocol type can be only Ethernet (0x6558).
 910  *   If the mask is NULL, default mask will be used.
 911  *   Ranging is not supported.
 912  * @param efx_spec[in, out]
 913  *   EFX filter specification to update.
 914  * @param[out] error
 915  *   Perform verbose error reporting if not NULL.
 916  */
 917 static int
 918 sfc_flow_parse_geneve(const struct rte_flow_item *item,
 919                       efx_filter_spec_t *efx_spec,
 920                       struct rte_flow_error *error)
 921 {
 922         int rc;
 923         const struct rte_flow_item_geneve *spec = NULL;
 924         const struct rte_flow_item_geneve *mask = NULL;
 925         const struct rte_flow_item_geneve supp_mask = {
 926                 .protocol = RTE_BE16(0xffff),
 927                 .vni = { 0xff, 0xff, 0xff }
 928         };
 929
 930         rc = sfc_flow_parse_init(item,
 931                                  (const void **)&spec,
 932                                  (const void **)&mask,
 933                                  &supp_mask,
 934                                  &rte_flow_item_geneve_mask,
 935                                  sizeof(struct rte_flow_item_geneve),
 936                                  error);
 937         if (rc != 0)
 938                 return rc;
 939
 940         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
 941                                                      EFX_IPPROTO_UDP, error);
 942         if (rc != 0)
 943                 return rc;
 944
 945         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_GENEVE;
 946         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 947
 948         if (spec == NULL)
 949                 return 0;
 950
 951         if (mask->protocol == supp_mask.protocol) {
 952                 if (spec->protocol != rte_cpu_to_be_16(ETHER_TYPE_TEB)) {
 953                         rte_flow_error_set(error, EINVAL,
 954                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 955                                 "GENEVE encap. protocol must be Ethernet "
 956                                 "(0x6558) in the GENEVE pattern item");
 957                         return -rte_errno;
 958                 }
 959         } else if (mask->protocol != 0) {
 960                 rte_flow_error_set(error, EINVAL,
 961                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 962                         "Unsupported mask for GENEVE encap. protocol");
 963                 return -rte_errno;
 964         }
 965
 966         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->vni,
 967                                                mask->vni, item, error);
 968
 969         return rc;
 970 }
 971
 972 /**
 973  * Convert NVGRE item to EFX filter specification.
 974  *
 975  * @param item[in]
 976  *   Item specification. Only virtual subnet ID field is supported.
 977  *   If the mask is NULL, default mask will be used.
 978  *   Ranging is not supported.
 979  * @param efx_spec[in, out]
 980  *   EFX filter specification to update.
 981  * @param[out] error
 982  *   Perform verbose error reporting if not NULL.
 983  */
 984 static int
 985 sfc_flow_parse_nvgre(const struct rte_flow_item *item,
 986                      efx_filter_spec_t *efx_spec,
 987                      struct rte_flow_error *error)
 988 {
 989         int rc;
 990         const struct rte_flow_item_nvgre *spec = NULL;
 991         const struct rte_flow_item_nvgre *mask = NULL;
 992         const struct rte_flow_item_nvgre supp_mask = {
 993                 .tni = { 0xff, 0xff, 0xff }
 994         };
 995
 996         rc = sfc_flow_parse_init(item,
 997                                  (const void **)&spec,
 998                                  (const void **)&mask,
 999                                  &supp_mask,
1000                                  &rte_flow_item_nvgre_mask,
1001                                  sizeof(struct rte_flow_item_nvgre),
1002                                  error);
1003         if (rc != 0)
1004                 return rc;
1005
1006         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
1007                                                      EFX_IPPROTO_GRE, error);
1008         if (rc != 0)
1009                 return rc;
1010
1011         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_NVGRE;
1012         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
1013
1014         if (spec == NULL)
1015                 return 0;
1016
1017         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->tni,
1018                                                mask->tni, item, error);
1019
1020         return rc;
1021 }
1022
1023 static const struct sfc_flow_item sfc_flow_items[] = {
1024         {
1025                 .type = RTE_FLOW_ITEM_TYPE_VOID,
1026                 .prev_layer = SFC_FLOW_ITEM_ANY_LAYER,
1027                 .layer = SFC_FLOW_ITEM_ANY_LAYER,
1028                 .parse = sfc_flow_parse_void,
1029         },
1030         {
1031                 .type = RTE_FLOW_ITEM_TYPE_ETH,
1032                 .prev_layer = SFC_FLOW_ITEM_START_LAYER,
1033                 .layer = SFC_FLOW_ITEM_L2,
1034                 .parse = sfc_flow_parse_eth,
1035         },
1036         {
1037                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
1038                 .prev_layer = SFC_FLOW_ITEM_L2,
1039                 .layer = SFC_FLOW_ITEM_L2,
1040                 .parse = sfc_flow_parse_vlan,
1041         },
1042         {
1043                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
1044                 .prev_layer = SFC_FLOW_ITEM_L2,
1045                 .layer = SFC_FLOW_ITEM_L3,
1046                 .parse = sfc_flow_parse_ipv4,
1047         },
1048         {
1049                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
1050                 .prev_layer = SFC_FLOW_ITEM_L2,
1051                 .layer = SFC_FLOW_ITEM_L3,
1052                 .parse = sfc_flow_parse_ipv6,
1053         },
1054         {
1055                 .type = RTE_FLOW_ITEM_TYPE_TCP,
1056                 .prev_layer = SFC_FLOW_ITEM_L3,
1057                 .layer = SFC_FLOW_ITEM_L4,
1058                 .parse = sfc_flow_parse_tcp,
1059         },
1060         {
1061                 .type = RTE_FLOW_ITEM_TYPE_UDP,
1062                 .prev_layer = SFC_FLOW_ITEM_L3,
1063                 .layer = SFC_FLOW_ITEM_L4,
1064                 .parse = sfc_flow_parse_udp,
1065         },
1066         {
1067                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
1068                 .prev_layer = SFC_FLOW_ITEM_L4,
1069                 .layer = SFC_FLOW_ITEM_START_LAYER,
1070                 .parse = sfc_flow_parse_vxlan,
1071         },
1072         {
1073                 .type = RTE_FLOW_ITEM_TYPE_GENEVE,
1074                 .prev_layer = SFC_FLOW_ITEM_L4,
1075                 .layer = SFC_FLOW_ITEM_START_LAYER,
1076                 .parse = sfc_flow_parse_geneve,
1077         },
1078         {
1079                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
1080                 .prev_layer = SFC_FLOW_ITEM_L3,
1081                 .layer = SFC_FLOW_ITEM_START_LAYER,
1082                 .parse = sfc_flow_parse_nvgre,
1083         },
1084 };
1085
1086 /*
1087  * Protocol-independent flow API support
1088  */
1089 static int
1090 sfc_flow_parse_attr(const struct rte_flow_attr *attr,
1091                     struct rte_flow *flow,
1092                     struct rte_flow_error *error)
1093 {
1094         if (attr == NULL) {
1095                 rte_flow_error_set(error, EINVAL,
1096                                    RTE_FLOW_ERROR_TYPE_ATTR, NULL,
1097                                    "NULL attribute");
1098                 return -rte_errno;
1099         }
1100         if (attr->group != 0) {
1101                 rte_flow_error_set(error, ENOTSUP,
1102                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1103                                    "Groups are not supported");
1104                 return -rte_errno;
1105         }
1106         if (attr->priority != 0) {
1107                 rte_flow_error_set(error, ENOTSUP,
1108                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
1109                                    "Priorities are not supported");
1110                 return -rte_errno;
1111         }
1112         if (attr->egress != 0) {
1113                 rte_flow_error_set(error, ENOTSUP,
1114                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
1115                                    "Egress is not supported");
1116                 return -rte_errno;
1117         }
1118         if (attr->transfer != 0) {
1119                 rte_flow_error_set(error, ENOTSUP,
1120                                    RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, attr,
1121                                    "Transfer is not supported");
1122                 return -rte_errno;
1123         }
1124         if (attr->ingress == 0) {
1125                 rte_flow_error_set(error, ENOTSUP,
1126                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
1127                                    "Only ingress is supported");
1128                 return -rte_errno;
1129         }
1130
1131         flow->spec.template.efs_flags |= EFX_FILTER_FLAG_RX;
1132         flow->spec.template.efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
1133
1134         return 0;
1135 }
1136
1137 /* Get item from array sfc_flow_items */
1138 static const struct sfc_flow_item *
1139 sfc_flow_get_item(enum rte_flow_item_type type)
1140 {
1141         unsigned int i;
1142
1143         for (i = 0; i < RTE_DIM(sfc_flow_items); i++)
1144                 if (sfc_flow_items[i].type == type)
1145                         return &sfc_flow_items[i];
1146
1147         return NULL;
1148 }
1149
1150 static int
1151 sfc_flow_parse_pattern(const struct rte_flow_item pattern[],
1152                        struct rte_flow *flow,
1153                        struct rte_flow_error *error)
1154 {
1155         int rc;
1156         unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER;
1157         boolean_t is_ifrm = B_FALSE;
1158         const struct sfc_flow_item *item;
1159
1160         if (pattern == NULL) {
1161                 rte_flow_error_set(error, EINVAL,
1162                                    RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
1163                                    "NULL pattern");
1164                 return -rte_errno;
1165         }
1166
1167         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
1168                 item = sfc_flow_get_item(pattern->type);
1169                 if (item == NULL) {
1170                         rte_flow_error_set(error, ENOTSUP,
1171                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
1172                                            "Unsupported pattern item");
1173                         return -rte_errno;
1174                 }
1175
1176                 /*
1177                  * Omitting one or several protocol layers at the beginning
1178                  * of pattern is supported
1179                  */
1180                 if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
1181                     prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
1182                     item->prev_layer != prev_layer) {
1183                         rte_flow_error_set(error, ENOTSUP,
1184                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
1185                                            "Unexpected sequence of pattern items");
1186                         return -rte_errno;
1187                 }
1188
1189                 /*
1190                  * Allow only VOID and ETH pattern items in the inner frame.
1191                  * Also check that there is only one tunneling protocol.
1192                  */
1193                 switch (item->type) {
1194                 case RTE_FLOW_ITEM_TYPE_VOID:
1195                 case RTE_FLOW_ITEM_TYPE_ETH:
1196                         break;
1197
1198                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1199                 case RTE_FLOW_ITEM_TYPE_GENEVE:
1200                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1201                         if (is_ifrm) {
1202                                 rte_flow_error_set(error, EINVAL,
1203                                         RTE_FLOW_ERROR_TYPE_ITEM,
1204                                         pattern,
1205                                         "More than one tunneling protocol");
1206                                 return -rte_errno;
1207                         }
1208                         is_ifrm = B_TRUE;
1209                         break;
1210
1211                 default:
1212                         if (is_ifrm) {
1213                                 rte_flow_error_set(error, EINVAL,
1214                                         RTE_FLOW_ERROR_TYPE_ITEM,
1215                                         pattern,
1216                                         "There is an unsupported pattern item "
1217                                         "in the inner frame");
1218                                 return -rte_errno;
1219                         }
1220                         break;
1221                 }
1222
1223                 rc = item->parse(pattern, &flow->spec.template, error);
1224                 if (rc != 0)
1225                         return rc;
1226
1227                 if (item->layer != SFC_FLOW_ITEM_ANY_LAYER)
1228                         prev_layer = item->layer;
1229         }
1230
1231         return 0;
1232 }
1233
1234 static int
1235 sfc_flow_parse_queue(struct sfc_adapter *sa,
1236                      const struct rte_flow_action_queue *queue,
1237                      struct rte_flow *flow)
1238 {
1239         struct sfc_rxq *rxq;
1240
1241         if (queue->index >= sa->rxq_count)
1242                 return -EINVAL;
1243
1244         rxq = sa->rxq_info[queue->index].rxq;
1245         flow->spec.template.efs_dmaq_id = (uint16_t)rxq->hw_index;
1246
1247         return 0;
1248 }
1249
1250 static int
1251 sfc_flow_parse_rss(struct sfc_adapter *sa,
1252                    const struct rte_flow_action_rss *action_rss,
1253                    struct rte_flow *flow)
1254 {
1255         struct sfc_rss *rss = &sa->rss;
1256         unsigned int rxq_sw_index;
1257         struct sfc_rxq *rxq;
1258         unsigned int rxq_hw_index_min;
1259         unsigned int rxq_hw_index_max;
1260         efx_rx_hash_type_t efx_hash_types;
1261         const uint8_t *rss_key;
1262         struct sfc_flow_rss *sfc_rss_conf = &flow->rss_conf;
1263         unsigned int i;
1264
1265         if (action_rss->queue_num == 0)
1266                 return -EINVAL;
1267
1268         rxq_sw_index = sa->rxq_count - 1;
1269         rxq = sa->rxq_info[rxq_sw_index].rxq;
1270         rxq_hw_index_min = rxq->hw_index;
1271         rxq_hw_index_max = 0;
1272
1273         for (i = 0; i < action_rss->queue_num; ++i) {
1274                 rxq_sw_index = action_rss->queue[i];
1275
1276                 if (rxq_sw_index >= sa->rxq_count)
1277                         return -EINVAL;
1278
1279                 rxq = sa->rxq_info[rxq_sw_index].rxq;
1280
1281                 if (rxq->hw_index < rxq_hw_index_min)
1282                         rxq_hw_index_min = rxq->hw_index;
1283
1284                 if (rxq->hw_index > rxq_hw_index_max)
1285                         rxq_hw_index_max = rxq->hw_index;
1286         }
1287
1288         switch (action_rss->func) {
1289         case RTE_ETH_HASH_FUNCTION_DEFAULT:
1290         case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
1291                 break;
1292         default:
1293                 return -EINVAL;
1294         }
1295
1296         if (action_rss->level)
1297                 return -EINVAL;
1298
1299         /*
1300          * Dummy RSS action with only one queue and no specific settings
1301          * for hash types and key does not require dedicated RSS context
1302          * and may be simplified to single queue action.
1303          */
1304         if (action_rss->queue_num == 1 && action_rss->types == 0 &&
1305             action_rss->key_len == 0) {
1306                 flow->spec.template.efs_dmaq_id = rxq_hw_index_min;
1307                 return 0;
1308         }
1309
1310         if (action_rss->types) {
1311                 int rc;
1312
1313                 rc = sfc_rx_hf_rte_to_efx(sa, action_rss->types,
1314                                           &efx_hash_types);
1315                 if (rc != 0)
1316                         return -rc;
1317         } else {
1318                 unsigned int i;
1319
1320                 efx_hash_types = 0;
1321                 for (i = 0; i < rss->hf_map_nb_entries; ++i)
1322                         efx_hash_types |= rss->hf_map[i].efx;
1323         }
1324
1325         if (action_rss->key_len) {
1326                 if (action_rss->key_len != sizeof(rss->key))
1327                         return -EINVAL;
1328
1329                 rss_key = action_rss->key;
1330         } else {
1331                 rss_key = rss->key;
1332         }
1333
1334         flow->rss = B_TRUE;
1335
1336         sfc_rss_conf->rxq_hw_index_min = rxq_hw_index_min;
1337         sfc_rss_conf->rxq_hw_index_max = rxq_hw_index_max;
1338         sfc_rss_conf->rss_hash_types = efx_hash_types;
1339         rte_memcpy(sfc_rss_conf->rss_key, rss_key, sizeof(rss->key));
1340
1341         for (i = 0; i < RTE_DIM(sfc_rss_conf->rss_tbl); ++i) {
1342                 unsigned int nb_queues = action_rss->queue_num;
1343                 unsigned int rxq_sw_index = action_rss->queue[i % nb_queues];
1344                 struct sfc_rxq *rxq = sa->rxq_info[rxq_sw_index].rxq;
1345
1346                 sfc_rss_conf->rss_tbl[i] = rxq->hw_index - rxq_hw_index_min;
1347         }
1348
1349         return 0;
1350 }
1351
1352 static int
1353 sfc_flow_spec_flush(struct sfc_adapter *sa, struct sfc_flow_spec *spec,
1354                     unsigned int filters_count)
1355 {
1356         unsigned int i;
1357         int ret = 0;
1358
1359         for (i = 0; i < filters_count; i++) {
1360                 int rc;
1361
1362                 rc = efx_filter_remove(sa->nic, &spec->filters[i]);
1363                 if (ret == 0 && rc != 0) {
1364                         sfc_err(sa, "failed to remove filter specification "
1365                                 "(rc = %d)", rc);
1366                         ret = rc;
1367                 }
1368         }
1369
1370         return ret;
1371 }
1372
1373 static int
1374 sfc_flow_spec_insert(struct sfc_adapter *sa, struct sfc_flow_spec *spec)
1375 {
1376         unsigned int i;
1377         int rc = 0;
1378
1379         for (i = 0; i < spec->count; i++) {
1380                 rc = efx_filter_insert(sa->nic, &spec->filters[i]);
1381                 if (rc != 0) {
1382                         sfc_flow_spec_flush(sa, spec, i);
1383                         break;
1384                 }
1385         }
1386
1387         return rc;
1388 }
1389
1390 static int
1391 sfc_flow_spec_remove(struct sfc_adapter *sa, struct sfc_flow_spec *spec)
1392 {
1393         return sfc_flow_spec_flush(sa, spec, spec->count);
1394 }
1395
1396 static int
1397 sfc_flow_filter_insert(struct sfc_adapter *sa,
1398                        struct rte_flow *flow)
1399 {
1400         struct sfc_rss *rss = &sa->rss;
1401         struct sfc_flow_rss *flow_rss = &flow->rss_conf;
1402         uint32_t efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
1403         unsigned int i;
1404         int rc = 0;
1405
1406         if (flow->rss) {
1407                 unsigned int rss_spread = MIN(flow_rss->rxq_hw_index_max -
1408                                               flow_rss->rxq_hw_index_min + 1,
1409                                               EFX_MAXRSS);
1410
1411                 rc = efx_rx_scale_context_alloc(sa->nic,
1412                                                 EFX_RX_SCALE_EXCLUSIVE,
1413                                                 rss_spread,
1414                                                 &efs_rss_context);
1415                 if (rc != 0)
1416                         goto fail_scale_context_alloc;
1417
1418                 rc = efx_rx_scale_mode_set(sa->nic, efs_rss_context,
1419                                            rss->hash_alg,
1420                                            flow_rss->rss_hash_types, B_TRUE);
1421                 if (rc != 0)
1422                         goto fail_scale_mode_set;
1423
1424                 rc = efx_rx_scale_key_set(sa->nic, efs_rss_context,
1425                                           flow_rss->rss_key,
1426                                           sizeof(rss->key));
1427                 if (rc != 0)
1428                         goto fail_scale_key_set;
1429
1430                 /*
1431                  * At this point, fully elaborated filter specifications
1432                  * have been produced from the template. To make sure that
1433                  * RSS behaviour is consistent between them, set the same
1434                  * RSS context value everywhere.
1435                  */
1436                 for (i = 0; i < flow->spec.count; i++) {
1437                         efx_filter_spec_t *spec = &flow->spec.filters[i];
1438
1439                         spec->efs_rss_context = efs_rss_context;
1440                         spec->efs_dmaq_id = flow_rss->rxq_hw_index_min;
1441                         spec->efs_flags |= EFX_FILTER_FLAG_RX_RSS;
1442                 }
1443         }
1444
1445         rc = sfc_flow_spec_insert(sa, &flow->spec);
1446         if (rc != 0)
1447                 goto fail_filter_insert;
1448
1449         if (flow->rss) {
1450                 /*
1451                  * Scale table is set after filter insertion because
1452                  * the table entries are relative to the base RxQ ID
1453                  * and the latter is submitted to the HW by means of
1454                  * inserting a filter, so by the time of the request
1455                  * the HW knows all the information needed to verify
1456                  * the table entries, and the operation will succeed
1457                  */
1458                 rc = efx_rx_scale_tbl_set(sa->nic, efs_rss_context,
1459                                           flow_rss->rss_tbl,
1460                                           RTE_DIM(flow_rss->rss_tbl));
1461                 if (rc != 0)
1462                         goto fail_scale_tbl_set;
1463         }
1464
1465         return 0;
1466
1467 fail_scale_tbl_set:
1468         sfc_flow_spec_remove(sa, &flow->spec);
1469
1470 fail_filter_insert:
1471 fail_scale_key_set:
1472 fail_scale_mode_set:
1473         if (efs_rss_context != EFX_RSS_CONTEXT_DEFAULT)
1474                 efx_rx_scale_context_free(sa->nic, efs_rss_context);
1475
1476 fail_scale_context_alloc:
1477         return rc;
1478 }
1479
1480 static int
1481 sfc_flow_filter_remove(struct sfc_adapter *sa,
1482                        struct rte_flow *flow)
1483 {
1484         int rc = 0;
1485
1486         rc = sfc_flow_spec_remove(sa, &flow->spec);
1487         if (rc != 0)
1488                 return rc;
1489
1490         if (flow->rss) {
1491                 /*
1492                  * All specifications for a given flow rule have the same RSS
1493                  * context, so that RSS context value is taken from the first
1494                  * filter specification
1495                  */
1496                 efx_filter_spec_t *spec = &flow->spec.filters[0];
1497
1498                 rc = efx_rx_scale_context_free(sa->nic, spec->efs_rss_context);
1499         }
1500
1501         return rc;
1502 }
1503
1504 static int
1505 sfc_flow_parse_mark(struct sfc_adapter *sa,
1506                     const struct rte_flow_action_mark *mark,
1507                     struct rte_flow *flow)
1508 {
1509         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
1510
1511         if (mark == NULL || mark->id > encp->enc_filter_action_mark_max)
1512                 return EINVAL;
1513
1514         flow->spec.template.efs_flags |= EFX_FILTER_FLAG_ACTION_MARK;
1515         flow->spec.template.efs_mark = mark->id;
1516
1517         return 0;
1518 }
1519
1520 static int
1521 sfc_flow_parse_actions(struct sfc_adapter *sa,
1522                        const struct rte_flow_action actions[],
1523                        struct rte_flow *flow,
1524                        struct rte_flow_error *error)
1525 {
1526         int rc;
1527         const unsigned int dp_rx_features = sa->dp_rx->features;
1528         uint32_t actions_set = 0;
1529         const uint32_t fate_actions_mask = (1UL << RTE_FLOW_ACTION_TYPE_QUEUE) |
1530                                            (1UL << RTE_FLOW_ACTION_TYPE_RSS) |
1531                                            (1UL << RTE_FLOW_ACTION_TYPE_DROP);
1532         const uint32_t mark_actions_mask = (1UL << RTE_FLOW_ACTION_TYPE_MARK) |
1533                                            (1UL << RTE_FLOW_ACTION_TYPE_FLAG);
1534
1535         if (actions == NULL) {
1536                 rte_flow_error_set(error, EINVAL,
1537                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
1538                                    "NULL actions");
1539                 return -rte_errno;
1540         }
1541
1542 #define SFC_BUILD_SET_OVERFLOW(_action, _set) \
1543         RTE_BUILD_BUG_ON(_action >= sizeof(_set) * CHAR_BIT)
1544
1545         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1546                 switch (actions->type) {
1547                 case RTE_FLOW_ACTION_TYPE_VOID:
1548                         SFC_BUILD_SET_OVERFLOW(RTE_FLOW_ACTION_TYPE_VOID,
1549                                                actions_set);
1550                         break;
1551
1552                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1553                         SFC_BUILD_SET_OVERFLOW(RTE_FLOW_ACTION_TYPE_QUEUE,
1554                                                actions_set);
1555                         if ((actions_set & fate_actions_mask) != 0)
1556                                 goto fail_fate_actions;
1557
1558                         rc = sfc_flow_parse_queue(sa, actions->conf, flow);
1559                         if (rc != 0) {
1560                                 rte_flow_error_set(error, EINVAL,
1561                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1562                                         "Bad QUEUE action");
1563                                 return -rte_errno;
1564                         }
1565                         break;
1566
1567                 case RTE_FLOW_ACTION_TYPE_RSS:
1568                         SFC_BUILD_SET_OVERFLOW(RTE_FLOW_ACTION_TYPE_RSS,
1569                                                actions_set);
1570                         if ((actions_set & fate_actions_mask) != 0)
1571                                 goto fail_fate_actions;
1572
1573                         rc = sfc_flow_parse_rss(sa, actions->conf, flow);
1574                         if (rc != 0) {
1575                                 rte_flow_error_set(error, -rc,
1576                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1577                                         "Bad RSS action");
1578                                 return -rte_errno;
1579                         }
1580                         break;
1581
1582                 case RTE_FLOW_ACTION_TYPE_DROP:
1583                         SFC_BUILD_SET_OVERFLOW(RTE_FLOW_ACTION_TYPE_DROP,
1584                                                actions_set);
1585                         if ((actions_set & fate_actions_mask) != 0)
1586                                 goto fail_fate_actions;
1587
1588                         flow->spec.template.efs_dmaq_id =
1589                                 EFX_FILTER_SPEC_RX_DMAQ_ID_DROP;
1590                         break;
1591
1592                 case RTE_FLOW_ACTION_TYPE_FLAG:
1593                         SFC_BUILD_SET_OVERFLOW(RTE_FLOW_ACTION_TYPE_FLAG,
1594                                                actions_set);
1595                         if ((actions_set & mark_actions_mask) != 0)
1596                                 goto fail_actions_overlap;
1597
1598                         if ((dp_rx_features & SFC_DP_RX_FEAT_FLOW_FLAG) == 0) {
1599                                 rte_flow_error_set(error, ENOTSUP,
1600                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1601                                         "FLAG action is not supported on the current Rx datapath");
1602                                 return -rte_errno;
1603                         }
1604
1605                         flow->spec.template.efs_flags |=
1606                                 EFX_FILTER_FLAG_ACTION_FLAG;
1607                         break;
1608
1609                 case RTE_FLOW_ACTION_TYPE_MARK:
1610                         SFC_BUILD_SET_OVERFLOW(RTE_FLOW_ACTION_TYPE_MARK,
1611                                                actions_set);
1612                         if ((actions_set & mark_actions_mask) != 0)
1613                                 goto fail_actions_overlap;
1614
1615                         if ((dp_rx_features & SFC_DP_RX_FEAT_FLOW_MARK) == 0) {
1616                                 rte_flow_error_set(error, ENOTSUP,
1617                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1618                                         "MARK action is not supported on the current Rx datapath");
1619                                 return -rte_errno;
1620                         }
1621
1622                         rc = sfc_flow_parse_mark(sa, actions->conf, flow);
1623                         if (rc != 0) {
1624                                 rte_flow_error_set(error, rc,
1625                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1626                                         "Bad MARK action");
1627                                 return -rte_errno;
1628                         }
1629                         break;
1630
1631                 default:
1632                         rte_flow_error_set(error, ENOTSUP,
1633                                            RTE_FLOW_ERROR_TYPE_ACTION, actions,
1634                                            "Action is not supported");
1635                         return -rte_errno;
1636                 }
1637
1638                 actions_set |= (1UL << actions->type);
1639         }
1640 #undef SFC_BUILD_SET_OVERFLOW
1641
1642         /* When fate is unknown, drop traffic. */
1643         if ((actions_set & fate_actions_mask) == 0) {
1644                 flow->spec.template.efs_dmaq_id =
1645                         EFX_FILTER_SPEC_RX_DMAQ_ID_DROP;
1646         }
1647
1648         return 0;
1649
1650 fail_fate_actions:
1651         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, actions,
1652                            "Cannot combine several fate-deciding actions, "
1653                            "choose between QUEUE, RSS or DROP");
1654         return -rte_errno;
1655
1656 fail_actions_overlap:
1657         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, actions,
1658                            "Overlapping actions are not supported");
1659         return -rte_errno;
1660 }
1661
1662 /**
1663  * Set the EFX_FILTER_MATCH_UNKNOWN_UCAST_DST
1664  * and EFX_FILTER_MATCH_UNKNOWN_MCAST_DST match flags in the same
1665  * specifications after copying.
1666  *
1667  * @param spec[in, out]
1668  *   SFC flow specification to update.
1669  * @param filters_count_for_one_val[in]
1670  *   How many specifications should have the same match flag, what is the
1671  *   number of specifications before copying.
1672  * @param error[out]
1673  *   Perform verbose error reporting if not NULL.
1674  */
1675 static int
1676 sfc_flow_set_unknown_dst_flags(struct sfc_flow_spec *spec,
1677                                unsigned int filters_count_for_one_val,
1678                                struct rte_flow_error *error)
1679 {
1680         unsigned int i;
1681         static const efx_filter_match_flags_t vals[] = {
1682                 EFX_FILTER_MATCH_UNKNOWN_UCAST_DST,
1683                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST
1684         };
1685
1686         if (filters_count_for_one_val * RTE_DIM(vals) != spec->count) {
1687                 rte_flow_error_set(error, EINVAL,
1688                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1689                         "Number of specifications is incorrect while copying "
1690                         "by unknown destination flags");
1691                 return -rte_errno;
1692         }
1693
1694         for (i = 0; i < spec->count; i++) {
1695                 /* The check above ensures that divisor can't be zero here */
1696                 spec->filters[i].efs_match_flags |=
1697                         vals[i / filters_count_for_one_val];
1698         }
1699
1700         return 0;
1701 }
1702
1703 /**
1704  * Check that the following conditions are met:
1705  * - the list of supported filters has a filter
1706  *   with EFX_FILTER_MATCH_UNKNOWN_MCAST_DST flag instead of
1707  *   EFX_FILTER_MATCH_UNKNOWN_UCAST_DST, since this filter will also
1708  *   be inserted.
1709  *
1710  * @param match[in]
1711  *   The match flags of filter.
1712  * @param spec[in]
1713  *   Specification to be supplemented.
1714  * @param filter[in]
1715  *   SFC filter with list of supported filters.
1716  */
1717 static boolean_t
1718 sfc_flow_check_unknown_dst_flags(efx_filter_match_flags_t match,
1719                                  __rte_unused efx_filter_spec_t *spec,
1720                                  struct sfc_filter *filter)
1721 {
1722         unsigned int i;
1723         efx_filter_match_flags_t match_mcast_dst;
1724
1725         match_mcast_dst =
1726                 (match & ~EFX_FILTER_MATCH_UNKNOWN_UCAST_DST) |
1727                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
1728         for (i = 0; i < filter->supported_match_num; i++) {
1729                 if (match_mcast_dst == filter->supported_match[i])
1730                         return B_TRUE;
1731         }
1732
1733         return B_FALSE;
1734 }
1735
1736 /**
1737  * Set the EFX_FILTER_MATCH_ETHER_TYPE match flag and EFX_ETHER_TYPE_IPV4 and
1738  * EFX_ETHER_TYPE_IPV6 values of the corresponding field in the same
1739  * specifications after copying.
1740  *
1741  * @param spec[in, out]
1742  *   SFC flow specification to update.
1743  * @param filters_count_for_one_val[in]
1744  *   How many specifications should have the same EtherType value, what is the
1745  *   number of specifications before copying.
1746  * @param error[out]
1747  *   Perform verbose error reporting if not NULL.
1748  */
1749 static int
1750 sfc_flow_set_ethertypes(struct sfc_flow_spec *spec,
1751                         unsigned int filters_count_for_one_val,
1752                         struct rte_flow_error *error)
1753 {
1754         unsigned int i;
1755         static const uint16_t vals[] = {
1756                 EFX_ETHER_TYPE_IPV4, EFX_ETHER_TYPE_IPV6
1757         };
1758
1759         if (filters_count_for_one_val * RTE_DIM(vals) != spec->count) {
1760                 rte_flow_error_set(error, EINVAL,
1761                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1762                         "Number of specifications is incorrect "
1763                         "while copying by Ethertype");
1764                 return -rte_errno;
1765         }
1766
1767         for (i = 0; i < spec->count; i++) {
1768                 spec->filters[i].efs_match_flags |=
1769                         EFX_FILTER_MATCH_ETHER_TYPE;
1770
1771                 /*
1772                  * The check above ensures that
1773                  * filters_count_for_one_val is not 0
1774                  */
1775                 spec->filters[i].efs_ether_type =
1776                         vals[i / filters_count_for_one_val];
1777         }
1778
1779         return 0;
1780 }
1781
1782 /**
1783  * Set the EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST and
1784  * EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST match flags in the same
1785  * specifications after copying.
1786  *
1787  * @param spec[in, out]
1788  *   SFC flow specification to update.
1789  * @param filters_count_for_one_val[in]
1790  *   How many specifications should have the same match flag, what is the
1791  *   number of specifications before copying.
1792  * @param error[out]
1793  *   Perform verbose error reporting if not NULL.
1794  */
1795 static int
1796 sfc_flow_set_ifrm_unknown_dst_flags(struct sfc_flow_spec *spec,
1797                                     unsigned int filters_count_for_one_val,
1798                                     struct rte_flow_error *error)
1799 {
1800         unsigned int i;
1801         static const efx_filter_match_flags_t vals[] = {
1802                 EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST,
1803                 EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST
1804         };
1805
1806         if (filters_count_for_one_val * RTE_DIM(vals) != spec->count) {
1807                 rte_flow_error_set(error, EINVAL,
1808                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1809                         "Number of specifications is incorrect while copying "
1810                         "by inner frame unknown destination flags");
1811                 return -rte_errno;
1812         }
1813
1814         for (i = 0; i < spec->count; i++) {
1815                 /* The check above ensures that divisor can't be zero here */
1816                 spec->filters[i].efs_match_flags |=
1817                         vals[i / filters_count_for_one_val];
1818         }
1819
1820         return 0;
1821 }
1822
1823 /**
1824  * Check that the following conditions are met:
1825  * - the specification corresponds to a filter for encapsulated traffic
1826  * - the list of supported filters has a filter
1827  *   with EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST flag instead of
1828  *   EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST, since this filter will also
1829  *   be inserted.
1830  *
1831  * @param match[in]
1832  *   The match flags of filter.
1833  * @param spec[in]
1834  *   Specification to be supplemented.
1835  * @param filter[in]
1836  *   SFC filter with list of supported filters.
1837  */
1838 static boolean_t
1839 sfc_flow_check_ifrm_unknown_dst_flags(efx_filter_match_flags_t match,
1840                                       efx_filter_spec_t *spec,
1841                                       struct sfc_filter *filter)
1842 {
1843         unsigned int i;
1844         efx_tunnel_protocol_t encap_type = spec->efs_encap_type;
1845         efx_filter_match_flags_t match_mcast_dst;
1846
1847         if (encap_type == EFX_TUNNEL_PROTOCOL_NONE)
1848                 return B_FALSE;
1849
1850         match_mcast_dst =
1851                 (match & ~EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST) |
1852                 EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
1853         for (i = 0; i < filter->supported_match_num; i++) {
1854                 if (match_mcast_dst == filter->supported_match[i])
1855                         return B_TRUE;
1856         }
1857
1858         return B_FALSE;
1859 }
1860
1861 /*
1862  * Match flags that can be automatically added to filters.
1863  * Selecting the last minimum when searching for the copy flag ensures that the
1864  * EFX_FILTER_MATCH_UNKNOWN_UCAST_DST flag has a higher priority than
1865  * EFX_FILTER_MATCH_ETHER_TYPE. This is because the filter
1866  * EFX_FILTER_MATCH_UNKNOWN_UCAST_DST is at the end of the list of supported
1867  * filters.
1868  */
1869 static const struct sfc_flow_copy_flag sfc_flow_copy_flags[] = {
1870         {
1871                 .flag = EFX_FILTER_MATCH_UNKNOWN_UCAST_DST,
1872                 .vals_count = 2,
1873                 .set_vals = sfc_flow_set_unknown_dst_flags,
1874                 .spec_check = sfc_flow_check_unknown_dst_flags,
1875         },
1876         {
1877                 .flag = EFX_FILTER_MATCH_ETHER_TYPE,
1878                 .vals_count = 2,
1879                 .set_vals = sfc_flow_set_ethertypes,
1880                 .spec_check = NULL,
1881         },
1882         {
1883                 .flag = EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST,
1884                 .vals_count = 2,
1885                 .set_vals = sfc_flow_set_ifrm_unknown_dst_flags,
1886                 .spec_check = sfc_flow_check_ifrm_unknown_dst_flags,
1887         },
1888 };
1889
1890 /* Get item from array sfc_flow_copy_flags */
1891 static const struct sfc_flow_copy_flag *
1892 sfc_flow_get_copy_flag(efx_filter_match_flags_t flag)
1893 {
1894         unsigned int i;
1895
1896         for (i = 0; i < RTE_DIM(sfc_flow_copy_flags); i++) {
1897                 if (sfc_flow_copy_flags[i].flag == flag)
1898                         return &sfc_flow_copy_flags[i];
1899         }
1900
1901         return NULL;
1902 }
1903
1904 /**
1905  * Make copies of the specifications, set match flag and values
1906  * of the field that corresponds to it.
1907  *
1908  * @param spec[in, out]
1909  *   SFC flow specification to update.
1910  * @param flag[in]
1911  *   The match flag to add.
1912  * @param error[out]
1913  *   Perform verbose error reporting if not NULL.
1914  */
1915 static int
1916 sfc_flow_spec_add_match_flag(struct sfc_flow_spec *spec,
1917                              efx_filter_match_flags_t flag,
1918                              struct rte_flow_error *error)
1919 {
1920         unsigned int i;
1921         unsigned int new_filters_count;
1922         unsigned int filters_count_for_one_val;
1923         const struct sfc_flow_copy_flag *copy_flag;
1924         int rc;
1925
1926         copy_flag = sfc_flow_get_copy_flag(flag);
1927         if (copy_flag == NULL) {
1928                 rte_flow_error_set(error, ENOTSUP,
1929                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1930                                    "Unsupported spec field for copying");
1931                 return -rte_errno;
1932         }
1933
1934         new_filters_count = spec->count * copy_flag->vals_count;
1935         if (new_filters_count > SF_FLOW_SPEC_NB_FILTERS_MAX) {
1936                 rte_flow_error_set(error, EINVAL,
1937                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1938                         "Too much EFX specifications in the flow rule");
1939                 return -rte_errno;
1940         }
1941
1942         /* Copy filters specifications */
1943         for (i = spec->count; i < new_filters_count; i++)
1944                 spec->filters[i] = spec->filters[i - spec->count];
1945
1946         filters_count_for_one_val = spec->count;
1947         spec->count = new_filters_count;
1948
1949         rc = copy_flag->set_vals(spec, filters_count_for_one_val, error);
1950         if (rc != 0)
1951                 return rc;
1952
1953         return 0;
1954 }
1955
1956 /**
1957  * Check that the given set of match flags missing in the original filter spec
1958  * could be covered by adding spec copies which specify the corresponding
1959  * flags and packet field values to match.
1960  *
1961  * @param miss_flags[in]
1962  *   Flags that are missing until the supported filter.
1963  * @param spec[in]
1964  *   Specification to be supplemented.
1965  * @param filter[in]
1966  *   SFC filter.
1967  *
1968  * @return
1969  *   Number of specifications after copy or 0, if the flags can not be added.
1970  */
1971 static unsigned int
1972 sfc_flow_check_missing_flags(efx_filter_match_flags_t miss_flags,
1973                              efx_filter_spec_t *spec,
1974                              struct sfc_filter *filter)
1975 {
1976         unsigned int i;
1977         efx_filter_match_flags_t copy_flags = 0;
1978         efx_filter_match_flags_t flag;
1979         efx_filter_match_flags_t match = spec->efs_match_flags | miss_flags;
1980         sfc_flow_spec_check *check;
1981         unsigned int multiplier = 1;
1982
1983         for (i = 0; i < RTE_DIM(sfc_flow_copy_flags); i++) {
1984                 flag = sfc_flow_copy_flags[i].flag;
1985                 check = sfc_flow_copy_flags[i].spec_check;
1986                 if ((flag & miss_flags) == flag) {
1987                         if (check != NULL && (!check(match, spec, filter)))
1988                                 continue;
1989
1990                         copy_flags |= flag;
1991                         multiplier *= sfc_flow_copy_flags[i].vals_count;
1992                 }
1993         }
1994
1995         if (copy_flags == miss_flags)
1996                 return multiplier;
1997
1998         return 0;
1999 }
2000
2001 /**
2002  * Attempt to supplement the specification template to the minimally
2003  * supported set of match flags. To do this, it is necessary to copy
2004  * the specifications, filling them with the values of fields that
2005  * correspond to the missing flags.
2006  * The necessary and sufficient filter is built from the fewest number
2007  * of copies which could be made to cover the minimally required set
2008  * of flags.
2009  *
2010  * @param sa[in]
2011  *   SFC adapter.
2012  * @param spec[in, out]
2013  *   SFC flow specification to update.
2014  * @param error[out]
2015  *   Perform verbose error reporting if not NULL.
2016  */
2017 static int
2018 sfc_flow_spec_filters_complete(struct sfc_adapter *sa,
2019                                struct sfc_flow_spec *spec,
2020                                struct rte_flow_error *error)
2021 {
2022         struct sfc_filter *filter = &sa->filter;
2023         efx_filter_match_flags_t miss_flags;
2024         efx_filter_match_flags_t min_miss_flags = 0;
2025         efx_filter_match_flags_t match;
2026         unsigned int min_multiplier = UINT_MAX;
2027         unsigned int multiplier;
2028         unsigned int i;
2029         int rc;
2030
2031         match = spec->template.efs_match_flags;
2032         for (i = 0; i < filter->supported_match_num; i++) {
2033                 if ((match & filter->supported_match[i]) == match) {
2034                         miss_flags = filter->supported_match[i] & (~match);
2035                         multiplier = sfc_flow_check_missing_flags(miss_flags,
2036                                 &spec->template, filter);
2037                         if (multiplier > 0) {
2038                                 if (multiplier <= min_multiplier) {
2039                                         min_multiplier = multiplier;
2040                                         min_miss_flags = miss_flags;
2041                                 }
2042                         }
2043                 }
2044         }
2045
2046         if (min_multiplier == UINT_MAX) {
2047                 rte_flow_error_set(error, ENOTSUP,
2048                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2049                                    "The flow rule pattern is unsupported");
2050                 return -rte_errno;
2051         }
2052
2053         for (i = 0; i < RTE_DIM(sfc_flow_copy_flags); i++) {
2054                 efx_filter_match_flags_t flag = sfc_flow_copy_flags[i].flag;
2055
2056                 if ((flag & min_miss_flags) == flag) {
2057                         rc = sfc_flow_spec_add_match_flag(spec, flag, error);
2058                         if (rc != 0)
2059                                 return rc;
2060                 }
2061         }
2062
2063         return 0;
2064 }
2065
2066 /**
2067  * Check that set of match flags is referred to by a filter. Filter is
2068  * described by match flags with the ability to add OUTER_VID and INNER_VID
2069  * flags.
2070  *
2071  * @param match_flags[in]
2072  *   Set of match flags.
2073  * @param flags_pattern[in]
2074  *   Pattern of filter match flags.
2075  */
2076 static boolean_t
2077 sfc_flow_is_match_with_vids(efx_filter_match_flags_t match_flags,
2078                             efx_filter_match_flags_t flags_pattern)
2079 {
2080         if ((match_flags & flags_pattern) != flags_pattern)
2081                 return B_FALSE;
2082
2083         switch (match_flags & ~flags_pattern) {
2084         case 0:
2085         case EFX_FILTER_MATCH_OUTER_VID:
2086         case EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_INNER_VID:
2087                 return B_TRUE;
2088         default:
2089                 return B_FALSE;
2090         }
2091 }
2092
2093 /**
2094  * Check whether the spec maps to a hardware filter which is known to be
2095  * ineffective despite being valid.
2096  *
2097  * @param spec[in]
2098  *   SFC flow specification.
2099  */
2100 static boolean_t
2101 sfc_flow_is_match_flags_exception(struct sfc_flow_spec *spec)
2102 {
2103         unsigned int i;
2104         uint16_t ether_type;
2105         uint8_t ip_proto;
2106         efx_filter_match_flags_t match_flags;
2107
2108         for (i = 0; i < spec->count; i++) {
2109                 match_flags = spec->filters[i].efs_match_flags;
2110
2111                 if (sfc_flow_is_match_with_vids(match_flags,
2112                                                 EFX_FILTER_MATCH_ETHER_TYPE) ||
2113                     sfc_flow_is_match_with_vids(match_flags,
2114                                                 EFX_FILTER_MATCH_ETHER_TYPE |
2115                                                 EFX_FILTER_MATCH_LOC_MAC)) {
2116                         ether_type = spec->filters[i].efs_ether_type;
2117                         if (ether_type == EFX_ETHER_TYPE_IPV4 ||
2118                             ether_type == EFX_ETHER_TYPE_IPV6)
2119                                 return B_TRUE;
2120                 } else if (sfc_flow_is_match_with_vids(match_flags,
2121                                 EFX_FILTER_MATCH_ETHER_TYPE |
2122                                 EFX_FILTER_MATCH_IP_PROTO) ||
2123                            sfc_flow_is_match_with_vids(match_flags,
2124                                 EFX_FILTER_MATCH_ETHER_TYPE |
2125                                 EFX_FILTER_MATCH_IP_PROTO |
2126                                 EFX_FILTER_MATCH_LOC_MAC)) {
2127                         ip_proto = spec->filters[i].efs_ip_proto;
2128                         if (ip_proto == EFX_IPPROTO_TCP ||
2129                             ip_proto == EFX_IPPROTO_UDP)
2130                                 return B_TRUE;
2131                 }
2132         }
2133
2134         return B_FALSE;
2135 }
2136
2137 static int
2138 sfc_flow_validate_match_flags(struct sfc_adapter *sa,
2139                               struct rte_flow *flow,
2140                               struct rte_flow_error *error)
2141 {
2142         efx_filter_spec_t *spec_tmpl = &flow->spec.template;
2143         efx_filter_match_flags_t match_flags = spec_tmpl->efs_match_flags;
2144         int rc;
2145
2146         /* Initialize the first filter spec with template */
2147         flow->spec.filters[0] = *spec_tmpl;
2148         flow->spec.count = 1;
2149
2150         if (!sfc_filter_is_match_supported(sa, match_flags)) {
2151                 rc = sfc_flow_spec_filters_complete(sa, &flow->spec, error);
2152                 if (rc != 0)
2153                         return rc;
2154         }
2155
2156         if (sfc_flow_is_match_flags_exception(&flow->spec)) {
2157                 rte_flow_error_set(error, ENOTSUP,
2158                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2159                         "The flow rule pattern is unsupported");
2160                 return -rte_errno;
2161         }
2162
2163         return 0;
2164 }
2165
2166 static int
2167 sfc_flow_parse(struct rte_eth_dev *dev,
2168                const struct rte_flow_attr *attr,
2169                const struct rte_flow_item pattern[],
2170                const struct rte_flow_action actions[],
2171                struct rte_flow *flow,
2172                struct rte_flow_error *error)
2173 {
2174         struct sfc_adapter *sa = dev->data->dev_private;
2175         int rc;
2176
2177         rc = sfc_flow_parse_attr(attr, flow, error);
2178         if (rc != 0)
2179                 goto fail_bad_value;
2180
2181         rc = sfc_flow_parse_pattern(pattern, flow, error);
2182         if (rc != 0)
2183                 goto fail_bad_value;
2184
2185         rc = sfc_flow_parse_actions(sa, actions, flow, error);
2186         if (rc != 0)
2187                 goto fail_bad_value;
2188
2189         rc = sfc_flow_validate_match_flags(sa, flow, error);
2190         if (rc != 0)
2191                 goto fail_bad_value;
2192
2193         return 0;
2194
2195 fail_bad_value:
2196         return rc;
2197 }
2198
2199 static int
2200 sfc_flow_validate(struct rte_eth_dev *dev,
2201                   const struct rte_flow_attr *attr,
2202                   const struct rte_flow_item pattern[],
2203                   const struct rte_flow_action actions[],
2204                   struct rte_flow_error *error)
2205 {
2206         struct rte_flow flow;
2207
2208         memset(&flow, 0, sizeof(flow));
2209
2210         return sfc_flow_parse(dev, attr, pattern, actions, &flow, error);
2211 }
2212
2213 static struct rte_flow *
2214 sfc_flow_create(struct rte_eth_dev *dev,
2215                 const struct rte_flow_attr *attr,
2216                 const struct rte_flow_item pattern[],
2217                 const struct rte_flow_action actions[],
2218                 struct rte_flow_error *error)
2219 {
2220         struct sfc_adapter *sa = dev->data->dev_private;
2221         struct rte_flow *flow = NULL;
2222         int rc;
2223
2224         flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0);
2225         if (flow == NULL) {
2226                 rte_flow_error_set(error, ENOMEM,
2227                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2228                                    "Failed to allocate memory");
2229                 goto fail_no_mem;
2230         }
2231
2232         rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error);
2233         if (rc != 0)
2234                 goto fail_bad_value;
2235
2236         TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries);
2237
2238         sfc_adapter_lock(sa);
2239
2240         if (sa->state == SFC_ADAPTER_STARTED) {
2241                 rc = sfc_flow_filter_insert(sa, flow);
2242                 if (rc != 0) {
2243                         rte_flow_error_set(error, rc,
2244                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2245                                 "Failed to insert filter");
2246                         goto fail_filter_insert;
2247                 }
2248         }
2249
2250         sfc_adapter_unlock(sa);
2251
2252         return flow;
2253
2254 fail_filter_insert:
2255         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
2256
2257 fail_bad_value:
2258         rte_free(flow);
2259         sfc_adapter_unlock(sa);
2260
2261 fail_no_mem:
2262         return NULL;
2263 }
2264
2265 static int
2266 sfc_flow_remove(struct sfc_adapter *sa,
2267                 struct rte_flow *flow,
2268                 struct rte_flow_error *error)
2269 {
2270         int rc = 0;
2271
2272         SFC_ASSERT(sfc_adapter_is_locked(sa));
2273
2274         if (sa->state == SFC_ADAPTER_STARTED) {
2275                 rc = sfc_flow_filter_remove(sa, flow);
2276                 if (rc != 0)
2277                         rte_flow_error_set(error, rc,
2278                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2279                                 "Failed to destroy flow rule");
2280         }
2281
2282         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
2283         rte_free(flow);
2284
2285         return rc;
2286 }
2287
2288 static int
2289 sfc_flow_destroy(struct rte_eth_dev *dev,
2290                  struct rte_flow *flow,
2291                  struct rte_flow_error *error)
2292 {
2293         struct sfc_adapter *sa = dev->data->dev_private;
2294         struct rte_flow *flow_ptr;
2295         int rc = EINVAL;
2296
2297         sfc_adapter_lock(sa);
2298
2299         TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) {
2300                 if (flow_ptr == flow)
2301                         rc = 0;
2302         }
2303         if (rc != 0) {
2304                 rte_flow_error_set(error, rc,
2305                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2306                                    "Failed to find flow rule to destroy");
2307                 goto fail_bad_value;
2308         }
2309
2310         rc = sfc_flow_remove(sa, flow, error);
2311
2312 fail_bad_value:
2313         sfc_adapter_unlock(sa);
2314
2315         return -rc;
2316 }
2317
2318 static int
2319 sfc_flow_flush(struct rte_eth_dev *dev,
2320                struct rte_flow_error *error)
2321 {
2322         struct sfc_adapter *sa = dev->data->dev_private;
2323         struct rte_flow *flow;
2324         int rc = 0;
2325         int ret = 0;
2326
2327         sfc_adapter_lock(sa);
2328
2329         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
2330                 rc = sfc_flow_remove(sa, flow, error);
2331                 if (rc != 0)
2332                         ret = rc;
2333         }
2334
2335         sfc_adapter_unlock(sa);
2336
2337         return -ret;
2338 }
2339
2340 static int
2341 sfc_flow_isolate(struct rte_eth_dev *dev, int enable,
2342                  struct rte_flow_error *error)
2343 {
2344         struct sfc_adapter *sa = dev->data->dev_private;
2345         struct sfc_port *port = &sa->port;
2346         int ret = 0;
2347
2348         sfc_adapter_lock(sa);
2349         if (sa->state != SFC_ADAPTER_INITIALIZED) {
2350                 rte_flow_error_set(error, EBUSY,
2351                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2352                                    NULL, "please close the port first");
2353                 ret = -rte_errno;
2354         } else {
2355                 port->isolated = (enable) ? B_TRUE : B_FALSE;
2356         }
2357         sfc_adapter_unlock(sa);
2358
2359         return ret;
2360 }
2361
2362 const struct rte_flow_ops sfc_flow_ops = {
2363         .validate = sfc_flow_validate,
2364         .create = sfc_flow_create,
2365         .destroy = sfc_flow_destroy,
2366         .flush = sfc_flow_flush,
2367         .query = NULL,
2368         .isolate = sfc_flow_isolate,
2369 };
2370
2371 void
2372 sfc_flow_init(struct sfc_adapter *sa)
2373 {
2374         SFC_ASSERT(sfc_adapter_is_locked(sa));
2375
2376         TAILQ_INIT(&sa->filter.flow_list);
2377 }
2378
2379 void
2380 sfc_flow_fini(struct sfc_adapter *sa)
2381 {
2382         struct rte_flow *flow;
2383
2384         SFC_ASSERT(sfc_adapter_is_locked(sa));
2385
2386         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
2387                 TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
2388                 rte_free(flow);
2389         }
2390 }
2391
2392 void
2393 sfc_flow_stop(struct sfc_adapter *sa)
2394 {
2395         struct rte_flow *flow;
2396
2397         SFC_ASSERT(sfc_adapter_is_locked(sa));
2398
2399         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries)
2400                 sfc_flow_filter_remove(sa, flow);
2401 }
2402
2403 int
2404 sfc_flow_start(struct sfc_adapter *sa)
2405 {
2406         struct rte_flow *flow;
2407         int rc = 0;
2408
2409         sfc_log_init(sa, "entry");
2410
2411         SFC_ASSERT(sfc_adapter_is_locked(sa));
2412
2413         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) {
2414                 rc = sfc_flow_filter_insert(sa, flow);
2415                 if (rc != 0)
2416                         goto fail_bad_flow;
2417         }
2418
2419         sfc_log_init(sa, "done");
2420
2421 fail_bad_flow:
2422         return rc;
2423 }