drivers/net/sfc/sfc_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2017-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * This software was jointly developed between OKTET Labs (under contract
   7  * for Solarflare) and Solarflare Communications, Inc.
   8  */
   9
  10 #include <rte_tailq.h>
  11 #include <rte_common.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_eth_ctrl.h>
  14 #include <rte_ether.h>
  15 #include <rte_flow.h>
  16 #include <rte_flow_driver.h>
  17
  18 #include "efx.h"
  19
  20 #include "sfc.h"
  21 #include "sfc_rx.h"
  22 #include "sfc_filter.h"
  23 #include "sfc_flow.h"
  24 #include "sfc_log.h"
  25
  26 /*
  27  * At now flow API is implemented in such a manner that each
  28  * flow rule is converted to one or more hardware filters.
  29  * All elements of flow rule (attributes, pattern items, actions)
  30  * correspond to one or more fields in the efx_filter_spec_s structure
  31  * that is responsible for the hardware filter.
  32  * If some required field is unset in the flow rule, then a handful
  33  * of filter copies will be created to cover all possible values
  34  * of such a field.
  35  */
  36
  37 enum sfc_flow_item_layers {
  38         SFC_FLOW_ITEM_ANY_LAYER,
  39         SFC_FLOW_ITEM_START_LAYER,
  40         SFC_FLOW_ITEM_L2,
  41         SFC_FLOW_ITEM_L3,
  42         SFC_FLOW_ITEM_L4,
  43 };
  44
  45 typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item,
  46                                   efx_filter_spec_t *spec,
  47                                   struct rte_flow_error *error);
  48
  49 struct sfc_flow_item {
  50         enum rte_flow_item_type type;           /* Type of item */
  51         enum sfc_flow_item_layers layer;        /* Layer of item */
  52         enum sfc_flow_item_layers prev_layer;   /* Previous layer of item */
  53         sfc_flow_item_parse *parse;             /* Parsing function */
  54 };
  55
  56 static sfc_flow_item_parse sfc_flow_parse_void;
  57 static sfc_flow_item_parse sfc_flow_parse_eth;
  58 static sfc_flow_item_parse sfc_flow_parse_vlan;
  59 static sfc_flow_item_parse sfc_flow_parse_ipv4;
  60 static sfc_flow_item_parse sfc_flow_parse_ipv6;
  61 static sfc_flow_item_parse sfc_flow_parse_tcp;
  62 static sfc_flow_item_parse sfc_flow_parse_udp;
  63 static sfc_flow_item_parse sfc_flow_parse_vxlan;
  64 static sfc_flow_item_parse sfc_flow_parse_geneve;
  65 static sfc_flow_item_parse sfc_flow_parse_nvgre;
  66
  67 static boolean_t
  68 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
  69 {
  70         uint8_t sum = 0;
  71         unsigned int i;
  72
  73         for (i = 0; i < size; i++)
  74                 sum |= buf[i];
  75
  76         return (sum == 0) ? B_TRUE : B_FALSE;
  77 }
  78
  79 /*
  80  * Validate item and prepare structures spec and mask for parsing
  81  */
  82 static int
  83 sfc_flow_parse_init(const struct rte_flow_item *item,
  84                     const void **spec_ptr,
  85                     const void **mask_ptr,
  86                     const void *supp_mask,
  87                     const void *def_mask,
  88                     unsigned int size,
  89                     struct rte_flow_error *error)
  90 {
  91         const uint8_t *spec;
  92         const uint8_t *mask;
  93         const uint8_t *last;
  94         uint8_t match;
  95         uint8_t supp;
  96         unsigned int i;
  97
  98         if (item == NULL) {
  99                 rte_flow_error_set(error, EINVAL,
 100                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 101                                    "NULL item");
 102                 return -rte_errno;
 103         }
 104
 105         if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) {
 106                 rte_flow_error_set(error, EINVAL,
 107                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 108                                    "Mask or last is set without spec");
 109                 return -rte_errno;
 110         }
 111
 112         /*
 113          * If "mask" is not set, default mask is used,
 114          * but if default mask is NULL, "mask" should be set
 115          */
 116         if (item->mask == NULL) {
 117                 if (def_mask == NULL) {
 118                         rte_flow_error_set(error, EINVAL,
 119                                 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 120                                 "Mask should be specified");
 121                         return -rte_errno;
 122                 }
 123
 124                 mask = def_mask;
 125         } else {
 126                 mask = item->mask;
 127         }
 128
 129         spec = item->spec;
 130         last = item->last;
 131
 132         if (spec == NULL)
 133                 goto exit;
 134
 135         /*
 136          * If field values in "last" are either 0 or equal to the corresponding
 137          * values in "spec" then they are ignored
 138          */
 139         if (last != NULL &&
 140             !sfc_flow_is_zero(last, size) &&
 141             memcmp(last, spec, size) != 0) {
 142                 rte_flow_error_set(error, ENOTSUP,
 143                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 144                                    "Ranging is not supported");
 145                 return -rte_errno;
 146         }
 147
 148         if (supp_mask == NULL) {
 149                 rte_flow_error_set(error, EINVAL,
 150                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 151                         "Supported mask for item should be specified");
 152                 return -rte_errno;
 153         }
 154
 155         /* Check that mask and spec not asks for more match than supp_mask */
 156         for (i = 0; i < size; i++) {
 157                 match = spec[i] | mask[i];
 158                 supp = ((const uint8_t *)supp_mask)[i];
 159
 160                 if ((match | supp) != supp) {
 161                         rte_flow_error_set(error, ENOTSUP,
 162                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 163                                            "Item's field is not supported");
 164                         return -rte_errno;
 165                 }
 166         }
 167
 168 exit:
 169         *spec_ptr = spec;
 170         *mask_ptr = mask;
 171         return 0;
 172 }
 173
 174 /*
 175  * Protocol parsers.
 176  * Masking is not supported, so masks in items should be either
 177  * full or empty (zeroed) and set only for supported fields which
 178  * are specified in the supp_mask.
 179  */
 180
 181 static int
 182 sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item,
 183                     __rte_unused efx_filter_spec_t *efx_spec,
 184                     __rte_unused struct rte_flow_error *error)
 185 {
 186         return 0;
 187 }
 188
 189 /**
 190  * Convert Ethernet item to EFX filter specification.
 191  *
 192  * @param item[in]
 193  *   Item specification. Outer frame specification may only comprise
 194  *   source/destination addresses and Ethertype field.
 195  *   Inner frame specification may contain destination address only.
 196  *   There is support for individual/group mask as well as for empty and full.
 197  *   If the mask is NULL, default mask will be used. Ranging is not supported.
 198  * @param efx_spec[in, out]
 199  *   EFX filter specification to update.
 200  * @param[out] error
 201  *   Perform verbose error reporting if not NULL.
 202  */
 203 static int
 204 sfc_flow_parse_eth(const struct rte_flow_item *item,
 205                    efx_filter_spec_t *efx_spec,
 206                    struct rte_flow_error *error)
 207 {
 208         int rc;
 209         const struct rte_flow_item_eth *spec = NULL;
 210         const struct rte_flow_item_eth *mask = NULL;
 211         const struct rte_flow_item_eth supp_mask = {
 212                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 213                 .src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 214                 .type = 0xffff,
 215         };
 216         const struct rte_flow_item_eth ifrm_supp_mask = {
 217                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 218         };
 219         const uint8_t ig_mask[EFX_MAC_ADDR_LEN] = {
 220                 0x01, 0x00, 0x00, 0x00, 0x00, 0x00
 221         };
 222         const struct rte_flow_item_eth *supp_mask_p;
 223         const struct rte_flow_item_eth *def_mask_p;
 224         uint8_t *loc_mac = NULL;
 225         boolean_t is_ifrm = (efx_spec->efs_encap_type !=
 226                 EFX_TUNNEL_PROTOCOL_NONE);
 227
 228         if (is_ifrm) {
 229                 supp_mask_p = &ifrm_supp_mask;
 230                 def_mask_p = &ifrm_supp_mask;
 231                 loc_mac = efx_spec->efs_ifrm_loc_mac;
 232         } else {
 233                 supp_mask_p = &supp_mask;
 234                 def_mask_p = &rte_flow_item_eth_mask;
 235                 loc_mac = efx_spec->efs_loc_mac;
 236         }
 237
 238         rc = sfc_flow_parse_init(item,
 239                                  (const void **)&spec,
 240                                  (const void **)&mask,
 241                                  supp_mask_p, def_mask_p,
 242                                  sizeof(struct rte_flow_item_eth),
 243                                  error);
 244         if (rc != 0)
 245                 return rc;
 246
 247         /*
 248          * If "spec" is not set, could be any Ethernet, but for the inner frame
 249          * type of destination MAC must be set
 250          */
 251         if (spec == NULL) {
 252                 if (is_ifrm)
 253                         goto fail_bad_ifrm_dst_mac;
 254                 else
 255                         return 0;
 256         }
 257
 258         if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) {
 259                 efx_spec->efs_match_flags |= is_ifrm ?
 260                         EFX_FILTER_MATCH_IFRM_LOC_MAC :
 261                         EFX_FILTER_MATCH_LOC_MAC;
 262                 rte_memcpy(loc_mac, spec->dst.addr_bytes,
 263                            EFX_MAC_ADDR_LEN);
 264         } else if (memcmp(mask->dst.addr_bytes, ig_mask,
 265                           EFX_MAC_ADDR_LEN) == 0) {
 266                 if (is_unicast_ether_addr(&spec->dst))
 267                         efx_spec->efs_match_flags |= is_ifrm ?
 268                                 EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST :
 269                                 EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 270                 else
 271                         efx_spec->efs_match_flags |= is_ifrm ?
 272                                 EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST :
 273                                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 274         } else if (!is_zero_ether_addr(&mask->dst)) {
 275                 goto fail_bad_mask;
 276         } else if (is_ifrm) {
 277                 goto fail_bad_ifrm_dst_mac;
 278         }
 279
 280         /*
 281          * ifrm_supp_mask ensures that the source address and
 282          * ethertype masks are equal to zero in inner frame,
 283          * so these fields are filled in only for the outer frame
 284          */
 285         if (is_same_ether_addr(&mask->src, &supp_mask.src)) {
 286                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC;
 287                 rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes,
 288                            EFX_MAC_ADDR_LEN);
 289         } else if (!is_zero_ether_addr(&mask->src)) {
 290                 goto fail_bad_mask;
 291         }
 292
 293         /*
 294          * Ether type is in big-endian byte order in item and
 295          * in little-endian in efx_spec, so byte swap is used
 296          */
 297         if (mask->type == supp_mask.type) {
 298                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 299                 efx_spec->efs_ether_type = rte_bswap16(spec->type);
 300         } else if (mask->type != 0) {
 301                 goto fail_bad_mask;
 302         }
 303
 304         return 0;
 305
 306 fail_bad_mask:
 307         rte_flow_error_set(error, EINVAL,
 308                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 309                            "Bad mask in the ETH pattern item");
 310         return -rte_errno;
 311
 312 fail_bad_ifrm_dst_mac:
 313         rte_flow_error_set(error, EINVAL,
 314                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 315                            "Type of destination MAC address in inner frame "
 316                            "must be set");
 317         return -rte_errno;
 318 }
 319
 320 /**
 321  * Convert VLAN item to EFX filter specification.
 322  *
 323  * @param item[in]
 324  *   Item specification. Only VID field is supported.
 325  *   The mask can not be NULL. Ranging is not supported.
 326  * @param efx_spec[in, out]
 327  *   EFX filter specification to update.
 328  * @param[out] error
 329  *   Perform verbose error reporting if not NULL.
 330  */
 331 static int
 332 sfc_flow_parse_vlan(const struct rte_flow_item *item,
 333                     efx_filter_spec_t *efx_spec,
 334                     struct rte_flow_error *error)
 335 {
 336         int rc;
 337         uint16_t vid;
 338         const struct rte_flow_item_vlan *spec = NULL;
 339         const struct rte_flow_item_vlan *mask = NULL;
 340         const struct rte_flow_item_vlan supp_mask = {
 341                 .tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX),
 342         };
 343
 344         rc = sfc_flow_parse_init(item,
 345                                  (const void **)&spec,
 346                                  (const void **)&mask,
 347                                  &supp_mask,
 348                                  NULL,
 349                                  sizeof(struct rte_flow_item_vlan),
 350                                  error);
 351         if (rc != 0)
 352                 return rc;
 353
 354         /*
 355          * VID is in big-endian byte order in item and
 356          * in little-endian in efx_spec, so byte swap is used.
 357          * If two VLAN items are included, the first matches
 358          * the outer tag and the next matches the inner tag.
 359          */
 360         if (mask->tci == supp_mask.tci) {
 361                 vid = rte_bswap16(spec->tci);
 362
 363                 if (!(efx_spec->efs_match_flags &
 364                       EFX_FILTER_MATCH_OUTER_VID)) {
 365                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 366                         efx_spec->efs_outer_vid = vid;
 367                 } else if (!(efx_spec->efs_match_flags &
 368                              EFX_FILTER_MATCH_INNER_VID)) {
 369                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID;
 370                         efx_spec->efs_inner_vid = vid;
 371                 } else {
 372                         rte_flow_error_set(error, EINVAL,
 373                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 374                                            "More than two VLAN items");
 375                         return -rte_errno;
 376                 }
 377         } else {
 378                 rte_flow_error_set(error, EINVAL,
 379                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 380                                    "VLAN ID in TCI match is required");
 381                 return -rte_errno;
 382         }
 383
 384         return 0;
 385 }
 386
 387 /**
 388  * Convert IPv4 item to EFX filter specification.
 389  *
 390  * @param item[in]
 391  *   Item specification. Only source and destination addresses and
 392  *   protocol fields are supported. If the mask is NULL, default
 393  *   mask will be used. Ranging is not supported.
 394  * @param efx_spec[in, out]
 395  *   EFX filter specification to update.
 396  * @param[out] error
 397  *   Perform verbose error reporting if not NULL.
 398  */
 399 static int
 400 sfc_flow_parse_ipv4(const struct rte_flow_item *item,
 401                     efx_filter_spec_t *efx_spec,
 402                     struct rte_flow_error *error)
 403 {
 404         int rc;
 405         const struct rte_flow_item_ipv4 *spec = NULL;
 406         const struct rte_flow_item_ipv4 *mask = NULL;
 407         const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4);
 408         const struct rte_flow_item_ipv4 supp_mask = {
 409                 .hdr = {
 410                         .src_addr = 0xffffffff,
 411                         .dst_addr = 0xffffffff,
 412                         .next_proto_id = 0xff,
 413                 }
 414         };
 415
 416         rc = sfc_flow_parse_init(item,
 417                                  (const void **)&spec,
 418                                  (const void **)&mask,
 419                                  &supp_mask,
 420                                  &rte_flow_item_ipv4_mask,
 421                                  sizeof(struct rte_flow_item_ipv4),
 422                                  error);
 423         if (rc != 0)
 424                 return rc;
 425
 426         /*
 427          * Filtering by IPv4 source and destination addresses requires
 428          * the appropriate ETHER_TYPE in hardware filters
 429          */
 430         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 431                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 432                 efx_spec->efs_ether_type = ether_type_ipv4;
 433         } else if (efx_spec->efs_ether_type != ether_type_ipv4) {
 434                 rte_flow_error_set(error, EINVAL,
 435                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 436                         "Ethertype in pattern with IPV4 item should be appropriate");
 437                 return -rte_errno;
 438         }
 439
 440         if (spec == NULL)
 441                 return 0;
 442
 443         /*
 444          * IPv4 addresses are in big-endian byte order in item and in
 445          * efx_spec
 446          */
 447         if (mask->hdr.src_addr == supp_mask.hdr.src_addr) {
 448                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 449                 efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr;
 450         } else if (mask->hdr.src_addr != 0) {
 451                 goto fail_bad_mask;
 452         }
 453
 454         if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) {
 455                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 456                 efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr;
 457         } else if (mask->hdr.dst_addr != 0) {
 458                 goto fail_bad_mask;
 459         }
 460
 461         if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) {
 462                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 463                 efx_spec->efs_ip_proto = spec->hdr.next_proto_id;
 464         } else if (mask->hdr.next_proto_id != 0) {
 465                 goto fail_bad_mask;
 466         }
 467
 468         return 0;
 469
 470 fail_bad_mask:
 471         rte_flow_error_set(error, EINVAL,
 472                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 473                            "Bad mask in the IPV4 pattern item");
 474         return -rte_errno;
 475 }
 476
 477 /**
 478  * Convert IPv6 item to EFX filter specification.
 479  *
 480  * @param item[in]
 481  *   Item specification. Only source and destination addresses and
 482  *   next header fields are supported. If the mask is NULL, default
 483  *   mask will be used. Ranging is not supported.
 484  * @param efx_spec[in, out]
 485  *   EFX filter specification to update.
 486  * @param[out] error
 487  *   Perform verbose error reporting if not NULL.
 488  */
 489 static int
 490 sfc_flow_parse_ipv6(const struct rte_flow_item *item,
 491                     efx_filter_spec_t *efx_spec,
 492                     struct rte_flow_error *error)
 493 {
 494         int rc;
 495         const struct rte_flow_item_ipv6 *spec = NULL;
 496         const struct rte_flow_item_ipv6 *mask = NULL;
 497         const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6);
 498         const struct rte_flow_item_ipv6 supp_mask = {
 499                 .hdr = {
 500                         .src_addr = { 0xff, 0xff, 0xff, 0xff,
 501                                       0xff, 0xff, 0xff, 0xff,
 502                                       0xff, 0xff, 0xff, 0xff,
 503                                       0xff, 0xff, 0xff, 0xff },
 504                         .dst_addr = { 0xff, 0xff, 0xff, 0xff,
 505                                       0xff, 0xff, 0xff, 0xff,
 506                                       0xff, 0xff, 0xff, 0xff,
 507                                       0xff, 0xff, 0xff, 0xff },
 508                         .proto = 0xff,
 509                 }
 510         };
 511
 512         rc = sfc_flow_parse_init(item,
 513                                  (const void **)&spec,
 514                                  (const void **)&mask,
 515                                  &supp_mask,
 516                                  &rte_flow_item_ipv6_mask,
 517                                  sizeof(struct rte_flow_item_ipv6),
 518                                  error);
 519         if (rc != 0)
 520                 return rc;
 521
 522         /*
 523          * Filtering by IPv6 source and destination addresses requires
 524          * the appropriate ETHER_TYPE in hardware filters
 525          */
 526         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 527                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 528                 efx_spec->efs_ether_type = ether_type_ipv6;
 529         } else if (efx_spec->efs_ether_type != ether_type_ipv6) {
 530                 rte_flow_error_set(error, EINVAL,
 531                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 532                         "Ethertype in pattern with IPV6 item should be appropriate");
 533                 return -rte_errno;
 534         }
 535
 536         if (spec == NULL)
 537                 return 0;
 538
 539         /*
 540          * IPv6 addresses are in big-endian byte order in item and in
 541          * efx_spec
 542          */
 543         if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr,
 544                    sizeof(mask->hdr.src_addr)) == 0) {
 545                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 546
 547                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) !=
 548                                  sizeof(spec->hdr.src_addr));
 549                 rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr,
 550                            sizeof(efx_spec->efs_rem_host));
 551         } else if (!sfc_flow_is_zero(mask->hdr.src_addr,
 552                                      sizeof(mask->hdr.src_addr))) {
 553                 goto fail_bad_mask;
 554         }
 555
 556         if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr,
 557                    sizeof(mask->hdr.dst_addr)) == 0) {
 558                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 559
 560                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) !=
 561                                  sizeof(spec->hdr.dst_addr));
 562                 rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr,
 563                            sizeof(efx_spec->efs_loc_host));
 564         } else if (!sfc_flow_is_zero(mask->hdr.dst_addr,
 565                                      sizeof(mask->hdr.dst_addr))) {
 566                 goto fail_bad_mask;
 567         }
 568
 569         if (mask->hdr.proto == supp_mask.hdr.proto) {
 570                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 571                 efx_spec->efs_ip_proto = spec->hdr.proto;
 572         } else if (mask->hdr.proto != 0) {
 573                 goto fail_bad_mask;
 574         }
 575
 576         return 0;
 577
 578 fail_bad_mask:
 579         rte_flow_error_set(error, EINVAL,
 580                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 581                            "Bad mask in the IPV6 pattern item");
 582         return -rte_errno;
 583 }
 584
 585 /**
 586  * Convert TCP item to EFX filter specification.
 587  *
 588  * @param item[in]
 589  *   Item specification. Only source and destination ports fields
 590  *   are supported. If the mask is NULL, default mask will be used.
 591  *   Ranging is not supported.
 592  * @param efx_spec[in, out]
 593  *   EFX filter specification to update.
 594  * @param[out] error
 595  *   Perform verbose error reporting if not NULL.
 596  */
 597 static int
 598 sfc_flow_parse_tcp(const struct rte_flow_item *item,
 599                    efx_filter_spec_t *efx_spec,
 600                    struct rte_flow_error *error)
 601 {
 602         int rc;
 603         const struct rte_flow_item_tcp *spec = NULL;
 604         const struct rte_flow_item_tcp *mask = NULL;
 605         const struct rte_flow_item_tcp supp_mask = {
 606                 .hdr = {
 607                         .src_port = 0xffff,
 608                         .dst_port = 0xffff,
 609                 }
 610         };
 611
 612         rc = sfc_flow_parse_init(item,
 613                                  (const void **)&spec,
 614                                  (const void **)&mask,
 615                                  &supp_mask,
 616                                  &rte_flow_item_tcp_mask,
 617                                  sizeof(struct rte_flow_item_tcp),
 618                                  error);
 619         if (rc != 0)
 620                 return rc;
 621
 622         /*
 623          * Filtering by TCP source and destination ports requires
 624          * the appropriate IP_PROTO in hardware filters
 625          */
 626         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 627                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 628                 efx_spec->efs_ip_proto = EFX_IPPROTO_TCP;
 629         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) {
 630                 rte_flow_error_set(error, EINVAL,
 631                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 632                         "IP proto in pattern with TCP item should be appropriate");
 633                 return -rte_errno;
 634         }
 635
 636         if (spec == NULL)
 637                 return 0;
 638
 639         /*
 640          * Source and destination ports are in big-endian byte order in item and
 641          * in little-endian in efx_spec, so byte swap is used
 642          */
 643         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 644                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 645                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 646         } else if (mask->hdr.src_port != 0) {
 647                 goto fail_bad_mask;
 648         }
 649
 650         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 651                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 652                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 653         } else if (mask->hdr.dst_port != 0) {
 654                 goto fail_bad_mask;
 655         }
 656
 657         return 0;
 658
 659 fail_bad_mask:
 660         rte_flow_error_set(error, EINVAL,
 661                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 662                            "Bad mask in the TCP pattern item");
 663         return -rte_errno;
 664 }
 665
 666 /**
 667  * Convert UDP item to EFX filter specification.
 668  *
 669  * @param item[in]
 670  *   Item specification. Only source and destination ports fields
 671  *   are supported. If the mask is NULL, default mask will be used.
 672  *   Ranging is not supported.
 673  * @param efx_spec[in, out]
 674  *   EFX filter specification to update.
 675  * @param[out] error
 676  *   Perform verbose error reporting if not NULL.
 677  */
 678 static int
 679 sfc_flow_parse_udp(const struct rte_flow_item *item,
 680                    efx_filter_spec_t *efx_spec,
 681                    struct rte_flow_error *error)
 682 {
 683         int rc;
 684         const struct rte_flow_item_udp *spec = NULL;
 685         const struct rte_flow_item_udp *mask = NULL;
 686         const struct rte_flow_item_udp supp_mask = {
 687                 .hdr = {
 688                         .src_port = 0xffff,
 689                         .dst_port = 0xffff,
 690                 }
 691         };
 692
 693         rc = sfc_flow_parse_init(item,
 694                                  (const void **)&spec,
 695                                  (const void **)&mask,
 696                                  &supp_mask,
 697                                  &rte_flow_item_udp_mask,
 698                                  sizeof(struct rte_flow_item_udp),
 699                                  error);
 700         if (rc != 0)
 701                 return rc;
 702
 703         /*
 704          * Filtering by UDP source and destination ports requires
 705          * the appropriate IP_PROTO in hardware filters
 706          */
 707         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 708                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 709                 efx_spec->efs_ip_proto = EFX_IPPROTO_UDP;
 710         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) {
 711                 rte_flow_error_set(error, EINVAL,
 712                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 713                         "IP proto in pattern with UDP item should be appropriate");
 714                 return -rte_errno;
 715         }
 716
 717         if (spec == NULL)
 718                 return 0;
 719
 720         /*
 721          * Source and destination ports are in big-endian byte order in item and
 722          * in little-endian in efx_spec, so byte swap is used
 723          */
 724         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 725                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 726                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 727         } else if (mask->hdr.src_port != 0) {
 728                 goto fail_bad_mask;
 729         }
 730
 731         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 732                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 733                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 734         } else if (mask->hdr.dst_port != 0) {
 735                 goto fail_bad_mask;
 736         }
 737
 738         return 0;
 739
 740 fail_bad_mask:
 741         rte_flow_error_set(error, EINVAL,
 742                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 743                            "Bad mask in the UDP pattern item");
 744         return -rte_errno;
 745 }
 746
 747 /*
 748  * Filters for encapsulated packets match based on the EtherType and IP
 749  * protocol in the outer frame.
 750  */
 751 static int
 752 sfc_flow_set_match_flags_for_encap_pkts(const struct rte_flow_item *item,
 753                                         efx_filter_spec_t *efx_spec,
 754                                         uint8_t ip_proto,
 755                                         struct rte_flow_error *error)
 756 {
 757         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 758                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 759                 efx_spec->efs_ip_proto = ip_proto;
 760         } else if (efx_spec->efs_ip_proto != ip_proto) {
 761                 switch (ip_proto) {
 762                 case EFX_IPPROTO_UDP:
 763                         rte_flow_error_set(error, EINVAL,
 764                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 765                                 "Outer IP header protocol must be UDP "
 766                                 "in VxLAN/GENEVE pattern");
 767                         return -rte_errno;
 768
 769                 case EFX_IPPROTO_GRE:
 770                         rte_flow_error_set(error, EINVAL,
 771                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 772                                 "Outer IP header protocol must be GRE "
 773                                 "in NVGRE pattern");
 774                         return -rte_errno;
 775
 776                 default:
 777                         rte_flow_error_set(error, EINVAL,
 778                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 779                                 "Only VxLAN/GENEVE/NVGRE tunneling patterns "
 780                                 "are supported");
 781                         return -rte_errno;
 782                 }
 783         }
 784
 785         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 786                 rte_flow_error_set(error, EINVAL,
 787                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 788                         "Outer frame EtherType in pattern with tunneling "
 789                         "must be set");
 790                 return -rte_errno;
 791         } else if (efx_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4 &&
 792                    efx_spec->efs_ether_type != EFX_ETHER_TYPE_IPV6) {
 793                 rte_flow_error_set(error, EINVAL,
 794                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 795                         "Outer frame EtherType in pattern with tunneling "
 796                         "must be IPv4 or IPv6");
 797                 return -rte_errno;
 798         }
 799
 800         return 0;
 801 }
 802
 803 static int
 804 sfc_flow_set_efx_spec_vni_or_vsid(efx_filter_spec_t *efx_spec,
 805                                   const uint8_t *vni_or_vsid_val,
 806                                   const uint8_t *vni_or_vsid_mask,
 807                                   const struct rte_flow_item *item,
 808                                   struct rte_flow_error *error)
 809 {
 810         const uint8_t vni_or_vsid_full_mask[EFX_VNI_OR_VSID_LEN] = {
 811                 0xff, 0xff, 0xff
 812         };
 813
 814         if (memcmp(vni_or_vsid_mask, vni_or_vsid_full_mask,
 815                    EFX_VNI_OR_VSID_LEN) == 0) {
 816                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 817                 rte_memcpy(efx_spec->efs_vni_or_vsid, vni_or_vsid_val,
 818                            EFX_VNI_OR_VSID_LEN);
 819         } else if (!sfc_flow_is_zero(vni_or_vsid_mask, EFX_VNI_OR_VSID_LEN)) {
 820                 rte_flow_error_set(error, EINVAL,
 821                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 822                                    "Unsupported VNI/VSID mask");
 823                 return -rte_errno;
 824         }
 825
 826         return 0;
 827 }
 828
 829 /**
 830  * Convert VXLAN item to EFX filter specification.
 831  *
 832  * @param item[in]
 833  *   Item specification. Only VXLAN network identifier field is supported.
 834  *   If the mask is NULL, default mask will be used.
 835  *   Ranging is not supported.
 836  * @param efx_spec[in, out]
 837  *   EFX filter specification to update.
 838  * @param[out] error
 839  *   Perform verbose error reporting if not NULL.
 840  */
 841 static int
 842 sfc_flow_parse_vxlan(const struct rte_flow_item *item,
 843                      efx_filter_spec_t *efx_spec,
 844                      struct rte_flow_error *error)
 845 {
 846         int rc;
 847         const struct rte_flow_item_vxlan *spec = NULL;
 848         const struct rte_flow_item_vxlan *mask = NULL;
 849         const struct rte_flow_item_vxlan supp_mask = {
 850                 .vni = { 0xff, 0xff, 0xff }
 851         };
 852
 853         rc = sfc_flow_parse_init(item,
 854                                  (const void **)&spec,
 855                                  (const void **)&mask,
 856                                  &supp_mask,
 857                                  &rte_flow_item_vxlan_mask,
 858                                  sizeof(struct rte_flow_item_vxlan),
 859                                  error);
 860         if (rc != 0)
 861                 return rc;
 862
 863         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
 864                                                      EFX_IPPROTO_UDP, error);
 865         if (rc != 0)
 866                 return rc;
 867
 868         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_VXLAN;
 869         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 870
 871         if (spec == NULL)
 872                 return 0;
 873
 874         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->vni,
 875                                                mask->vni, item, error);
 876
 877         return rc;
 878 }
 879
 880 /**
 881  * Convert GENEVE item to EFX filter specification.
 882  *
 883  * @param item[in]
 884  *   Item specification. Only Virtual Network Identifier and protocol type
 885  *   fields are supported. But protocol type can be only Ethernet (0x6558).
 886  *   If the mask is NULL, default mask will be used.
 887  *   Ranging is not supported.
 888  * @param efx_spec[in, out]
 889  *   EFX filter specification to update.
 890  * @param[out] error
 891  *   Perform verbose error reporting if not NULL.
 892  */
 893 static int
 894 sfc_flow_parse_geneve(const struct rte_flow_item *item,
 895                       efx_filter_spec_t *efx_spec,
 896                       struct rte_flow_error *error)
 897 {
 898         int rc;
 899         const struct rte_flow_item_geneve *spec = NULL;
 900         const struct rte_flow_item_geneve *mask = NULL;
 901         const struct rte_flow_item_geneve supp_mask = {
 902                 .protocol = RTE_BE16(0xffff),
 903                 .vni = { 0xff, 0xff, 0xff }
 904         };
 905
 906         rc = sfc_flow_parse_init(item,
 907                                  (const void **)&spec,
 908                                  (const void **)&mask,
 909                                  &supp_mask,
 910                                  &rte_flow_item_geneve_mask,
 911                                  sizeof(struct rte_flow_item_geneve),
 912                                  error);
 913         if (rc != 0)
 914                 return rc;
 915
 916         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
 917                                                      EFX_IPPROTO_UDP, error);
 918         if (rc != 0)
 919                 return rc;
 920
 921         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_GENEVE;
 922         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 923
 924         if (spec == NULL)
 925                 return 0;
 926
 927         if (mask->protocol == supp_mask.protocol) {
 928                 if (spec->protocol != rte_cpu_to_be_16(ETHER_TYPE_TEB)) {
 929                         rte_flow_error_set(error, EINVAL,
 930                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 931                                 "GENEVE encap. protocol must be Ethernet "
 932                                 "(0x6558) in the GENEVE pattern item");
 933                         return -rte_errno;
 934                 }
 935         } else if (mask->protocol != 0) {
 936                 rte_flow_error_set(error, EINVAL,
 937                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 938                         "Unsupported mask for GENEVE encap. protocol");
 939                 return -rte_errno;
 940         }
 941
 942         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->vni,
 943                                                mask->vni, item, error);
 944
 945         return rc;
 946 }
 947
 948 /**
 949  * Convert NVGRE item to EFX filter specification.
 950  *
 951  * @param item[in]
 952  *   Item specification. Only virtual subnet ID field is supported.
 953  *   If the mask is NULL, default mask will be used.
 954  *   Ranging is not supported.
 955  * @param efx_spec[in, out]
 956  *   EFX filter specification to update.
 957  * @param[out] error
 958  *   Perform verbose error reporting if not NULL.
 959  */
 960 static int
 961 sfc_flow_parse_nvgre(const struct rte_flow_item *item,
 962                      efx_filter_spec_t *efx_spec,
 963                      struct rte_flow_error *error)
 964 {
 965         int rc;
 966         const struct rte_flow_item_nvgre *spec = NULL;
 967         const struct rte_flow_item_nvgre *mask = NULL;
 968         const struct rte_flow_item_nvgre supp_mask = {
 969                 .tni = { 0xff, 0xff, 0xff }
 970         };
 971
 972         rc = sfc_flow_parse_init(item,
 973                                  (const void **)&spec,
 974                                  (const void **)&mask,
 975                                  &supp_mask,
 976                                  &rte_flow_item_nvgre_mask,
 977                                  sizeof(struct rte_flow_item_nvgre),
 978                                  error);
 979         if (rc != 0)
 980                 return rc;
 981
 982         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
 983                                                      EFX_IPPROTO_GRE, error);
 984         if (rc != 0)
 985                 return rc;
 986
 987         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_NVGRE;
 988         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 989
 990         if (spec == NULL)
 991                 return 0;
 992
 993         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->tni,
 994                                                mask->tni, item, error);
 995
 996         return rc;
 997 }
 998
 999 static const struct sfc_flow_item sfc_flow_items[] = {
1000         {
1001                 .type = RTE_FLOW_ITEM_TYPE_VOID,
1002                 .prev_layer = SFC_FLOW_ITEM_ANY_LAYER,
1003                 .layer = SFC_FLOW_ITEM_ANY_LAYER,
1004                 .parse = sfc_flow_parse_void,
1005         },
1006         {
1007                 .type = RTE_FLOW_ITEM_TYPE_ETH,
1008                 .prev_layer = SFC_FLOW_ITEM_START_LAYER,
1009                 .layer = SFC_FLOW_ITEM_L2,
1010                 .parse = sfc_flow_parse_eth,
1011         },
1012         {
1013                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
1014                 .prev_layer = SFC_FLOW_ITEM_L2,
1015                 .layer = SFC_FLOW_ITEM_L2,
1016                 .parse = sfc_flow_parse_vlan,
1017         },
1018         {
1019                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
1020                 .prev_layer = SFC_FLOW_ITEM_L2,
1021                 .layer = SFC_FLOW_ITEM_L3,
1022                 .parse = sfc_flow_parse_ipv4,
1023         },
1024         {
1025                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
1026                 .prev_layer = SFC_FLOW_ITEM_L2,
1027                 .layer = SFC_FLOW_ITEM_L3,
1028                 .parse = sfc_flow_parse_ipv6,
1029         },
1030         {
1031                 .type = RTE_FLOW_ITEM_TYPE_TCP,
1032                 .prev_layer = SFC_FLOW_ITEM_L3,
1033                 .layer = SFC_FLOW_ITEM_L4,
1034                 .parse = sfc_flow_parse_tcp,
1035         },
1036         {
1037                 .type = RTE_FLOW_ITEM_TYPE_UDP,
1038                 .prev_layer = SFC_FLOW_ITEM_L3,
1039                 .layer = SFC_FLOW_ITEM_L4,
1040                 .parse = sfc_flow_parse_udp,
1041         },
1042         {
1043                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
1044                 .prev_layer = SFC_FLOW_ITEM_L4,
1045                 .layer = SFC_FLOW_ITEM_START_LAYER,
1046                 .parse = sfc_flow_parse_vxlan,
1047         },
1048         {
1049                 .type = RTE_FLOW_ITEM_TYPE_GENEVE,
1050                 .prev_layer = SFC_FLOW_ITEM_L4,
1051                 .layer = SFC_FLOW_ITEM_START_LAYER,
1052                 .parse = sfc_flow_parse_geneve,
1053         },
1054         {
1055                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
1056                 .prev_layer = SFC_FLOW_ITEM_L3,
1057                 .layer = SFC_FLOW_ITEM_START_LAYER,
1058                 .parse = sfc_flow_parse_nvgre,
1059         },
1060 };
1061
1062 /*
1063  * Protocol-independent flow API support
1064  */
1065 static int
1066 sfc_flow_parse_attr(const struct rte_flow_attr *attr,
1067                     struct rte_flow *flow,
1068                     struct rte_flow_error *error)
1069 {
1070         if (attr == NULL) {
1071                 rte_flow_error_set(error, EINVAL,
1072                                    RTE_FLOW_ERROR_TYPE_ATTR, NULL,
1073                                    "NULL attribute");
1074                 return -rte_errno;
1075         }
1076         if (attr->group != 0) {
1077                 rte_flow_error_set(error, ENOTSUP,
1078                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1079                                    "Groups are not supported");
1080                 return -rte_errno;
1081         }
1082         if (attr->priority != 0) {
1083                 rte_flow_error_set(error, ENOTSUP,
1084                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
1085                                    "Priorities are not supported");
1086                 return -rte_errno;
1087         }
1088         if (attr->egress != 0) {
1089                 rte_flow_error_set(error, ENOTSUP,
1090                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
1091                                    "Egress is not supported");
1092                 return -rte_errno;
1093         }
1094         if (attr->ingress == 0) {
1095                 rte_flow_error_set(error, ENOTSUP,
1096                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
1097                                    "Only ingress is supported");
1098                 return -rte_errno;
1099         }
1100
1101         flow->spec.template.efs_flags |= EFX_FILTER_FLAG_RX;
1102         flow->spec.template.efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
1103
1104         return 0;
1105 }
1106
1107 /* Get item from array sfc_flow_items */
1108 static const struct sfc_flow_item *
1109 sfc_flow_get_item(enum rte_flow_item_type type)
1110 {
1111         unsigned int i;
1112
1113         for (i = 0; i < RTE_DIM(sfc_flow_items); i++)
1114                 if (sfc_flow_items[i].type == type)
1115                         return &sfc_flow_items[i];
1116
1117         return NULL;
1118 }
1119
1120 static int
1121 sfc_flow_parse_pattern(const struct rte_flow_item pattern[],
1122                        struct rte_flow *flow,
1123                        struct rte_flow_error *error)
1124 {
1125         int rc;
1126         unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER;
1127         boolean_t is_ifrm = B_FALSE;
1128         const struct sfc_flow_item *item;
1129
1130         if (pattern == NULL) {
1131                 rte_flow_error_set(error, EINVAL,
1132                                    RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
1133                                    "NULL pattern");
1134                 return -rte_errno;
1135         }
1136
1137         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
1138                 item = sfc_flow_get_item(pattern->type);
1139                 if (item == NULL) {
1140                         rte_flow_error_set(error, ENOTSUP,
1141                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
1142                                            "Unsupported pattern item");
1143                         return -rte_errno;
1144                 }
1145
1146                 /*
1147                  * Omitting one or several protocol layers at the beginning
1148                  * of pattern is supported
1149                  */
1150                 if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
1151                     prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
1152                     item->prev_layer != prev_layer) {
1153                         rte_flow_error_set(error, ENOTSUP,
1154                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
1155                                            "Unexpected sequence of pattern items");
1156                         return -rte_errno;
1157                 }
1158
1159                 /*
1160                  * Allow only VOID and ETH pattern items in the inner frame.
1161                  * Also check that there is only one tunneling protocol.
1162                  */
1163                 switch (item->type) {
1164                 case RTE_FLOW_ITEM_TYPE_VOID:
1165                 case RTE_FLOW_ITEM_TYPE_ETH:
1166                         break;
1167
1168                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1169                 case RTE_FLOW_ITEM_TYPE_GENEVE:
1170                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1171                         if (is_ifrm) {
1172                                 rte_flow_error_set(error, EINVAL,
1173                                         RTE_FLOW_ERROR_TYPE_ITEM,
1174                                         pattern,
1175                                         "More than one tunneling protocol");
1176                                 return -rte_errno;
1177                         }
1178                         is_ifrm = B_TRUE;
1179                         break;
1180
1181                 default:
1182                         if (is_ifrm) {
1183                                 rte_flow_error_set(error, EINVAL,
1184                                         RTE_FLOW_ERROR_TYPE_ITEM,
1185                                         pattern,
1186                                         "There is an unsupported pattern item "
1187                                         "in the inner frame");
1188                                 return -rte_errno;
1189                         }
1190                         break;
1191                 }
1192
1193                 rc = item->parse(pattern, &flow->spec.template, error);
1194                 if (rc != 0)
1195                         return rc;
1196
1197                 if (item->layer != SFC_FLOW_ITEM_ANY_LAYER)
1198                         prev_layer = item->layer;
1199         }
1200
1201         return 0;
1202 }
1203
1204 static int
1205 sfc_flow_parse_queue(struct sfc_adapter *sa,
1206                      const struct rte_flow_action_queue *queue,
1207                      struct rte_flow *flow)
1208 {
1209         struct sfc_rxq *rxq;
1210
1211         if (queue->index >= sa->rxq_count)
1212                 return -EINVAL;
1213
1214         rxq = sa->rxq_info[queue->index].rxq;
1215         flow->spec.template.efs_dmaq_id = (uint16_t)rxq->hw_index;
1216
1217         return 0;
1218 }
1219
1220 #if EFSYS_OPT_RX_SCALE
1221 static int
1222 sfc_flow_parse_rss(struct sfc_adapter *sa,
1223                    const struct rte_flow_action_rss *rss,
1224                    struct rte_flow *flow)
1225 {
1226         unsigned int rxq_sw_index;
1227         struct sfc_rxq *rxq;
1228         unsigned int rxq_hw_index_min;
1229         unsigned int rxq_hw_index_max;
1230         const struct rte_eth_rss_conf *rss_conf = rss->rss_conf;
1231         uint64_t rss_hf;
1232         uint8_t *rss_key = NULL;
1233         struct sfc_flow_rss *sfc_rss_conf = &flow->rss_conf;
1234         unsigned int i;
1235
1236         if (rss->num == 0)
1237                 return -EINVAL;
1238
1239         rxq_sw_index = sa->rxq_count - 1;
1240         rxq = sa->rxq_info[rxq_sw_index].rxq;
1241         rxq_hw_index_min = rxq->hw_index;
1242         rxq_hw_index_max = 0;
1243
1244         for (i = 0; i < rss->num; ++i) {
1245                 rxq_sw_index = rss->queue[i];
1246
1247                 if (rxq_sw_index >= sa->rxq_count)
1248                         return -EINVAL;
1249
1250                 rxq = sa->rxq_info[rxq_sw_index].rxq;
1251
1252                 if (rxq->hw_index < rxq_hw_index_min)
1253                         rxq_hw_index_min = rxq->hw_index;
1254
1255                 if (rxq->hw_index > rxq_hw_index_max)
1256                         rxq_hw_index_max = rxq->hw_index;
1257         }
1258
1259         rss_hf = (rss_conf != NULL) ? rss_conf->rss_hf : SFC_RSS_OFFLOADS;
1260         if ((rss_hf & ~SFC_RSS_OFFLOADS) != 0)
1261                 return -EINVAL;
1262
1263         if (rss_conf != NULL) {
1264                 if (rss_conf->rss_key_len != sizeof(sa->rss_key))
1265                         return -EINVAL;
1266
1267                 rss_key = rss_conf->rss_key;
1268         } else {
1269                 rss_key = sa->rss_key;
1270         }
1271
1272         flow->rss = B_TRUE;
1273
1274         sfc_rss_conf->rxq_hw_index_min = rxq_hw_index_min;
1275         sfc_rss_conf->rxq_hw_index_max = rxq_hw_index_max;
1276         sfc_rss_conf->rss_hash_types = sfc_rte_to_efx_hash_type(rss_hf);
1277         rte_memcpy(sfc_rss_conf->rss_key, rss_key, sizeof(sa->rss_key));
1278
1279         for (i = 0; i < RTE_DIM(sfc_rss_conf->rss_tbl); ++i) {
1280                 unsigned int rxq_sw_index = rss->queue[i % rss->num];
1281                 struct sfc_rxq *rxq = sa->rxq_info[rxq_sw_index].rxq;
1282
1283                 sfc_rss_conf->rss_tbl[i] = rxq->hw_index - rxq_hw_index_min;
1284         }
1285
1286         return 0;
1287 }
1288 #endif /* EFSYS_OPT_RX_SCALE */
1289
1290 static int
1291 sfc_flow_spec_flush(struct sfc_adapter *sa, struct sfc_flow_spec *spec,
1292                     unsigned int filters_count)
1293 {
1294         unsigned int i;
1295         int ret = 0;
1296
1297         for (i = 0; i < filters_count; i++) {
1298                 int rc;
1299
1300                 rc = efx_filter_remove(sa->nic, &spec->filters[i]);
1301                 if (ret == 0 && rc != 0) {
1302                         sfc_err(sa, "failed to remove filter specification "
1303                                 "(rc = %d)", rc);
1304                         ret = rc;
1305                 }
1306         }
1307
1308         return ret;
1309 }
1310
1311 static int
1312 sfc_flow_spec_insert(struct sfc_adapter *sa, struct sfc_flow_spec *spec)
1313 {
1314         unsigned int i;
1315         int rc = 0;
1316
1317         for (i = 0; i < spec->count; i++) {
1318                 rc = efx_filter_insert(sa->nic, &spec->filters[i]);
1319                 if (rc != 0) {
1320                         sfc_flow_spec_flush(sa, spec, i);
1321                         break;
1322                 }
1323         }
1324
1325         return rc;
1326 }
1327
1328 static int
1329 sfc_flow_spec_remove(struct sfc_adapter *sa, struct sfc_flow_spec *spec)
1330 {
1331         return sfc_flow_spec_flush(sa, spec, spec->count);
1332 }
1333
1334 static int
1335 sfc_flow_filter_insert(struct sfc_adapter *sa,
1336                        struct rte_flow *flow)
1337 {
1338 #if EFSYS_OPT_RX_SCALE
1339         struct sfc_flow_rss *rss = &flow->rss_conf;
1340         uint32_t efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
1341         unsigned int i;
1342         int rc = 0;
1343
1344         if (flow->rss) {
1345                 unsigned int rss_spread = MIN(rss->rxq_hw_index_max -
1346                                               rss->rxq_hw_index_min + 1,
1347                                               EFX_MAXRSS);
1348
1349                 rc = efx_rx_scale_context_alloc(sa->nic,
1350                                                 EFX_RX_SCALE_EXCLUSIVE,
1351                                                 rss_spread,
1352                                                 &efs_rss_context);
1353                 if (rc != 0)
1354                         goto fail_scale_context_alloc;
1355
1356                 rc = efx_rx_scale_mode_set(sa->nic, efs_rss_context,
1357                                            EFX_RX_HASHALG_TOEPLITZ,
1358                                            rss->rss_hash_types, B_TRUE);
1359                 if (rc != 0)
1360                         goto fail_scale_mode_set;
1361
1362                 rc = efx_rx_scale_key_set(sa->nic, efs_rss_context,
1363                                           rss->rss_key,
1364                                           sizeof(sa->rss_key));
1365                 if (rc != 0)
1366                         goto fail_scale_key_set;
1367
1368                 /*
1369                  * At this point, fully elaborated filter specifications
1370                  * have been produced from the template. To make sure that
1371                  * RSS behaviour is consistent between them, set the same
1372                  * RSS context value everywhere.
1373                  */
1374                 for (i = 0; i < flow->spec.count; i++) {
1375                         efx_filter_spec_t *spec = &flow->spec.filters[i];
1376
1377                         spec->efs_rss_context = efs_rss_context;
1378                         spec->efs_dmaq_id = rss->rxq_hw_index_min;
1379                         spec->efs_flags |= EFX_FILTER_FLAG_RX_RSS;
1380                 }
1381         }
1382
1383         rc = sfc_flow_spec_insert(sa, &flow->spec);
1384         if (rc != 0)
1385                 goto fail_filter_insert;
1386
1387         if (flow->rss) {
1388                 /*
1389                  * Scale table is set after filter insertion because
1390                  * the table entries are relative to the base RxQ ID
1391                  * and the latter is submitted to the HW by means of
1392                  * inserting a filter, so by the time of the request
1393                  * the HW knows all the information needed to verify
1394                  * the table entries, and the operation will succeed
1395                  */
1396                 rc = efx_rx_scale_tbl_set(sa->nic, efs_rss_context,
1397                                           rss->rss_tbl, RTE_DIM(rss->rss_tbl));
1398                 if (rc != 0)
1399                         goto fail_scale_tbl_set;
1400         }
1401
1402         return 0;
1403
1404 fail_scale_tbl_set:
1405         sfc_flow_spec_remove(sa, &flow->spec);
1406
1407 fail_filter_insert:
1408 fail_scale_key_set:
1409 fail_scale_mode_set:
1410         if (efs_rss_context != EFX_RSS_CONTEXT_DEFAULT)
1411                 efx_rx_scale_context_free(sa->nic, efs_rss_context);
1412
1413 fail_scale_context_alloc:
1414         return rc;
1415 #else /* !EFSYS_OPT_RX_SCALE */
1416         return sfc_flow_spec_insert(sa, &flow->spec);
1417 #endif /* EFSYS_OPT_RX_SCALE */
1418 }
1419
1420 static int
1421 sfc_flow_filter_remove(struct sfc_adapter *sa,
1422                        struct rte_flow *flow)
1423 {
1424         int rc = 0;
1425
1426         rc = sfc_flow_spec_remove(sa, &flow->spec);
1427         if (rc != 0)
1428                 return rc;
1429
1430 #if EFSYS_OPT_RX_SCALE
1431         if (flow->rss) {
1432                 /*
1433                  * All specifications for a given flow rule have the same RSS
1434                  * context, so that RSS context value is taken from the first
1435                  * filter specification
1436                  */
1437                 efx_filter_spec_t *spec = &flow->spec.filters[0];
1438
1439                 rc = efx_rx_scale_context_free(sa->nic, spec->efs_rss_context);
1440         }
1441 #endif /* EFSYS_OPT_RX_SCALE */
1442
1443         return rc;
1444 }
1445
1446 static int
1447 sfc_flow_parse_actions(struct sfc_adapter *sa,
1448                        const struct rte_flow_action actions[],
1449                        struct rte_flow *flow,
1450                        struct rte_flow_error *error)
1451 {
1452         int rc;
1453         boolean_t is_specified = B_FALSE;
1454
1455         if (actions == NULL) {
1456                 rte_flow_error_set(error, EINVAL,
1457                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
1458                                    "NULL actions");
1459                 return -rte_errno;
1460         }
1461
1462         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1463                 switch (actions->type) {
1464                 case RTE_FLOW_ACTION_TYPE_VOID:
1465                         break;
1466
1467                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1468                         rc = sfc_flow_parse_queue(sa, actions->conf, flow);
1469                         if (rc != 0) {
1470                                 rte_flow_error_set(error, EINVAL,
1471                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1472                                         "Bad QUEUE action");
1473                                 return -rte_errno;
1474                         }
1475
1476                         is_specified = B_TRUE;
1477                         break;
1478
1479 #if EFSYS_OPT_RX_SCALE
1480                 case RTE_FLOW_ACTION_TYPE_RSS:
1481                         rc = sfc_flow_parse_rss(sa, actions->conf, flow);
1482                         if (rc != 0) {
1483                                 rte_flow_error_set(error, rc,
1484                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1485                                         "Bad RSS action");
1486                                 return -rte_errno;
1487                         }
1488
1489                         is_specified = B_TRUE;
1490                         break;
1491 #endif /* EFSYS_OPT_RX_SCALE */
1492
1493                 default:
1494                         rte_flow_error_set(error, ENOTSUP,
1495                                            RTE_FLOW_ERROR_TYPE_ACTION, actions,
1496                                            "Action is not supported");
1497                         return -rte_errno;
1498                 }
1499         }
1500
1501         if (!is_specified) {
1502                 rte_flow_error_set(error, EINVAL,
1503                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, actions,
1504                                    "Action is unspecified");
1505                 return -rte_errno;
1506         }
1507
1508         return 0;
1509 }
1510
1511 static int
1512 sfc_flow_parse(struct rte_eth_dev *dev,
1513                const struct rte_flow_attr *attr,
1514                const struct rte_flow_item pattern[],
1515                const struct rte_flow_action actions[],
1516                struct rte_flow *flow,
1517                struct rte_flow_error *error)
1518 {
1519         struct sfc_adapter *sa = dev->data->dev_private;
1520         efx_filter_match_flags_t match_flags =
1521                 flow->spec.template.efs_match_flags;
1522         int rc;
1523
1524         rc = sfc_flow_parse_attr(attr, flow, error);
1525         if (rc != 0)
1526                 goto fail_bad_value;
1527
1528         rc = sfc_flow_parse_pattern(pattern, flow, error);
1529         if (rc != 0)
1530                 goto fail_bad_value;
1531
1532         rc = sfc_flow_parse_actions(sa, actions, flow, error);
1533         if (rc != 0)
1534                 goto fail_bad_value;
1535
1536         if (!sfc_filter_is_match_supported(sa, match_flags)) {
1537                 rte_flow_error_set(error, ENOTSUP,
1538                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1539                                    "Flow rule pattern is not supported");
1540                 return -rte_errno;
1541         }
1542
1543         /*
1544          * At this point, template specification simply becomes the first
1545          * fully elaborated spec
1546          */
1547         flow->spec.filters[0] = flow->spec.template;
1548         flow->spec.count = 1;
1549
1550 fail_bad_value:
1551         return rc;
1552 }
1553
1554 static int
1555 sfc_flow_validate(struct rte_eth_dev *dev,
1556                   const struct rte_flow_attr *attr,
1557                   const struct rte_flow_item pattern[],
1558                   const struct rte_flow_action actions[],
1559                   struct rte_flow_error *error)
1560 {
1561         struct rte_flow flow;
1562
1563         memset(&flow, 0, sizeof(flow));
1564
1565         return sfc_flow_parse(dev, attr, pattern, actions, &flow, error);
1566 }
1567
1568 static struct rte_flow *
1569 sfc_flow_create(struct rte_eth_dev *dev,
1570                 const struct rte_flow_attr *attr,
1571                 const struct rte_flow_item pattern[],
1572                 const struct rte_flow_action actions[],
1573                 struct rte_flow_error *error)
1574 {
1575         struct sfc_adapter *sa = dev->data->dev_private;
1576         struct rte_flow *flow = NULL;
1577         int rc;
1578
1579         flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0);
1580         if (flow == NULL) {
1581                 rte_flow_error_set(error, ENOMEM,
1582                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1583                                    "Failed to allocate memory");
1584                 goto fail_no_mem;
1585         }
1586
1587         rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error);
1588         if (rc != 0)
1589                 goto fail_bad_value;
1590
1591         TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries);
1592
1593         sfc_adapter_lock(sa);
1594
1595         if (sa->state == SFC_ADAPTER_STARTED) {
1596                 rc = sfc_flow_filter_insert(sa, flow);
1597                 if (rc != 0) {
1598                         rte_flow_error_set(error, rc,
1599                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1600                                 "Failed to insert filter");
1601                         goto fail_filter_insert;
1602                 }
1603         }
1604
1605         sfc_adapter_unlock(sa);
1606
1607         return flow;
1608
1609 fail_filter_insert:
1610         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1611
1612 fail_bad_value:
1613         rte_free(flow);
1614         sfc_adapter_unlock(sa);
1615
1616 fail_no_mem:
1617         return NULL;
1618 }
1619
1620 static int
1621 sfc_flow_remove(struct sfc_adapter *sa,
1622                 struct rte_flow *flow,
1623                 struct rte_flow_error *error)
1624 {
1625         int rc = 0;
1626
1627         SFC_ASSERT(sfc_adapter_is_locked(sa));
1628
1629         if (sa->state == SFC_ADAPTER_STARTED) {
1630                 rc = sfc_flow_filter_remove(sa, flow);
1631                 if (rc != 0)
1632                         rte_flow_error_set(error, rc,
1633                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1634                                 "Failed to destroy flow rule");
1635         }
1636
1637         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1638         rte_free(flow);
1639
1640         return rc;
1641 }
1642
1643 static int
1644 sfc_flow_destroy(struct rte_eth_dev *dev,
1645                  struct rte_flow *flow,
1646                  struct rte_flow_error *error)
1647 {
1648         struct sfc_adapter *sa = dev->data->dev_private;
1649         struct rte_flow *flow_ptr;
1650         int rc = EINVAL;
1651
1652         sfc_adapter_lock(sa);
1653
1654         TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) {
1655                 if (flow_ptr == flow)
1656                         rc = 0;
1657         }
1658         if (rc != 0) {
1659                 rte_flow_error_set(error, rc,
1660                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1661                                    "Failed to find flow rule to destroy");
1662                 goto fail_bad_value;
1663         }
1664
1665         rc = sfc_flow_remove(sa, flow, error);
1666
1667 fail_bad_value:
1668         sfc_adapter_unlock(sa);
1669
1670         return -rc;
1671 }
1672
1673 static int
1674 sfc_flow_flush(struct rte_eth_dev *dev,
1675                struct rte_flow_error *error)
1676 {
1677         struct sfc_adapter *sa = dev->data->dev_private;
1678         struct rte_flow *flow;
1679         int rc = 0;
1680         int ret = 0;
1681
1682         sfc_adapter_lock(sa);
1683
1684         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1685                 rc = sfc_flow_remove(sa, flow, error);
1686                 if (rc != 0)
1687                         ret = rc;
1688         }
1689
1690         sfc_adapter_unlock(sa);
1691
1692         return -ret;
1693 }
1694
1695 static int
1696 sfc_flow_isolate(struct rte_eth_dev *dev, int enable,
1697                  struct rte_flow_error *error)
1698 {
1699         struct sfc_adapter *sa = dev->data->dev_private;
1700         struct sfc_port *port = &sa->port;
1701         int ret = 0;
1702
1703         sfc_adapter_lock(sa);
1704         if (sa->state != SFC_ADAPTER_INITIALIZED) {
1705                 rte_flow_error_set(error, EBUSY,
1706                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1707                                    NULL, "please close the port first");
1708                 ret = -rte_errno;
1709         } else {
1710                 port->isolated = (enable) ? B_TRUE : B_FALSE;
1711         }
1712         sfc_adapter_unlock(sa);
1713
1714         return ret;
1715 }
1716
1717 const struct rte_flow_ops sfc_flow_ops = {
1718         .validate = sfc_flow_validate,
1719         .create = sfc_flow_create,
1720         .destroy = sfc_flow_destroy,
1721         .flush = sfc_flow_flush,
1722         .query = NULL,
1723         .isolate = sfc_flow_isolate,
1724 };
1725
1726 void
1727 sfc_flow_init(struct sfc_adapter *sa)
1728 {
1729         SFC_ASSERT(sfc_adapter_is_locked(sa));
1730
1731         TAILQ_INIT(&sa->filter.flow_list);
1732 }
1733
1734 void
1735 sfc_flow_fini(struct sfc_adapter *sa)
1736 {
1737         struct rte_flow *flow;
1738
1739         SFC_ASSERT(sfc_adapter_is_locked(sa));
1740
1741         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1742                 TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1743                 rte_free(flow);
1744         }
1745 }
1746
1747 void
1748 sfc_flow_stop(struct sfc_adapter *sa)
1749 {
1750         struct rte_flow *flow;
1751
1752         SFC_ASSERT(sfc_adapter_is_locked(sa));
1753
1754         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries)
1755                 sfc_flow_filter_remove(sa, flow);
1756 }
1757
1758 int
1759 sfc_flow_start(struct sfc_adapter *sa)
1760 {
1761         struct rte_flow *flow;
1762         int rc = 0;
1763
1764         sfc_log_init(sa, "entry");
1765
1766         SFC_ASSERT(sfc_adapter_is_locked(sa));
1767
1768         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) {
1769                 rc = sfc_flow_filter_insert(sa, flow);
1770                 if (rc != 0)
1771                         goto fail_bad_flow;
1772         }
1773
1774         sfc_log_init(sa, "done");
1775
1776 fail_bad_flow:
1777         return rc;
1778 }