drivers/net/sfc/sfc_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2017-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * This software was jointly developed between OKTET Labs (under contract
   7  * for Solarflare) and Solarflare Communications, Inc.
   8  */
   9
  10 #include <rte_tailq.h>
  11 #include <rte_common.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_eth_ctrl.h>
  14 #include <rte_ether.h>
  15 #include <rte_flow.h>
  16 #include <rte_flow_driver.h>
  17
  18 #include "efx.h"
  19
  20 #include "sfc.h"
  21 #include "sfc_rx.h"
  22 #include "sfc_filter.h"
  23 #include "sfc_flow.h"
  24 #include "sfc_log.h"
  25
  26 /*
  27  * At now flow API is implemented in such a manner that each
  28  * flow rule is converted to a hardware filter.
  29  * All elements of flow rule (attributes, pattern items, actions)
  30  * correspond to one or more fields in the efx_filter_spec_s structure
  31  * that is responsible for the hardware filter.
  32  */
  33
  34 enum sfc_flow_item_layers {
  35         SFC_FLOW_ITEM_ANY_LAYER,
  36         SFC_FLOW_ITEM_START_LAYER,
  37         SFC_FLOW_ITEM_L2,
  38         SFC_FLOW_ITEM_L3,
  39         SFC_FLOW_ITEM_L4,
  40 };
  41
  42 typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item,
  43                                   efx_filter_spec_t *spec,
  44                                   struct rte_flow_error *error);
  45
  46 struct sfc_flow_item {
  47         enum rte_flow_item_type type;           /* Type of item */
  48         enum sfc_flow_item_layers layer;        /* Layer of item */
  49         enum sfc_flow_item_layers prev_layer;   /* Previous layer of item */
  50         sfc_flow_item_parse *parse;             /* Parsing function */
  51 };
  52
  53 static sfc_flow_item_parse sfc_flow_parse_void;
  54 static sfc_flow_item_parse sfc_flow_parse_eth;
  55 static sfc_flow_item_parse sfc_flow_parse_vlan;
  56 static sfc_flow_item_parse sfc_flow_parse_ipv4;
  57 static sfc_flow_item_parse sfc_flow_parse_ipv6;
  58 static sfc_flow_item_parse sfc_flow_parse_tcp;
  59 static sfc_flow_item_parse sfc_flow_parse_udp;
  60 static sfc_flow_item_parse sfc_flow_parse_vxlan;
  61 static sfc_flow_item_parse sfc_flow_parse_nvgre;
  62
  63 static boolean_t
  64 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
  65 {
  66         uint8_t sum = 0;
  67         unsigned int i;
  68
  69         for (i = 0; i < size; i++)
  70                 sum |= buf[i];
  71
  72         return (sum == 0) ? B_TRUE : B_FALSE;
  73 }
  74
  75 /*
  76  * Validate item and prepare structures spec and mask for parsing
  77  */
  78 static int
  79 sfc_flow_parse_init(const struct rte_flow_item *item,
  80                     const void **spec_ptr,
  81                     const void **mask_ptr,
  82                     const void *supp_mask,
  83                     const void *def_mask,
  84                     unsigned int size,
  85                     struct rte_flow_error *error)
  86 {
  87         const uint8_t *spec;
  88         const uint8_t *mask;
  89         const uint8_t *last;
  90         uint8_t match;
  91         uint8_t supp;
  92         unsigned int i;
  93
  94         if (item == NULL) {
  95                 rte_flow_error_set(error, EINVAL,
  96                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
  97                                    "NULL item");
  98                 return -rte_errno;
  99         }
 100
 101         if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) {
 102                 rte_flow_error_set(error, EINVAL,
 103                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 104                                    "Mask or last is set without spec");
 105                 return -rte_errno;
 106         }
 107
 108         /*
 109          * If "mask" is not set, default mask is used,
 110          * but if default mask is NULL, "mask" should be set
 111          */
 112         if (item->mask == NULL) {
 113                 if (def_mask == NULL) {
 114                         rte_flow_error_set(error, EINVAL,
 115                                 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 116                                 "Mask should be specified");
 117                         return -rte_errno;
 118                 }
 119
 120                 mask = def_mask;
 121         } else {
 122                 mask = item->mask;
 123         }
 124
 125         spec = item->spec;
 126         last = item->last;
 127
 128         if (spec == NULL)
 129                 goto exit;
 130
 131         /*
 132          * If field values in "last" are either 0 or equal to the corresponding
 133          * values in "spec" then they are ignored
 134          */
 135         if (last != NULL &&
 136             !sfc_flow_is_zero(last, size) &&
 137             memcmp(last, spec, size) != 0) {
 138                 rte_flow_error_set(error, ENOTSUP,
 139                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 140                                    "Ranging is not supported");
 141                 return -rte_errno;
 142         }
 143
 144         if (supp_mask == NULL) {
 145                 rte_flow_error_set(error, EINVAL,
 146                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 147                         "Supported mask for item should be specified");
 148                 return -rte_errno;
 149         }
 150
 151         /* Check that mask and spec not asks for more match than supp_mask */
 152         for (i = 0; i < size; i++) {
 153                 match = spec[i] | mask[i];
 154                 supp = ((const uint8_t *)supp_mask)[i];
 155
 156                 if ((match | supp) != supp) {
 157                         rte_flow_error_set(error, ENOTSUP,
 158                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 159                                            "Item's field is not supported");
 160                         return -rte_errno;
 161                 }
 162         }
 163
 164 exit:
 165         *spec_ptr = spec;
 166         *mask_ptr = mask;
 167         return 0;
 168 }
 169
 170 /*
 171  * Protocol parsers.
 172  * Masking is not supported, so masks in items should be either
 173  * full or empty (zeroed) and set only for supported fields which
 174  * are specified in the supp_mask.
 175  */
 176
 177 static int
 178 sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item,
 179                     __rte_unused efx_filter_spec_t *efx_spec,
 180                     __rte_unused struct rte_flow_error *error)
 181 {
 182         return 0;
 183 }
 184
 185 /**
 186  * Convert Ethernet item to EFX filter specification.
 187  *
 188  * @param item[in]
 189  *   Item specification. Only source and destination addresses and
 190  *   Ethernet type fields are supported. In addition to full and
 191  *   empty masks of destination address, individual/group mask is
 192  *   also supported. If the mask is NULL, default mask will be used.
 193  *   Ranging is not supported.
 194  * @param efx_spec[in, out]
 195  *   EFX filter specification to update.
 196  * @param[out] error
 197  *   Perform verbose error reporting if not NULL.
 198  */
 199 static int
 200 sfc_flow_parse_eth(const struct rte_flow_item *item,
 201                    efx_filter_spec_t *efx_spec,
 202                    struct rte_flow_error *error)
 203 {
 204         int rc;
 205         const struct rte_flow_item_eth *spec = NULL;
 206         const struct rte_flow_item_eth *mask = NULL;
 207         const struct rte_flow_item_eth supp_mask = {
 208                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 209                 .src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 210                 .type = 0xffff,
 211         };
 212         const uint8_t ig_mask[EFX_MAC_ADDR_LEN] = {
 213                 0x01, 0x00, 0x00, 0x00, 0x00, 0x00
 214         };
 215
 216         rc = sfc_flow_parse_init(item,
 217                                  (const void **)&spec,
 218                                  (const void **)&mask,
 219                                  &supp_mask,
 220                                  &rte_flow_item_eth_mask,
 221                                  sizeof(struct rte_flow_item_eth),
 222                                  error);
 223         if (rc != 0)
 224                 return rc;
 225
 226         /* If "spec" is not set, could be any Ethernet */
 227         if (spec == NULL)
 228                 return 0;
 229
 230         if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) {
 231                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 232                 rte_memcpy(efx_spec->efs_loc_mac, spec->dst.addr_bytes,
 233                            EFX_MAC_ADDR_LEN);
 234         } else if (memcmp(mask->dst.addr_bytes, ig_mask,
 235                           EFX_MAC_ADDR_LEN) == 0) {
 236                 if (is_unicast_ether_addr(&spec->dst))
 237                         efx_spec->efs_match_flags |=
 238                                 EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 239                 else
 240                         efx_spec->efs_match_flags |=
 241                                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 242         } else if (!is_zero_ether_addr(&mask->dst)) {
 243                 goto fail_bad_mask;
 244         }
 245
 246         if (is_same_ether_addr(&mask->src, &supp_mask.src)) {
 247                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC;
 248                 rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes,
 249                            EFX_MAC_ADDR_LEN);
 250         } else if (!is_zero_ether_addr(&mask->src)) {
 251                 goto fail_bad_mask;
 252         }
 253
 254         /*
 255          * Ether type is in big-endian byte order in item and
 256          * in little-endian in efx_spec, so byte swap is used
 257          */
 258         if (mask->type == supp_mask.type) {
 259                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 260                 efx_spec->efs_ether_type = rte_bswap16(spec->type);
 261         } else if (mask->type != 0) {
 262                 goto fail_bad_mask;
 263         }
 264
 265         return 0;
 266
 267 fail_bad_mask:
 268         rte_flow_error_set(error, EINVAL,
 269                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 270                            "Bad mask in the ETH pattern item");
 271         return -rte_errno;
 272 }
 273
 274 /**
 275  * Convert VLAN item to EFX filter specification.
 276  *
 277  * @param item[in]
 278  *   Item specification. Only VID field is supported.
 279  *   The mask can not be NULL. Ranging is not supported.
 280  * @param efx_spec[in, out]
 281  *   EFX filter specification to update.
 282  * @param[out] error
 283  *   Perform verbose error reporting if not NULL.
 284  */
 285 static int
 286 sfc_flow_parse_vlan(const struct rte_flow_item *item,
 287                     efx_filter_spec_t *efx_spec,
 288                     struct rte_flow_error *error)
 289 {
 290         int rc;
 291         uint16_t vid;
 292         const struct rte_flow_item_vlan *spec = NULL;
 293         const struct rte_flow_item_vlan *mask = NULL;
 294         const struct rte_flow_item_vlan supp_mask = {
 295                 .tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX),
 296         };
 297
 298         rc = sfc_flow_parse_init(item,
 299                                  (const void **)&spec,
 300                                  (const void **)&mask,
 301                                  &supp_mask,
 302                                  NULL,
 303                                  sizeof(struct rte_flow_item_vlan),
 304                                  error);
 305         if (rc != 0)
 306                 return rc;
 307
 308         /*
 309          * VID is in big-endian byte order in item and
 310          * in little-endian in efx_spec, so byte swap is used.
 311          * If two VLAN items are included, the first matches
 312          * the outer tag and the next matches the inner tag.
 313          */
 314         if (mask->tci == supp_mask.tci) {
 315                 vid = rte_bswap16(spec->tci);
 316
 317                 if (!(efx_spec->efs_match_flags &
 318                       EFX_FILTER_MATCH_OUTER_VID)) {
 319                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 320                         efx_spec->efs_outer_vid = vid;
 321                 } else if (!(efx_spec->efs_match_flags &
 322                              EFX_FILTER_MATCH_INNER_VID)) {
 323                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID;
 324                         efx_spec->efs_inner_vid = vid;
 325                 } else {
 326                         rte_flow_error_set(error, EINVAL,
 327                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 328                                            "More than two VLAN items");
 329                         return -rte_errno;
 330                 }
 331         } else {
 332                 rte_flow_error_set(error, EINVAL,
 333                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 334                                    "VLAN ID in TCI match is required");
 335                 return -rte_errno;
 336         }
 337
 338         return 0;
 339 }
 340
 341 /**
 342  * Convert IPv4 item to EFX filter specification.
 343  *
 344  * @param item[in]
 345  *   Item specification. Only source and destination addresses and
 346  *   protocol fields are supported. If the mask is NULL, default
 347  *   mask will be used. Ranging is not supported.
 348  * @param efx_spec[in, out]
 349  *   EFX filter specification to update.
 350  * @param[out] error
 351  *   Perform verbose error reporting if not NULL.
 352  */
 353 static int
 354 sfc_flow_parse_ipv4(const struct rte_flow_item *item,
 355                     efx_filter_spec_t *efx_spec,
 356                     struct rte_flow_error *error)
 357 {
 358         int rc;
 359         const struct rte_flow_item_ipv4 *spec = NULL;
 360         const struct rte_flow_item_ipv4 *mask = NULL;
 361         const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4);
 362         const struct rte_flow_item_ipv4 supp_mask = {
 363                 .hdr = {
 364                         .src_addr = 0xffffffff,
 365                         .dst_addr = 0xffffffff,
 366                         .next_proto_id = 0xff,
 367                 }
 368         };
 369
 370         rc = sfc_flow_parse_init(item,
 371                                  (const void **)&spec,
 372                                  (const void **)&mask,
 373                                  &supp_mask,
 374                                  &rte_flow_item_ipv4_mask,
 375                                  sizeof(struct rte_flow_item_ipv4),
 376                                  error);
 377         if (rc != 0)
 378                 return rc;
 379
 380         /*
 381          * Filtering by IPv4 source and destination addresses requires
 382          * the appropriate ETHER_TYPE in hardware filters
 383          */
 384         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 385                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 386                 efx_spec->efs_ether_type = ether_type_ipv4;
 387         } else if (efx_spec->efs_ether_type != ether_type_ipv4) {
 388                 rte_flow_error_set(error, EINVAL,
 389                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 390                         "Ethertype in pattern with IPV4 item should be appropriate");
 391                 return -rte_errno;
 392         }
 393
 394         if (spec == NULL)
 395                 return 0;
 396
 397         /*
 398          * IPv4 addresses are in big-endian byte order in item and in
 399          * efx_spec
 400          */
 401         if (mask->hdr.src_addr == supp_mask.hdr.src_addr) {
 402                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 403                 efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr;
 404         } else if (mask->hdr.src_addr != 0) {
 405                 goto fail_bad_mask;
 406         }
 407
 408         if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) {
 409                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 410                 efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr;
 411         } else if (mask->hdr.dst_addr != 0) {
 412                 goto fail_bad_mask;
 413         }
 414
 415         if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) {
 416                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 417                 efx_spec->efs_ip_proto = spec->hdr.next_proto_id;
 418         } else if (mask->hdr.next_proto_id != 0) {
 419                 goto fail_bad_mask;
 420         }
 421
 422         return 0;
 423
 424 fail_bad_mask:
 425         rte_flow_error_set(error, EINVAL,
 426                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 427                            "Bad mask in the IPV4 pattern item");
 428         return -rte_errno;
 429 }
 430
 431 /**
 432  * Convert IPv6 item to EFX filter specification.
 433  *
 434  * @param item[in]
 435  *   Item specification. Only source and destination addresses and
 436  *   next header fields are supported. If the mask is NULL, default
 437  *   mask will be used. Ranging is not supported.
 438  * @param efx_spec[in, out]
 439  *   EFX filter specification to update.
 440  * @param[out] error
 441  *   Perform verbose error reporting if not NULL.
 442  */
 443 static int
 444 sfc_flow_parse_ipv6(const struct rte_flow_item *item,
 445                     efx_filter_spec_t *efx_spec,
 446                     struct rte_flow_error *error)
 447 {
 448         int rc;
 449         const struct rte_flow_item_ipv6 *spec = NULL;
 450         const struct rte_flow_item_ipv6 *mask = NULL;
 451         const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6);
 452         const struct rte_flow_item_ipv6 supp_mask = {
 453                 .hdr = {
 454                         .src_addr = { 0xff, 0xff, 0xff, 0xff,
 455                                       0xff, 0xff, 0xff, 0xff,
 456                                       0xff, 0xff, 0xff, 0xff,
 457                                       0xff, 0xff, 0xff, 0xff },
 458                         .dst_addr = { 0xff, 0xff, 0xff, 0xff,
 459                                       0xff, 0xff, 0xff, 0xff,
 460                                       0xff, 0xff, 0xff, 0xff,
 461                                       0xff, 0xff, 0xff, 0xff },
 462                         .proto = 0xff,
 463                 }
 464         };
 465
 466         rc = sfc_flow_parse_init(item,
 467                                  (const void **)&spec,
 468                                  (const void **)&mask,
 469                                  &supp_mask,
 470                                  &rte_flow_item_ipv6_mask,
 471                                  sizeof(struct rte_flow_item_ipv6),
 472                                  error);
 473         if (rc != 0)
 474                 return rc;
 475
 476         /*
 477          * Filtering by IPv6 source and destination addresses requires
 478          * the appropriate ETHER_TYPE in hardware filters
 479          */
 480         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 481                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 482                 efx_spec->efs_ether_type = ether_type_ipv6;
 483         } else if (efx_spec->efs_ether_type != ether_type_ipv6) {
 484                 rte_flow_error_set(error, EINVAL,
 485                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 486                         "Ethertype in pattern with IPV6 item should be appropriate");
 487                 return -rte_errno;
 488         }
 489
 490         if (spec == NULL)
 491                 return 0;
 492
 493         /*
 494          * IPv6 addresses are in big-endian byte order in item and in
 495          * efx_spec
 496          */
 497         if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr,
 498                    sizeof(mask->hdr.src_addr)) == 0) {
 499                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 500
 501                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) !=
 502                                  sizeof(spec->hdr.src_addr));
 503                 rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr,
 504                            sizeof(efx_spec->efs_rem_host));
 505         } else if (!sfc_flow_is_zero(mask->hdr.src_addr,
 506                                      sizeof(mask->hdr.src_addr))) {
 507                 goto fail_bad_mask;
 508         }
 509
 510         if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr,
 511                    sizeof(mask->hdr.dst_addr)) == 0) {
 512                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 513
 514                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) !=
 515                                  sizeof(spec->hdr.dst_addr));
 516                 rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr,
 517                            sizeof(efx_spec->efs_loc_host));
 518         } else if (!sfc_flow_is_zero(mask->hdr.dst_addr,
 519                                      sizeof(mask->hdr.dst_addr))) {
 520                 goto fail_bad_mask;
 521         }
 522
 523         if (mask->hdr.proto == supp_mask.hdr.proto) {
 524                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 525                 efx_spec->efs_ip_proto = spec->hdr.proto;
 526         } else if (mask->hdr.proto != 0) {
 527                 goto fail_bad_mask;
 528         }
 529
 530         return 0;
 531
 532 fail_bad_mask:
 533         rte_flow_error_set(error, EINVAL,
 534                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 535                            "Bad mask in the IPV6 pattern item");
 536         return -rte_errno;
 537 }
 538
 539 /**
 540  * Convert TCP item to EFX filter specification.
 541  *
 542  * @param item[in]
 543  *   Item specification. Only source and destination ports fields
 544  *   are supported. If the mask is NULL, default mask will be used.
 545  *   Ranging is not supported.
 546  * @param efx_spec[in, out]
 547  *   EFX filter specification to update.
 548  * @param[out] error
 549  *   Perform verbose error reporting if not NULL.
 550  */
 551 static int
 552 sfc_flow_parse_tcp(const struct rte_flow_item *item,
 553                    efx_filter_spec_t *efx_spec,
 554                    struct rte_flow_error *error)
 555 {
 556         int rc;
 557         const struct rte_flow_item_tcp *spec = NULL;
 558         const struct rte_flow_item_tcp *mask = NULL;
 559         const struct rte_flow_item_tcp supp_mask = {
 560                 .hdr = {
 561                         .src_port = 0xffff,
 562                         .dst_port = 0xffff,
 563                 }
 564         };
 565
 566         rc = sfc_flow_parse_init(item,
 567                                  (const void **)&spec,
 568                                  (const void **)&mask,
 569                                  &supp_mask,
 570                                  &rte_flow_item_tcp_mask,
 571                                  sizeof(struct rte_flow_item_tcp),
 572                                  error);
 573         if (rc != 0)
 574                 return rc;
 575
 576         /*
 577          * Filtering by TCP source and destination ports requires
 578          * the appropriate IP_PROTO in hardware filters
 579          */
 580         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 581                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 582                 efx_spec->efs_ip_proto = EFX_IPPROTO_TCP;
 583         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) {
 584                 rte_flow_error_set(error, EINVAL,
 585                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 586                         "IP proto in pattern with TCP item should be appropriate");
 587                 return -rte_errno;
 588         }
 589
 590         if (spec == NULL)
 591                 return 0;
 592
 593         /*
 594          * Source and destination ports are in big-endian byte order in item and
 595          * in little-endian in efx_spec, so byte swap is used
 596          */
 597         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 598                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 599                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 600         } else if (mask->hdr.src_port != 0) {
 601                 goto fail_bad_mask;
 602         }
 603
 604         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 605                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 606                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 607         } else if (mask->hdr.dst_port != 0) {
 608                 goto fail_bad_mask;
 609         }
 610
 611         return 0;
 612
 613 fail_bad_mask:
 614         rte_flow_error_set(error, EINVAL,
 615                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 616                            "Bad mask in the TCP pattern item");
 617         return -rte_errno;
 618 }
 619
 620 /**
 621  * Convert UDP item to EFX filter specification.
 622  *
 623  * @param item[in]
 624  *   Item specification. Only source and destination ports fields
 625  *   are supported. If the mask is NULL, default mask will be used.
 626  *   Ranging is not supported.
 627  * @param efx_spec[in, out]
 628  *   EFX filter specification to update.
 629  * @param[out] error
 630  *   Perform verbose error reporting if not NULL.
 631  */
 632 static int
 633 sfc_flow_parse_udp(const struct rte_flow_item *item,
 634                    efx_filter_spec_t *efx_spec,
 635                    struct rte_flow_error *error)
 636 {
 637         int rc;
 638         const struct rte_flow_item_udp *spec = NULL;
 639         const struct rte_flow_item_udp *mask = NULL;
 640         const struct rte_flow_item_udp supp_mask = {
 641                 .hdr = {
 642                         .src_port = 0xffff,
 643                         .dst_port = 0xffff,
 644                 }
 645         };
 646
 647         rc = sfc_flow_parse_init(item,
 648                                  (const void **)&spec,
 649                                  (const void **)&mask,
 650                                  &supp_mask,
 651                                  &rte_flow_item_udp_mask,
 652                                  sizeof(struct rte_flow_item_udp),
 653                                  error);
 654         if (rc != 0)
 655                 return rc;
 656
 657         /*
 658          * Filtering by UDP source and destination ports requires
 659          * the appropriate IP_PROTO in hardware filters
 660          */
 661         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 662                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 663                 efx_spec->efs_ip_proto = EFX_IPPROTO_UDP;
 664         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) {
 665                 rte_flow_error_set(error, EINVAL,
 666                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 667                         "IP proto in pattern with UDP item should be appropriate");
 668                 return -rte_errno;
 669         }
 670
 671         if (spec == NULL)
 672                 return 0;
 673
 674         /*
 675          * Source and destination ports are in big-endian byte order in item and
 676          * in little-endian in efx_spec, so byte swap is used
 677          */
 678         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 679                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 680                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 681         } else if (mask->hdr.src_port != 0) {
 682                 goto fail_bad_mask;
 683         }
 684
 685         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 686                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 687                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 688         } else if (mask->hdr.dst_port != 0) {
 689                 goto fail_bad_mask;
 690         }
 691
 692         return 0;
 693
 694 fail_bad_mask:
 695         rte_flow_error_set(error, EINVAL,
 696                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 697                            "Bad mask in the UDP pattern item");
 698         return -rte_errno;
 699 }
 700
 701 /*
 702  * Filters for encapsulated packets match based on the EtherType and IP
 703  * protocol in the outer frame.
 704  */
 705 static int
 706 sfc_flow_set_match_flags_for_encap_pkts(const struct rte_flow_item *item,
 707                                         efx_filter_spec_t *efx_spec,
 708                                         uint8_t ip_proto,
 709                                         struct rte_flow_error *error)
 710 {
 711         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 712                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 713                 efx_spec->efs_ip_proto = ip_proto;
 714         } else if (efx_spec->efs_ip_proto != ip_proto) {
 715                 switch (ip_proto) {
 716                 case EFX_IPPROTO_UDP:
 717                         rte_flow_error_set(error, EINVAL,
 718                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 719                                 "Outer IP header protocol must be UDP "
 720                                 "in VxLAN pattern");
 721                         return -rte_errno;
 722
 723                 case EFX_IPPROTO_GRE:
 724                         rte_flow_error_set(error, EINVAL,
 725                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 726                                 "Outer IP header protocol must be GRE "
 727                                 "in NVGRE pattern");
 728                         return -rte_errno;
 729
 730                 default:
 731                         rte_flow_error_set(error, EINVAL,
 732                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
 733                                 "Only VxLAN/NVGRE tunneling patterns "
 734                                 "are supported");
 735                         return -rte_errno;
 736                 }
 737         }
 738
 739         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 740                 rte_flow_error_set(error, EINVAL,
 741                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 742                         "Outer frame EtherType in pattern with tunneling "
 743                         "must be set");
 744                 return -rte_errno;
 745         } else if (efx_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4 &&
 746                    efx_spec->efs_ether_type != EFX_ETHER_TYPE_IPV6) {
 747                 rte_flow_error_set(error, EINVAL,
 748                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 749                         "Outer frame EtherType in pattern with tunneling "
 750                         "must be IPv4 or IPv6");
 751                 return -rte_errno;
 752         }
 753
 754         return 0;
 755 }
 756
 757 static int
 758 sfc_flow_set_efx_spec_vni_or_vsid(efx_filter_spec_t *efx_spec,
 759                                   const uint8_t *vni_or_vsid_val,
 760                                   const uint8_t *vni_or_vsid_mask,
 761                                   const struct rte_flow_item *item,
 762                                   struct rte_flow_error *error)
 763 {
 764         const uint8_t vni_or_vsid_full_mask[EFX_VNI_OR_VSID_LEN] = {
 765                 0xff, 0xff, 0xff
 766         };
 767
 768         if (memcmp(vni_or_vsid_mask, vni_or_vsid_full_mask,
 769                    EFX_VNI_OR_VSID_LEN) == 0) {
 770                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 771                 rte_memcpy(efx_spec->efs_vni_or_vsid, vni_or_vsid_val,
 772                            EFX_VNI_OR_VSID_LEN);
 773         } else if (!sfc_flow_is_zero(vni_or_vsid_mask, EFX_VNI_OR_VSID_LEN)) {
 774                 rte_flow_error_set(error, EINVAL,
 775                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 776                                    "Unsupported VNI/VSID mask");
 777                 return -rte_errno;
 778         }
 779
 780         return 0;
 781 }
 782
 783 /**
 784  * Convert VXLAN item to EFX filter specification.
 785  *
 786  * @param item[in]
 787  *   Item specification. Only VXLAN network identifier field is supported.
 788  *   If the mask is NULL, default mask will be used.
 789  *   Ranging is not supported.
 790  * @param efx_spec[in, out]
 791  *   EFX filter specification to update.
 792  * @param[out] error
 793  *   Perform verbose error reporting if not NULL.
 794  */
 795 static int
 796 sfc_flow_parse_vxlan(const struct rte_flow_item *item,
 797                      efx_filter_spec_t *efx_spec,
 798                      struct rte_flow_error *error)
 799 {
 800         int rc;
 801         const struct rte_flow_item_vxlan *spec = NULL;
 802         const struct rte_flow_item_vxlan *mask = NULL;
 803         const struct rte_flow_item_vxlan supp_mask = {
 804                 .vni = { 0xff, 0xff, 0xff }
 805         };
 806
 807         rc = sfc_flow_parse_init(item,
 808                                  (const void **)&spec,
 809                                  (const void **)&mask,
 810                                  &supp_mask,
 811                                  &rte_flow_item_vxlan_mask,
 812                                  sizeof(struct rte_flow_item_vxlan),
 813                                  error);
 814         if (rc != 0)
 815                 return rc;
 816
 817         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
 818                                                      EFX_IPPROTO_UDP, error);
 819         if (rc != 0)
 820                 return rc;
 821
 822         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_VXLAN;
 823         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 824
 825         if (spec == NULL)
 826                 return 0;
 827
 828         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->vni,
 829                                                mask->vni, item, error);
 830
 831         return rc;
 832 }
 833
 834 /**
 835  * Convert NVGRE item to EFX filter specification.
 836  *
 837  * @param item[in]
 838  *   Item specification. Only virtual subnet ID field is supported.
 839  *   If the mask is NULL, default mask will be used.
 840  *   Ranging is not supported.
 841  * @param efx_spec[in, out]
 842  *   EFX filter specification to update.
 843  * @param[out] error
 844  *   Perform verbose error reporting if not NULL.
 845  */
 846 static int
 847 sfc_flow_parse_nvgre(const struct rte_flow_item *item,
 848                      efx_filter_spec_t *efx_spec,
 849                      struct rte_flow_error *error)
 850 {
 851         int rc;
 852         const struct rte_flow_item_nvgre *spec = NULL;
 853         const struct rte_flow_item_nvgre *mask = NULL;
 854         const struct rte_flow_item_nvgre supp_mask = {
 855                 .tni = { 0xff, 0xff, 0xff }
 856         };
 857
 858         rc = sfc_flow_parse_init(item,
 859                                  (const void **)&spec,
 860                                  (const void **)&mask,
 861                                  &supp_mask,
 862                                  &rte_flow_item_nvgre_mask,
 863                                  sizeof(struct rte_flow_item_nvgre),
 864                                  error);
 865         if (rc != 0)
 866                 return rc;
 867
 868         rc = sfc_flow_set_match_flags_for_encap_pkts(item, efx_spec,
 869                                                      EFX_IPPROTO_GRE, error);
 870         if (rc != 0)
 871                 return rc;
 872
 873         efx_spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_NVGRE;
 874         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 875
 876         if (spec == NULL)
 877                 return 0;
 878
 879         rc = sfc_flow_set_efx_spec_vni_or_vsid(efx_spec, spec->tni,
 880                                                mask->tni, item, error);
 881
 882         return rc;
 883 }
 884
 885 static const struct sfc_flow_item sfc_flow_items[] = {
 886         {
 887                 .type = RTE_FLOW_ITEM_TYPE_VOID,
 888                 .prev_layer = SFC_FLOW_ITEM_ANY_LAYER,
 889                 .layer = SFC_FLOW_ITEM_ANY_LAYER,
 890                 .parse = sfc_flow_parse_void,
 891         },
 892         {
 893                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 894                 .prev_layer = SFC_FLOW_ITEM_START_LAYER,
 895                 .layer = SFC_FLOW_ITEM_L2,
 896                 .parse = sfc_flow_parse_eth,
 897         },
 898         {
 899                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 900                 .prev_layer = SFC_FLOW_ITEM_L2,
 901                 .layer = SFC_FLOW_ITEM_L2,
 902                 .parse = sfc_flow_parse_vlan,
 903         },
 904         {
 905                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 906                 .prev_layer = SFC_FLOW_ITEM_L2,
 907                 .layer = SFC_FLOW_ITEM_L3,
 908                 .parse = sfc_flow_parse_ipv4,
 909         },
 910         {
 911                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 912                 .prev_layer = SFC_FLOW_ITEM_L2,
 913                 .layer = SFC_FLOW_ITEM_L3,
 914                 .parse = sfc_flow_parse_ipv6,
 915         },
 916         {
 917                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 918                 .prev_layer = SFC_FLOW_ITEM_L3,
 919                 .layer = SFC_FLOW_ITEM_L4,
 920                 .parse = sfc_flow_parse_tcp,
 921         },
 922         {
 923                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 924                 .prev_layer = SFC_FLOW_ITEM_L3,
 925                 .layer = SFC_FLOW_ITEM_L4,
 926                 .parse = sfc_flow_parse_udp,
 927         },
 928         {
 929                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 930                 .prev_layer = SFC_FLOW_ITEM_L4,
 931                 .layer = SFC_FLOW_ITEM_START_LAYER,
 932                 .parse = sfc_flow_parse_vxlan,
 933         },
 934         {
 935                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
 936                 .prev_layer = SFC_FLOW_ITEM_L3,
 937                 .layer = SFC_FLOW_ITEM_START_LAYER,
 938                 .parse = sfc_flow_parse_nvgre,
 939         },
 940 };
 941
 942 /*
 943  * Protocol-independent flow API support
 944  */
 945 static int
 946 sfc_flow_parse_attr(const struct rte_flow_attr *attr,
 947                     struct rte_flow *flow,
 948                     struct rte_flow_error *error)
 949 {
 950         if (attr == NULL) {
 951                 rte_flow_error_set(error, EINVAL,
 952                                    RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 953                                    "NULL attribute");
 954                 return -rte_errno;
 955         }
 956         if (attr->group != 0) {
 957                 rte_flow_error_set(error, ENOTSUP,
 958                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
 959                                    "Groups are not supported");
 960                 return -rte_errno;
 961         }
 962         if (attr->priority != 0) {
 963                 rte_flow_error_set(error, ENOTSUP,
 964                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
 965                                    "Priorities are not supported");
 966                 return -rte_errno;
 967         }
 968         if (attr->egress != 0) {
 969                 rte_flow_error_set(error, ENOTSUP,
 970                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
 971                                    "Egress is not supported");
 972                 return -rte_errno;
 973         }
 974         if (attr->ingress == 0) {
 975                 rte_flow_error_set(error, ENOTSUP,
 976                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
 977                                    "Only ingress is supported");
 978                 return -rte_errno;
 979         }
 980
 981         flow->spec.efs_flags |= EFX_FILTER_FLAG_RX;
 982         flow->spec.efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 983
 984         return 0;
 985 }
 986
 987 /* Get item from array sfc_flow_items */
 988 static const struct sfc_flow_item *
 989 sfc_flow_get_item(enum rte_flow_item_type type)
 990 {
 991         unsigned int i;
 992
 993         for (i = 0; i < RTE_DIM(sfc_flow_items); i++)
 994                 if (sfc_flow_items[i].type == type)
 995                         return &sfc_flow_items[i];
 996
 997         return NULL;
 998 }
 999
1000 static int
1001 sfc_flow_parse_pattern(const struct rte_flow_item pattern[],
1002                        struct rte_flow *flow,
1003                        struct rte_flow_error *error)
1004 {
1005         int rc;
1006         unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER;
1007         boolean_t is_ifrm = B_FALSE;
1008         const struct sfc_flow_item *item;
1009
1010         if (pattern == NULL) {
1011                 rte_flow_error_set(error, EINVAL,
1012                                    RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
1013                                    "NULL pattern");
1014                 return -rte_errno;
1015         }
1016
1017         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
1018                 item = sfc_flow_get_item(pattern->type);
1019                 if (item == NULL) {
1020                         rte_flow_error_set(error, ENOTSUP,
1021                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
1022                                            "Unsupported pattern item");
1023                         return -rte_errno;
1024                 }
1025
1026                 /*
1027                  * Omitting one or several protocol layers at the beginning
1028                  * of pattern is supported
1029                  */
1030                 if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
1031                     prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
1032                     item->prev_layer != prev_layer) {
1033                         rte_flow_error_set(error, ENOTSUP,
1034                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
1035                                            "Unexpected sequence of pattern items");
1036                         return -rte_errno;
1037                 }
1038
1039                 /*
1040                  * Allow only VOID pattern item in the inner frame.
1041                  * Also check that there is only one tunneling protocol.
1042                  */
1043                 switch (item->type) {
1044                 case RTE_FLOW_ITEM_TYPE_VOID:
1045                         break;
1046
1047                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1048                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1049                         if (is_ifrm) {
1050                                 rte_flow_error_set(error, EINVAL,
1051                                         RTE_FLOW_ERROR_TYPE_ITEM,
1052                                         pattern,
1053                                         "More than one tunneling protocol");
1054                                 return -rte_errno;
1055                         }
1056                         is_ifrm = B_TRUE;
1057                         break;
1058
1059                 default:
1060                         if (is_ifrm) {
1061                                 rte_flow_error_set(error, EINVAL,
1062                                         RTE_FLOW_ERROR_TYPE_ITEM,
1063                                         pattern,
1064                                         "There is an unsupported pattern item "
1065                                         "in the inner frame");
1066                                 return -rte_errno;
1067                         }
1068                         break;
1069                 }
1070
1071                 rc = item->parse(pattern, &flow->spec, error);
1072                 if (rc != 0)
1073                         return rc;
1074
1075                 if (item->layer != SFC_FLOW_ITEM_ANY_LAYER)
1076                         prev_layer = item->layer;
1077         }
1078
1079         return 0;
1080 }
1081
1082 static int
1083 sfc_flow_parse_queue(struct sfc_adapter *sa,
1084                      const struct rte_flow_action_queue *queue,
1085                      struct rte_flow *flow)
1086 {
1087         struct sfc_rxq *rxq;
1088
1089         if (queue->index >= sa->rxq_count)
1090                 return -EINVAL;
1091
1092         rxq = sa->rxq_info[queue->index].rxq;
1093         flow->spec.efs_dmaq_id = (uint16_t)rxq->hw_index;
1094
1095         return 0;
1096 }
1097
1098 #if EFSYS_OPT_RX_SCALE
1099 static int
1100 sfc_flow_parse_rss(struct sfc_adapter *sa,
1101                    const struct rte_flow_action_rss *rss,
1102                    struct rte_flow *flow)
1103 {
1104         unsigned int rxq_sw_index;
1105         struct sfc_rxq *rxq;
1106         unsigned int rxq_hw_index_min;
1107         unsigned int rxq_hw_index_max;
1108         const struct rte_eth_rss_conf *rss_conf = rss->rss_conf;
1109         uint64_t rss_hf;
1110         uint8_t *rss_key = NULL;
1111         struct sfc_flow_rss *sfc_rss_conf = &flow->rss_conf;
1112         unsigned int i;
1113
1114         if (rss->num == 0)
1115                 return -EINVAL;
1116
1117         rxq_sw_index = sa->rxq_count - 1;
1118         rxq = sa->rxq_info[rxq_sw_index].rxq;
1119         rxq_hw_index_min = rxq->hw_index;
1120         rxq_hw_index_max = 0;
1121
1122         for (i = 0; i < rss->num; ++i) {
1123                 rxq_sw_index = rss->queue[i];
1124
1125                 if (rxq_sw_index >= sa->rxq_count)
1126                         return -EINVAL;
1127
1128                 rxq = sa->rxq_info[rxq_sw_index].rxq;
1129
1130                 if (rxq->hw_index < rxq_hw_index_min)
1131                         rxq_hw_index_min = rxq->hw_index;
1132
1133                 if (rxq->hw_index > rxq_hw_index_max)
1134                         rxq_hw_index_max = rxq->hw_index;
1135         }
1136
1137         rss_hf = (rss_conf != NULL) ? rss_conf->rss_hf : SFC_RSS_OFFLOADS;
1138         if ((rss_hf & ~SFC_RSS_OFFLOADS) != 0)
1139                 return -EINVAL;
1140
1141         if (rss_conf != NULL) {
1142                 if (rss_conf->rss_key_len != sizeof(sa->rss_key))
1143                         return -EINVAL;
1144
1145                 rss_key = rss_conf->rss_key;
1146         } else {
1147                 rss_key = sa->rss_key;
1148         }
1149
1150         flow->rss = B_TRUE;
1151
1152         sfc_rss_conf->rxq_hw_index_min = rxq_hw_index_min;
1153         sfc_rss_conf->rxq_hw_index_max = rxq_hw_index_max;
1154         sfc_rss_conf->rss_hash_types = sfc_rte_to_efx_hash_type(rss_hf);
1155         rte_memcpy(sfc_rss_conf->rss_key, rss_key, sizeof(sa->rss_key));
1156
1157         for (i = 0; i < RTE_DIM(sfc_rss_conf->rss_tbl); ++i) {
1158                 unsigned int rxq_sw_index = rss->queue[i % rss->num];
1159                 struct sfc_rxq *rxq = sa->rxq_info[rxq_sw_index].rxq;
1160
1161                 sfc_rss_conf->rss_tbl[i] = rxq->hw_index - rxq_hw_index_min;
1162         }
1163
1164         return 0;
1165 }
1166 #endif /* EFSYS_OPT_RX_SCALE */
1167
1168 static int
1169 sfc_flow_filter_insert(struct sfc_adapter *sa,
1170                        struct rte_flow *flow)
1171 {
1172         efx_filter_spec_t *spec = &flow->spec;
1173
1174 #if EFSYS_OPT_RX_SCALE
1175         struct sfc_flow_rss *rss = &flow->rss_conf;
1176         int rc = 0;
1177
1178         if (flow->rss) {
1179                 unsigned int rss_spread = MIN(rss->rxq_hw_index_max -
1180                                               rss->rxq_hw_index_min + 1,
1181                                               EFX_MAXRSS);
1182
1183                 rc = efx_rx_scale_context_alloc(sa->nic,
1184                                                 EFX_RX_SCALE_EXCLUSIVE,
1185                                                 rss_spread,
1186                                                 &spec->efs_rss_context);
1187                 if (rc != 0)
1188                         goto fail_scale_context_alloc;
1189
1190                 rc = efx_rx_scale_mode_set(sa->nic, spec->efs_rss_context,
1191                                            EFX_RX_HASHALG_TOEPLITZ,
1192                                            rss->rss_hash_types, B_TRUE);
1193                 if (rc != 0)
1194                         goto fail_scale_mode_set;
1195
1196                 rc = efx_rx_scale_key_set(sa->nic, spec->efs_rss_context,
1197                                           rss->rss_key,
1198                                           sizeof(sa->rss_key));
1199                 if (rc != 0)
1200                         goto fail_scale_key_set;
1201
1202                 spec->efs_dmaq_id = rss->rxq_hw_index_min;
1203                 spec->efs_flags |= EFX_FILTER_FLAG_RX_RSS;
1204         }
1205
1206         rc = efx_filter_insert(sa->nic, spec);
1207         if (rc != 0)
1208                 goto fail_filter_insert;
1209
1210         if (flow->rss) {
1211                 /*
1212                  * Scale table is set after filter insertion because
1213                  * the table entries are relative to the base RxQ ID
1214                  * and the latter is submitted to the HW by means of
1215                  * inserting a filter, so by the time of the request
1216                  * the HW knows all the information needed to verify
1217                  * the table entries, and the operation will succeed
1218                  */
1219                 rc = efx_rx_scale_tbl_set(sa->nic, spec->efs_rss_context,
1220                                           rss->rss_tbl, RTE_DIM(rss->rss_tbl));
1221                 if (rc != 0)
1222                         goto fail_scale_tbl_set;
1223         }
1224
1225         return 0;
1226
1227 fail_scale_tbl_set:
1228         efx_filter_remove(sa->nic, spec);
1229
1230 fail_filter_insert:
1231 fail_scale_key_set:
1232 fail_scale_mode_set:
1233         if (flow->rss)
1234                 efx_rx_scale_context_free(sa->nic, spec->efs_rss_context);
1235
1236 fail_scale_context_alloc:
1237         return rc;
1238 #else /* !EFSYS_OPT_RX_SCALE */
1239         return efx_filter_insert(sa->nic, spec);
1240 #endif /* EFSYS_OPT_RX_SCALE */
1241 }
1242
1243 static int
1244 sfc_flow_filter_remove(struct sfc_adapter *sa,
1245                        struct rte_flow *flow)
1246 {
1247         efx_filter_spec_t *spec = &flow->spec;
1248         int rc = 0;
1249
1250         rc = efx_filter_remove(sa->nic, spec);
1251         if (rc != 0)
1252                 return rc;
1253
1254 #if EFSYS_OPT_RX_SCALE
1255         if (flow->rss)
1256                 rc = efx_rx_scale_context_free(sa->nic, spec->efs_rss_context);
1257 #endif /* EFSYS_OPT_RX_SCALE */
1258
1259         return rc;
1260 }
1261
1262 static int
1263 sfc_flow_parse_actions(struct sfc_adapter *sa,
1264                        const struct rte_flow_action actions[],
1265                        struct rte_flow *flow,
1266                        struct rte_flow_error *error)
1267 {
1268         int rc;
1269         boolean_t is_specified = B_FALSE;
1270
1271         if (actions == NULL) {
1272                 rte_flow_error_set(error, EINVAL,
1273                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
1274                                    "NULL actions");
1275                 return -rte_errno;
1276         }
1277
1278         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1279                 switch (actions->type) {
1280                 case RTE_FLOW_ACTION_TYPE_VOID:
1281                         break;
1282
1283                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1284                         rc = sfc_flow_parse_queue(sa, actions->conf, flow);
1285                         if (rc != 0) {
1286                                 rte_flow_error_set(error, EINVAL,
1287                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1288                                         "Bad QUEUE action");
1289                                 return -rte_errno;
1290                         }
1291
1292                         is_specified = B_TRUE;
1293                         break;
1294
1295 #if EFSYS_OPT_RX_SCALE
1296                 case RTE_FLOW_ACTION_TYPE_RSS:
1297                         rc = sfc_flow_parse_rss(sa, actions->conf, flow);
1298                         if (rc != 0) {
1299                                 rte_flow_error_set(error, rc,
1300                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
1301                                         "Bad RSS action");
1302                                 return -rte_errno;
1303                         }
1304
1305                         is_specified = B_TRUE;
1306                         break;
1307 #endif /* EFSYS_OPT_RX_SCALE */
1308
1309                 default:
1310                         rte_flow_error_set(error, ENOTSUP,
1311                                            RTE_FLOW_ERROR_TYPE_ACTION, actions,
1312                                            "Action is not supported");
1313                         return -rte_errno;
1314                 }
1315         }
1316
1317         if (!is_specified) {
1318                 rte_flow_error_set(error, EINVAL,
1319                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, actions,
1320                                    "Action is unspecified");
1321                 return -rte_errno;
1322         }
1323
1324         return 0;
1325 }
1326
1327 static int
1328 sfc_flow_parse(struct rte_eth_dev *dev,
1329                const struct rte_flow_attr *attr,
1330                const struct rte_flow_item pattern[],
1331                const struct rte_flow_action actions[],
1332                struct rte_flow *flow,
1333                struct rte_flow_error *error)
1334 {
1335         struct sfc_adapter *sa = dev->data->dev_private;
1336         int rc;
1337
1338         rc = sfc_flow_parse_attr(attr, flow, error);
1339         if (rc != 0)
1340                 goto fail_bad_value;
1341
1342         rc = sfc_flow_parse_pattern(pattern, flow, error);
1343         if (rc != 0)
1344                 goto fail_bad_value;
1345
1346         rc = sfc_flow_parse_actions(sa, actions, flow, error);
1347         if (rc != 0)
1348                 goto fail_bad_value;
1349
1350         if (!sfc_filter_is_match_supported(sa, flow->spec.efs_match_flags)) {
1351                 rte_flow_error_set(error, ENOTSUP,
1352                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1353                                    "Flow rule pattern is not supported");
1354                 return -rte_errno;
1355         }
1356
1357 fail_bad_value:
1358         return rc;
1359 }
1360
1361 static int
1362 sfc_flow_validate(struct rte_eth_dev *dev,
1363                   const struct rte_flow_attr *attr,
1364                   const struct rte_flow_item pattern[],
1365                   const struct rte_flow_action actions[],
1366                   struct rte_flow_error *error)
1367 {
1368         struct rte_flow flow;
1369
1370         memset(&flow, 0, sizeof(flow));
1371
1372         return sfc_flow_parse(dev, attr, pattern, actions, &flow, error);
1373 }
1374
1375 static struct rte_flow *
1376 sfc_flow_create(struct rte_eth_dev *dev,
1377                 const struct rte_flow_attr *attr,
1378                 const struct rte_flow_item pattern[],
1379                 const struct rte_flow_action actions[],
1380                 struct rte_flow_error *error)
1381 {
1382         struct sfc_adapter *sa = dev->data->dev_private;
1383         struct rte_flow *flow = NULL;
1384         int rc;
1385
1386         flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0);
1387         if (flow == NULL) {
1388                 rte_flow_error_set(error, ENOMEM,
1389                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1390                                    "Failed to allocate memory");
1391                 goto fail_no_mem;
1392         }
1393
1394         rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error);
1395         if (rc != 0)
1396                 goto fail_bad_value;
1397
1398         TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries);
1399
1400         sfc_adapter_lock(sa);
1401
1402         if (sa->state == SFC_ADAPTER_STARTED) {
1403                 rc = sfc_flow_filter_insert(sa, flow);
1404                 if (rc != 0) {
1405                         rte_flow_error_set(error, rc,
1406                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1407                                 "Failed to insert filter");
1408                         goto fail_filter_insert;
1409                 }
1410         }
1411
1412         sfc_adapter_unlock(sa);
1413
1414         return flow;
1415
1416 fail_filter_insert:
1417         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1418
1419 fail_bad_value:
1420         rte_free(flow);
1421         sfc_adapter_unlock(sa);
1422
1423 fail_no_mem:
1424         return NULL;
1425 }
1426
1427 static int
1428 sfc_flow_remove(struct sfc_adapter *sa,
1429                 struct rte_flow *flow,
1430                 struct rte_flow_error *error)
1431 {
1432         int rc = 0;
1433
1434         SFC_ASSERT(sfc_adapter_is_locked(sa));
1435
1436         if (sa->state == SFC_ADAPTER_STARTED) {
1437                 rc = sfc_flow_filter_remove(sa, flow);
1438                 if (rc != 0)
1439                         rte_flow_error_set(error, rc,
1440                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1441                                 "Failed to destroy flow rule");
1442         }
1443
1444         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1445         rte_free(flow);
1446
1447         return rc;
1448 }
1449
1450 static int
1451 sfc_flow_destroy(struct rte_eth_dev *dev,
1452                  struct rte_flow *flow,
1453                  struct rte_flow_error *error)
1454 {
1455         struct sfc_adapter *sa = dev->data->dev_private;
1456         struct rte_flow *flow_ptr;
1457         int rc = EINVAL;
1458
1459         sfc_adapter_lock(sa);
1460
1461         TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) {
1462                 if (flow_ptr == flow)
1463                         rc = 0;
1464         }
1465         if (rc != 0) {
1466                 rte_flow_error_set(error, rc,
1467                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1468                                    "Failed to find flow rule to destroy");
1469                 goto fail_bad_value;
1470         }
1471
1472         rc = sfc_flow_remove(sa, flow, error);
1473
1474 fail_bad_value:
1475         sfc_adapter_unlock(sa);
1476
1477         return -rc;
1478 }
1479
1480 static int
1481 sfc_flow_flush(struct rte_eth_dev *dev,
1482                struct rte_flow_error *error)
1483 {
1484         struct sfc_adapter *sa = dev->data->dev_private;
1485         struct rte_flow *flow;
1486         int rc = 0;
1487         int ret = 0;
1488
1489         sfc_adapter_lock(sa);
1490
1491         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1492                 rc = sfc_flow_remove(sa, flow, error);
1493                 if (rc != 0)
1494                         ret = rc;
1495         }
1496
1497         sfc_adapter_unlock(sa);
1498
1499         return -ret;
1500 }
1501
1502 static int
1503 sfc_flow_isolate(struct rte_eth_dev *dev, int enable,
1504                  struct rte_flow_error *error)
1505 {
1506         struct sfc_adapter *sa = dev->data->dev_private;
1507         struct sfc_port *port = &sa->port;
1508         int ret = 0;
1509
1510         sfc_adapter_lock(sa);
1511         if (sa->state != SFC_ADAPTER_INITIALIZED) {
1512                 rte_flow_error_set(error, EBUSY,
1513                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1514                                    NULL, "please close the port first");
1515                 ret = -rte_errno;
1516         } else {
1517                 port->isolated = (enable) ? B_TRUE : B_FALSE;
1518         }
1519         sfc_adapter_unlock(sa);
1520
1521         return ret;
1522 }
1523
1524 const struct rte_flow_ops sfc_flow_ops = {
1525         .validate = sfc_flow_validate,
1526         .create = sfc_flow_create,
1527         .destroy = sfc_flow_destroy,
1528         .flush = sfc_flow_flush,
1529         .query = NULL,
1530         .isolate = sfc_flow_isolate,
1531 };
1532
1533 void
1534 sfc_flow_init(struct sfc_adapter *sa)
1535 {
1536         SFC_ASSERT(sfc_adapter_is_locked(sa));
1537
1538         TAILQ_INIT(&sa->filter.flow_list);
1539 }
1540
1541 void
1542 sfc_flow_fini(struct sfc_adapter *sa)
1543 {
1544         struct rte_flow *flow;
1545
1546         SFC_ASSERT(sfc_adapter_is_locked(sa));
1547
1548         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1549                 TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1550                 rte_free(flow);
1551         }
1552 }
1553
1554 void
1555 sfc_flow_stop(struct sfc_adapter *sa)
1556 {
1557         struct rte_flow *flow;
1558
1559         SFC_ASSERT(sfc_adapter_is_locked(sa));
1560
1561         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries)
1562                 sfc_flow_filter_remove(sa, flow);
1563 }
1564
1565 int
1566 sfc_flow_start(struct sfc_adapter *sa)
1567 {
1568         struct rte_flow *flow;
1569         int rc = 0;
1570
1571         sfc_log_init(sa, "entry");
1572
1573         SFC_ASSERT(sfc_adapter_is_locked(sa));
1574
1575         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) {
1576                 rc = sfc_flow_filter_insert(sa, flow);
1577                 if (rc != 0)
1578                         goto fail_bad_flow;
1579         }
1580
1581         sfc_log_init(sa, "done");
1582
1583 fail_bad_flow:
1584         return rc;
1585 }