drivers/net/sfc/sfc_flow.c

   1 /*-
   2  * Copyright (c) 2017 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * This software was jointly developed between OKTET Labs (under contract
   6  * for Solarflare) and Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright notice,
  12  *    this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright notice,
  14  *    this list of conditions and the following disclaimer in the documentation
  15  *    and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 #include <rte_tailq.h>
  31 #include <rte_common.h>
  32 #include <rte_ethdev.h>
  33 #include <rte_eth_ctrl.h>
  34 #include <rte_ether.h>
  35 #include <rte_flow.h>
  36 #include <rte_flow_driver.h>
  37
  38 #include "efx.h"
  39
  40 #include "sfc.h"
  41 #include "sfc_rx.h"
  42 #include "sfc_filter.h"
  43 #include "sfc_flow.h"
  44 #include "sfc_log.h"
  45
  46 /*
  47  * At now flow API is implemented in such a manner that each
  48  * flow rule is converted to a hardware filter.
  49  * All elements of flow rule (attributes, pattern items, actions)
  50  * correspond to one or more fields in the efx_filter_spec_s structure
  51  * that is responsible for the hardware filter.
  52  */
  53
  54 enum sfc_flow_item_layers {
  55         SFC_FLOW_ITEM_ANY_LAYER,
  56         SFC_FLOW_ITEM_START_LAYER,
  57         SFC_FLOW_ITEM_L2,
  58         SFC_FLOW_ITEM_L3,
  59         SFC_FLOW_ITEM_L4,
  60 };
  61
  62 typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item,
  63                                   efx_filter_spec_t *spec,
  64                                   struct rte_flow_error *error);
  65
  66 struct sfc_flow_item {
  67         enum rte_flow_item_type type;           /* Type of item */
  68         enum sfc_flow_item_layers layer;        /* Layer of item */
  69         enum sfc_flow_item_layers prev_layer;   /* Previous layer of item */
  70         sfc_flow_item_parse *parse;             /* Parsing function */
  71 };
  72
  73 static sfc_flow_item_parse sfc_flow_parse_void;
  74 static sfc_flow_item_parse sfc_flow_parse_eth;
  75 static sfc_flow_item_parse sfc_flow_parse_vlan;
  76 static sfc_flow_item_parse sfc_flow_parse_ipv4;
  77 static sfc_flow_item_parse sfc_flow_parse_ipv6;
  78 static sfc_flow_item_parse sfc_flow_parse_tcp;
  79 static sfc_flow_item_parse sfc_flow_parse_udp;
  80
  81 static boolean_t
  82 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
  83 {
  84         uint8_t sum = 0;
  85         unsigned int i;
  86
  87         for (i = 0; i < size; i++)
  88                 sum |= buf[i];
  89
  90         return (sum == 0) ? B_TRUE : B_FALSE;
  91 }
  92
  93 /*
  94  * Validate item and prepare structures spec and mask for parsing
  95  */
  96 static int
  97 sfc_flow_parse_init(const struct rte_flow_item *item,
  98                     const void **spec_ptr,
  99                     const void **mask_ptr,
 100                     const void *supp_mask,
 101                     const void *def_mask,
 102                     unsigned int size,
 103                     struct rte_flow_error *error)
 104 {
 105         const uint8_t *spec;
 106         const uint8_t *mask;
 107         const uint8_t *last;
 108         uint8_t match;
 109         uint8_t supp;
 110         unsigned int i;
 111
 112         if (item == NULL) {
 113                 rte_flow_error_set(error, EINVAL,
 114                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 115                                    "NULL item");
 116                 return -rte_errno;
 117         }
 118
 119         if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) {
 120                 rte_flow_error_set(error, EINVAL,
 121                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 122                                    "Mask or last is set without spec");
 123                 return -rte_errno;
 124         }
 125
 126         /*
 127          * If "mask" is not set, default mask is used,
 128          * but if default mask is NULL, "mask" should be set
 129          */
 130         if (item->mask == NULL) {
 131                 if (def_mask == NULL) {
 132                         rte_flow_error_set(error, EINVAL,
 133                                 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 134                                 "Mask should be specified");
 135                         return -rte_errno;
 136                 }
 137
 138                 mask = (const uint8_t *)def_mask;
 139         } else {
 140                 mask = (const uint8_t *)item->mask;
 141         }
 142
 143         spec = (const uint8_t *)item->spec;
 144         last = (const uint8_t *)item->last;
 145
 146         if (spec == NULL)
 147                 goto exit;
 148
 149         /*
 150          * If field values in "last" are either 0 or equal to the corresponding
 151          * values in "spec" then they are ignored
 152          */
 153         if (last != NULL &&
 154             !sfc_flow_is_zero(last, size) &&
 155             memcmp(last, spec, size) != 0) {
 156                 rte_flow_error_set(error, ENOTSUP,
 157                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 158                                    "Ranging is not supported");
 159                 return -rte_errno;
 160         }
 161
 162         if (supp_mask == NULL) {
 163                 rte_flow_error_set(error, EINVAL,
 164                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 165                         "Supported mask for item should be specified");
 166                 return -rte_errno;
 167         }
 168
 169         /* Check that mask and spec not asks for more match than supp_mask */
 170         for (i = 0; i < size; i++) {
 171                 match = spec[i] | mask[i];
 172                 supp = ((const uint8_t *)supp_mask)[i];
 173
 174                 if ((match | supp) != supp) {
 175                         rte_flow_error_set(error, ENOTSUP,
 176                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 177                                            "Item's field is not supported");
 178                         return -rte_errno;
 179                 }
 180         }
 181
 182 exit:
 183         *spec_ptr = spec;
 184         *mask_ptr = mask;
 185         return 0;
 186 }
 187
 188 /*
 189  * Protocol parsers.
 190  * Masking is not supported, so masks in items should be either
 191  * full or empty (zeroed) and set only for supported fields which
 192  * are specified in the supp_mask.
 193  */
 194
 195 static int
 196 sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item,
 197                     __rte_unused efx_filter_spec_t *efx_spec,
 198                     __rte_unused struct rte_flow_error *error)
 199 {
 200         return 0;
 201 }
 202
 203 /**
 204  * Convert Ethernet item to EFX filter specification.
 205  *
 206  * @param item[in]
 207  *   Item specification. Only source and destination addresses and
 208  *   Ethernet type fields are supported. If the mask is NULL, default
 209  *   mask will be used. Ranging is not supported.
 210  * @param efx_spec[in, out]
 211  *   EFX filter specification to update.
 212  * @param[out] error
 213  *   Perform verbose error reporting if not NULL.
 214  */
 215 static int
 216 sfc_flow_parse_eth(const struct rte_flow_item *item,
 217                    efx_filter_spec_t *efx_spec,
 218                    struct rte_flow_error *error)
 219 {
 220         int rc;
 221         const struct rte_flow_item_eth *spec = NULL;
 222         const struct rte_flow_item_eth *mask = NULL;
 223         const struct rte_flow_item_eth supp_mask = {
 224                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 225                 .src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 226                 .type = 0xffff,
 227         };
 228
 229         rc = sfc_flow_parse_init(item,
 230                                  (const void **)&spec,
 231                                  (const void **)&mask,
 232                                  &supp_mask,
 233                                  &rte_flow_item_eth_mask,
 234                                  sizeof(struct rte_flow_item_eth),
 235                                  error);
 236         if (rc != 0)
 237                 return rc;
 238
 239         /* If "spec" is not set, could be any Ethernet */
 240         if (spec == NULL)
 241                 return 0;
 242
 243         if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) {
 244                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 245                 rte_memcpy(efx_spec->efs_loc_mac, spec->dst.addr_bytes,
 246                            EFX_MAC_ADDR_LEN);
 247         } else if (!is_zero_ether_addr(&mask->dst)) {
 248                 goto fail_bad_mask;
 249         }
 250
 251         if (is_same_ether_addr(&mask->src, &supp_mask.src)) {
 252                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC;
 253                 rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes,
 254                            EFX_MAC_ADDR_LEN);
 255         } else if (!is_zero_ether_addr(&mask->src)) {
 256                 goto fail_bad_mask;
 257         }
 258
 259         /*
 260          * Ether type is in big-endian byte order in item and
 261          * in little-endian in efx_spec, so byte swap is used
 262          */
 263         if (mask->type == supp_mask.type) {
 264                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 265                 efx_spec->efs_ether_type = rte_bswap16(spec->type);
 266         } else if (mask->type != 0) {
 267                 goto fail_bad_mask;
 268         }
 269
 270         return 0;
 271
 272 fail_bad_mask:
 273         rte_flow_error_set(error, EINVAL,
 274                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 275                            "Bad mask in the ETH pattern item");
 276         return -rte_errno;
 277 }
 278
 279 /**
 280  * Convert VLAN item to EFX filter specification.
 281  *
 282  * @param item[in]
 283  *   Item specification. Only VID field is supported.
 284  *   The mask can not be NULL. Ranging is not supported.
 285  * @param efx_spec[in, out]
 286  *   EFX filter specification to update.
 287  * @param[out] error
 288  *   Perform verbose error reporting if not NULL.
 289  */
 290 static int
 291 sfc_flow_parse_vlan(const struct rte_flow_item *item,
 292                     efx_filter_spec_t *efx_spec,
 293                     struct rte_flow_error *error)
 294 {
 295         int rc;
 296         uint16_t vid;
 297         const struct rte_flow_item_vlan *spec = NULL;
 298         const struct rte_flow_item_vlan *mask = NULL;
 299         const struct rte_flow_item_vlan supp_mask = {
 300                 .tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX),
 301         };
 302
 303         rc = sfc_flow_parse_init(item,
 304                                  (const void **)&spec,
 305                                  (const void **)&mask,
 306                                  &supp_mask,
 307                                  NULL,
 308                                  sizeof(struct rte_flow_item_vlan),
 309                                  error);
 310         if (rc != 0)
 311                 return rc;
 312
 313         /*
 314          * VID is in big-endian byte order in item and
 315          * in little-endian in efx_spec, so byte swap is used.
 316          * If two VLAN items are included, the first matches
 317          * the outer tag and the next matches the inner tag.
 318          */
 319         if (mask->tci == supp_mask.tci) {
 320                 vid = rte_bswap16(spec->tci);
 321
 322                 if (!(efx_spec->efs_match_flags &
 323                       EFX_FILTER_MATCH_OUTER_VID)) {
 324                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 325                         efx_spec->efs_outer_vid = vid;
 326                 } else if (!(efx_spec->efs_match_flags &
 327                              EFX_FILTER_MATCH_INNER_VID)) {
 328                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID;
 329                         efx_spec->efs_inner_vid = vid;
 330                 } else {
 331                         rte_flow_error_set(error, EINVAL,
 332                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 333                                            "More than two VLAN items");
 334                         return -rte_errno;
 335                 }
 336         } else {
 337                 rte_flow_error_set(error, EINVAL,
 338                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 339                                    "VLAN ID in TCI match is required");
 340                 return -rte_errno;
 341         }
 342
 343         return 0;
 344 }
 345
 346 /**
 347  * Convert IPv4 item to EFX filter specification.
 348  *
 349  * @param item[in]
 350  *   Item specification. Only source and destination addresses and
 351  *   protocol fields are supported. If the mask is NULL, default
 352  *   mask will be used. Ranging is not supported.
 353  * @param efx_spec[in, out]
 354  *   EFX filter specification to update.
 355  * @param[out] error
 356  *   Perform verbose error reporting if not NULL.
 357  */
 358 static int
 359 sfc_flow_parse_ipv4(const struct rte_flow_item *item,
 360                     efx_filter_spec_t *efx_spec,
 361                     struct rte_flow_error *error)
 362 {
 363         int rc;
 364         const struct rte_flow_item_ipv4 *spec = NULL;
 365         const struct rte_flow_item_ipv4 *mask = NULL;
 366         const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4);
 367         const struct rte_flow_item_ipv4 supp_mask = {
 368                 .hdr = {
 369                         .src_addr = 0xffffffff,
 370                         .dst_addr = 0xffffffff,
 371                         .next_proto_id = 0xff,
 372                 }
 373         };
 374
 375         rc = sfc_flow_parse_init(item,
 376                                  (const void **)&spec,
 377                                  (const void **)&mask,
 378                                  &supp_mask,
 379                                  &rte_flow_item_ipv4_mask,
 380                                  sizeof(struct rte_flow_item_ipv4),
 381                                  error);
 382         if (rc != 0)
 383                 return rc;
 384
 385         /*
 386          * Filtering by IPv4 source and destination addresses requires
 387          * the appropriate ETHER_TYPE in hardware filters
 388          */
 389         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 390                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 391                 efx_spec->efs_ether_type = ether_type_ipv4;
 392         } else if (efx_spec->efs_ether_type != ether_type_ipv4) {
 393                 rte_flow_error_set(error, EINVAL,
 394                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 395                         "Ethertype in pattern with IPV4 item should be appropriate");
 396                 return -rte_errno;
 397         }
 398
 399         if (spec == NULL)
 400                 return 0;
 401
 402         /*
 403          * IPv4 addresses are in big-endian byte order in item and in
 404          * efx_spec
 405          */
 406         if (mask->hdr.src_addr == supp_mask.hdr.src_addr) {
 407                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 408                 efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr;
 409         } else if (mask->hdr.src_addr != 0) {
 410                 goto fail_bad_mask;
 411         }
 412
 413         if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) {
 414                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 415                 efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr;
 416         } else if (mask->hdr.dst_addr != 0) {
 417                 goto fail_bad_mask;
 418         }
 419
 420         if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) {
 421                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 422                 efx_spec->efs_ip_proto = spec->hdr.next_proto_id;
 423         } else if (mask->hdr.next_proto_id != 0) {
 424                 goto fail_bad_mask;
 425         }
 426
 427         return 0;
 428
 429 fail_bad_mask:
 430         rte_flow_error_set(error, EINVAL,
 431                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 432                            "Bad mask in the IPV4 pattern item");
 433         return -rte_errno;
 434 }
 435
 436 /**
 437  * Convert IPv6 item to EFX filter specification.
 438  *
 439  * @param item[in]
 440  *   Item specification. Only source and destination addresses and
 441  *   next header fields are supported. If the mask is NULL, default
 442  *   mask will be used. Ranging is not supported.
 443  * @param efx_spec[in, out]
 444  *   EFX filter specification to update.
 445  * @param[out] error
 446  *   Perform verbose error reporting if not NULL.
 447  */
 448 static int
 449 sfc_flow_parse_ipv6(const struct rte_flow_item *item,
 450                     efx_filter_spec_t *efx_spec,
 451                     struct rte_flow_error *error)
 452 {
 453         int rc;
 454         const struct rte_flow_item_ipv6 *spec = NULL;
 455         const struct rte_flow_item_ipv6 *mask = NULL;
 456         const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6);
 457         const struct rte_flow_item_ipv6 supp_mask = {
 458                 .hdr = {
 459                         .src_addr = { 0xff, 0xff, 0xff, 0xff,
 460                                       0xff, 0xff, 0xff, 0xff,
 461                                       0xff, 0xff, 0xff, 0xff,
 462                                       0xff, 0xff, 0xff, 0xff },
 463                         .dst_addr = { 0xff, 0xff, 0xff, 0xff,
 464                                       0xff, 0xff, 0xff, 0xff,
 465                                       0xff, 0xff, 0xff, 0xff,
 466                                       0xff, 0xff, 0xff, 0xff },
 467                         .proto = 0xff,
 468                 }
 469         };
 470
 471         rc = sfc_flow_parse_init(item,
 472                                  (const void **)&spec,
 473                                  (const void **)&mask,
 474                                  &supp_mask,
 475                                  &rte_flow_item_ipv6_mask,
 476                                  sizeof(struct rte_flow_item_ipv6),
 477                                  error);
 478         if (rc != 0)
 479                 return rc;
 480
 481         /*
 482          * Filtering by IPv6 source and destination addresses requires
 483          * the appropriate ETHER_TYPE in hardware filters
 484          */
 485         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 486                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 487                 efx_spec->efs_ether_type = ether_type_ipv6;
 488         } else if (efx_spec->efs_ether_type != ether_type_ipv6) {
 489                 rte_flow_error_set(error, EINVAL,
 490                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 491                         "Ethertype in pattern with IPV6 item should be appropriate");
 492                 return -rte_errno;
 493         }
 494
 495         if (spec == NULL)
 496                 return 0;
 497
 498         /*
 499          * IPv6 addresses are in big-endian byte order in item and in
 500          * efx_spec
 501          */
 502         if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr,
 503                    sizeof(mask->hdr.src_addr)) == 0) {
 504                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 505
 506                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) !=
 507                                  sizeof(spec->hdr.src_addr));
 508                 rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr,
 509                            sizeof(efx_spec->efs_rem_host));
 510         } else if (!sfc_flow_is_zero(mask->hdr.src_addr,
 511                                      sizeof(mask->hdr.src_addr))) {
 512                 goto fail_bad_mask;
 513         }
 514
 515         if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr,
 516                    sizeof(mask->hdr.dst_addr)) == 0) {
 517                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 518
 519                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) !=
 520                                  sizeof(spec->hdr.dst_addr));
 521                 rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr,
 522                            sizeof(efx_spec->efs_loc_host));
 523         } else if (!sfc_flow_is_zero(mask->hdr.dst_addr,
 524                                      sizeof(mask->hdr.dst_addr))) {
 525                 goto fail_bad_mask;
 526         }
 527
 528         if (mask->hdr.proto == supp_mask.hdr.proto) {
 529                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 530                 efx_spec->efs_ip_proto = spec->hdr.proto;
 531         } else if (mask->hdr.proto != 0) {
 532                 goto fail_bad_mask;
 533         }
 534
 535         return 0;
 536
 537 fail_bad_mask:
 538         rte_flow_error_set(error, EINVAL,
 539                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 540                            "Bad mask in the IPV6 pattern item");
 541         return -rte_errno;
 542 }
 543
 544 /**
 545  * Convert TCP item to EFX filter specification.
 546  *
 547  * @param item[in]
 548  *   Item specification. Only source and destination ports fields
 549  *   are supported. If the mask is NULL, default mask will be used.
 550  *   Ranging is not supported.
 551  * @param efx_spec[in, out]
 552  *   EFX filter specification to update.
 553  * @param[out] error
 554  *   Perform verbose error reporting if not NULL.
 555  */
 556 static int
 557 sfc_flow_parse_tcp(const struct rte_flow_item *item,
 558                    efx_filter_spec_t *efx_spec,
 559                    struct rte_flow_error *error)
 560 {
 561         int rc;
 562         const struct rte_flow_item_tcp *spec = NULL;
 563         const struct rte_flow_item_tcp *mask = NULL;
 564         const struct rte_flow_item_tcp supp_mask = {
 565                 .hdr = {
 566                         .src_port = 0xffff,
 567                         .dst_port = 0xffff,
 568                 }
 569         };
 570
 571         rc = sfc_flow_parse_init(item,
 572                                  (const void **)&spec,
 573                                  (const void **)&mask,
 574                                  &supp_mask,
 575                                  &rte_flow_item_tcp_mask,
 576                                  sizeof(struct rte_flow_item_tcp),
 577                                  error);
 578         if (rc != 0)
 579                 return rc;
 580
 581         /*
 582          * Filtering by TCP source and destination ports requires
 583          * the appropriate IP_PROTO in hardware filters
 584          */
 585         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 586                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 587                 efx_spec->efs_ip_proto = EFX_IPPROTO_TCP;
 588         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) {
 589                 rte_flow_error_set(error, EINVAL,
 590                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 591                         "IP proto in pattern with TCP item should be appropriate");
 592                 return -rte_errno;
 593         }
 594
 595         if (spec == NULL)
 596                 return 0;
 597
 598         /*
 599          * Source and destination ports are in big-endian byte order in item and
 600          * in little-endian in efx_spec, so byte swap is used
 601          */
 602         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 603                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 604                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 605         } else if (mask->hdr.src_port != 0) {
 606                 goto fail_bad_mask;
 607         }
 608
 609         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 610                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 611                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 612         } else if (mask->hdr.dst_port != 0) {
 613                 goto fail_bad_mask;
 614         }
 615
 616         return 0;
 617
 618 fail_bad_mask:
 619         rte_flow_error_set(error, EINVAL,
 620                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 621                            "Bad mask in the TCP pattern item");
 622         return -rte_errno;
 623 }
 624
 625 /**
 626  * Convert UDP item to EFX filter specification.
 627  *
 628  * @param item[in]
 629  *   Item specification. Only source and destination ports fields
 630  *   are supported. If the mask is NULL, default mask will be used.
 631  *   Ranging is not supported.
 632  * @param efx_spec[in, out]
 633  *   EFX filter specification to update.
 634  * @param[out] error
 635  *   Perform verbose error reporting if not NULL.
 636  */
 637 static int
 638 sfc_flow_parse_udp(const struct rte_flow_item *item,
 639                    efx_filter_spec_t *efx_spec,
 640                    struct rte_flow_error *error)
 641 {
 642         int rc;
 643         const struct rte_flow_item_udp *spec = NULL;
 644         const struct rte_flow_item_udp *mask = NULL;
 645         const struct rte_flow_item_udp supp_mask = {
 646                 .hdr = {
 647                         .src_port = 0xffff,
 648                         .dst_port = 0xffff,
 649                 }
 650         };
 651
 652         rc = sfc_flow_parse_init(item,
 653                                  (const void **)&spec,
 654                                  (const void **)&mask,
 655                                  &supp_mask,
 656                                  &rte_flow_item_udp_mask,
 657                                  sizeof(struct rte_flow_item_udp),
 658                                  error);
 659         if (rc != 0)
 660                 return rc;
 661
 662         /*
 663          * Filtering by UDP source and destination ports requires
 664          * the appropriate IP_PROTO in hardware filters
 665          */
 666         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 667                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 668                 efx_spec->efs_ip_proto = EFX_IPPROTO_UDP;
 669         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) {
 670                 rte_flow_error_set(error, EINVAL,
 671                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 672                         "IP proto in pattern with UDP item should be appropriate");
 673                 return -rte_errno;
 674         }
 675
 676         if (spec == NULL)
 677                 return 0;
 678
 679         /*
 680          * Source and destination ports are in big-endian byte order in item and
 681          * in little-endian in efx_spec, so byte swap is used
 682          */
 683         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 684                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 685                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 686         } else if (mask->hdr.src_port != 0) {
 687                 goto fail_bad_mask;
 688         }
 689
 690         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 691                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 692                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 693         } else if (mask->hdr.dst_port != 0) {
 694                 goto fail_bad_mask;
 695         }
 696
 697         return 0;
 698
 699 fail_bad_mask:
 700         rte_flow_error_set(error, EINVAL,
 701                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 702                            "Bad mask in the UDP pattern item");
 703         return -rte_errno;
 704 }
 705
 706 static const struct sfc_flow_item sfc_flow_items[] = {
 707         {
 708                 .type = RTE_FLOW_ITEM_TYPE_VOID,
 709                 .prev_layer = SFC_FLOW_ITEM_ANY_LAYER,
 710                 .layer = SFC_FLOW_ITEM_ANY_LAYER,
 711                 .parse = sfc_flow_parse_void,
 712         },
 713         {
 714                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 715                 .prev_layer = SFC_FLOW_ITEM_START_LAYER,
 716                 .layer = SFC_FLOW_ITEM_L2,
 717                 .parse = sfc_flow_parse_eth,
 718         },
 719         {
 720                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 721                 .prev_layer = SFC_FLOW_ITEM_L2,
 722                 .layer = SFC_FLOW_ITEM_L2,
 723                 .parse = sfc_flow_parse_vlan,
 724         },
 725         {
 726                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 727                 .prev_layer = SFC_FLOW_ITEM_L2,
 728                 .layer = SFC_FLOW_ITEM_L3,
 729                 .parse = sfc_flow_parse_ipv4,
 730         },
 731         {
 732                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 733                 .prev_layer = SFC_FLOW_ITEM_L2,
 734                 .layer = SFC_FLOW_ITEM_L3,
 735                 .parse = sfc_flow_parse_ipv6,
 736         },
 737         {
 738                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 739                 .prev_layer = SFC_FLOW_ITEM_L3,
 740                 .layer = SFC_FLOW_ITEM_L4,
 741                 .parse = sfc_flow_parse_tcp,
 742         },
 743         {
 744                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 745                 .prev_layer = SFC_FLOW_ITEM_L3,
 746                 .layer = SFC_FLOW_ITEM_L4,
 747                 .parse = sfc_flow_parse_udp,
 748         },
 749 };
 750
 751 /*
 752  * Protocol-independent flow API support
 753  */
 754 static int
 755 sfc_flow_parse_attr(const struct rte_flow_attr *attr,
 756                     struct rte_flow *flow,
 757                     struct rte_flow_error *error)
 758 {
 759         if (attr == NULL) {
 760                 rte_flow_error_set(error, EINVAL,
 761                                    RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 762                                    "NULL attribute");
 763                 return -rte_errno;
 764         }
 765         if (attr->group != 0) {
 766                 rte_flow_error_set(error, ENOTSUP,
 767                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
 768                                    "Groups are not supported");
 769                 return -rte_errno;
 770         }
 771         if (attr->priority != 0) {
 772                 rte_flow_error_set(error, ENOTSUP,
 773                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
 774                                    "Priorities are not supported");
 775                 return -rte_errno;
 776         }
 777         if (attr->egress != 0) {
 778                 rte_flow_error_set(error, ENOTSUP,
 779                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
 780                                    "Egress is not supported");
 781                 return -rte_errno;
 782         }
 783         if (attr->ingress == 0) {
 784                 rte_flow_error_set(error, ENOTSUP,
 785                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
 786                                    "Only ingress is supported");
 787                 return -rte_errno;
 788         }
 789
 790         flow->spec.efs_flags |= EFX_FILTER_FLAG_RX;
 791         flow->spec.efs_rss_context = EFX_FILTER_SPEC_RSS_CONTEXT_DEFAULT;
 792
 793         return 0;
 794 }
 795
 796 /* Get item from array sfc_flow_items */
 797 static const struct sfc_flow_item *
 798 sfc_flow_get_item(enum rte_flow_item_type type)
 799 {
 800         unsigned int i;
 801
 802         for (i = 0; i < RTE_DIM(sfc_flow_items); i++)
 803                 if (sfc_flow_items[i].type == type)
 804                         return &sfc_flow_items[i];
 805
 806         return NULL;
 807 }
 808
 809 static int
 810 sfc_flow_parse_pattern(const struct rte_flow_item pattern[],
 811                        struct rte_flow *flow,
 812                        struct rte_flow_error *error)
 813 {
 814         int rc;
 815         unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER;
 816         const struct sfc_flow_item *item;
 817
 818         if (pattern == NULL) {
 819                 rte_flow_error_set(error, EINVAL,
 820                                    RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
 821                                    "NULL pattern");
 822                 return -rte_errno;
 823         }
 824
 825         for (; pattern != NULL &&
 826                pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
 827                 item = sfc_flow_get_item(pattern->type);
 828                 if (item == NULL) {
 829                         rte_flow_error_set(error, ENOTSUP,
 830                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 831                                            "Unsupported pattern item");
 832                         return -rte_errno;
 833                 }
 834
 835                 /*
 836                  * Omitting one or several protocol layers at the beginning
 837                  * of pattern is supported
 838                  */
 839                 if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
 840                     prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
 841                     item->prev_layer != prev_layer) {
 842                         rte_flow_error_set(error, ENOTSUP,
 843                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 844                                            "Unexpected sequence of pattern items");
 845                         return -rte_errno;
 846                 }
 847
 848                 rc = item->parse(pattern, &flow->spec, error);
 849                 if (rc != 0)
 850                         return rc;
 851
 852                 if (item->layer != SFC_FLOW_ITEM_ANY_LAYER)
 853                         prev_layer = item->layer;
 854         }
 855
 856         if (pattern == NULL) {
 857                 rte_flow_error_set(error, EINVAL,
 858                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 859                                    "NULL item");
 860                 return -rte_errno;
 861         }
 862
 863         return 0;
 864 }
 865
 866 static int
 867 sfc_flow_parse_queue(struct sfc_adapter *sa,
 868                      const struct rte_flow_action_queue *queue,
 869                      struct rte_flow *flow)
 870 {
 871         struct sfc_rxq *rxq;
 872
 873         if (queue->index >= sa->rxq_count)
 874                 return -EINVAL;
 875
 876         rxq = sa->rxq_info[queue->index].rxq;
 877         flow->spec.efs_dmaq_id = (uint16_t)rxq->hw_index;
 878
 879         return 0;
 880 }
 881
 882 static int
 883 sfc_flow_parse_actions(struct sfc_adapter *sa,
 884                        const struct rte_flow_action actions[],
 885                        struct rte_flow *flow,
 886                        struct rte_flow_error *error)
 887 {
 888         int rc;
 889         boolean_t is_specified = B_FALSE;
 890
 891         if (actions == NULL) {
 892                 rte_flow_error_set(error, EINVAL,
 893                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
 894                                    "NULL actions");
 895                 return -rte_errno;
 896         }
 897
 898         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 899                 switch (actions->type) {
 900                 case RTE_FLOW_ACTION_TYPE_VOID:
 901                         break;
 902
 903                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 904                         rc = sfc_flow_parse_queue(sa, actions->conf, flow);
 905                         if (rc != 0) {
 906                                 rte_flow_error_set(error, EINVAL,
 907                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
 908                                         "Bad QUEUE action");
 909                                 return -rte_errno;
 910                         }
 911
 912                         is_specified = B_TRUE;
 913                         break;
 914
 915                 default:
 916                         rte_flow_error_set(error, ENOTSUP,
 917                                            RTE_FLOW_ERROR_TYPE_ACTION, actions,
 918                                            "Action is not supported");
 919                         return -rte_errno;
 920                 }
 921         }
 922
 923         if (!is_specified) {
 924                 rte_flow_error_set(error, EINVAL,
 925                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, actions,
 926                                    "Action is unspecified");
 927                 return -rte_errno;
 928         }
 929
 930         return 0;
 931 }
 932
 933 static int
 934 sfc_flow_parse(struct rte_eth_dev *dev,
 935                const struct rte_flow_attr *attr,
 936                const struct rte_flow_item pattern[],
 937                const struct rte_flow_action actions[],
 938                struct rte_flow *flow,
 939                struct rte_flow_error *error)
 940 {
 941         struct sfc_adapter *sa = dev->data->dev_private;
 942         int rc;
 943
 944         memset(&flow->spec, 0, sizeof(flow->spec));
 945
 946         rc = sfc_flow_parse_attr(attr, flow, error);
 947         if (rc != 0)
 948                 goto fail_bad_value;
 949
 950         rc = sfc_flow_parse_pattern(pattern, flow, error);
 951         if (rc != 0)
 952                 goto fail_bad_value;
 953
 954         rc = sfc_flow_parse_actions(sa, actions, flow, error);
 955         if (rc != 0)
 956                 goto fail_bad_value;
 957
 958         if (!sfc_filter_is_match_supported(sa, flow->spec.efs_match_flags)) {
 959                 rte_flow_error_set(error, ENOTSUP,
 960                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 961                                    "Flow rule pattern is not supported");
 962                 return -rte_errno;
 963         }
 964
 965 fail_bad_value:
 966         return rc;
 967 }
 968
 969 static int
 970 sfc_flow_validate(struct rte_eth_dev *dev,
 971                   const struct rte_flow_attr *attr,
 972                   const struct rte_flow_item pattern[],
 973                   const struct rte_flow_action actions[],
 974                   struct rte_flow_error *error)
 975 {
 976         struct rte_flow flow;
 977
 978         return sfc_flow_parse(dev, attr, pattern, actions, &flow, error);
 979 }
 980
 981 static struct rte_flow *
 982 sfc_flow_create(struct rte_eth_dev *dev,
 983                 const struct rte_flow_attr *attr,
 984                 const struct rte_flow_item pattern[],
 985                 const struct rte_flow_action actions[],
 986                 struct rte_flow_error *error)
 987 {
 988         struct sfc_adapter *sa = dev->data->dev_private;
 989         struct rte_flow *flow = NULL;
 990         int rc;
 991
 992         flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0);
 993         if (flow == NULL) {
 994                 rte_flow_error_set(error, ENOMEM,
 995                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 996                                    "Failed to allocate memory");
 997                 goto fail_no_mem;
 998         }
 999
1000         rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error);
1001         if (rc != 0)
1002                 goto fail_bad_value;
1003
1004         TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries);
1005
1006         sfc_adapter_lock(sa);
1007
1008         if (sa->state == SFC_ADAPTER_STARTED) {
1009                 rc = efx_filter_insert(sa->nic, &flow->spec);
1010                 if (rc != 0) {
1011                         rte_flow_error_set(error, rc,
1012                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1013                                 "Failed to insert filter");
1014                         goto fail_filter_insert;
1015                 }
1016         }
1017
1018         sfc_adapter_unlock(sa);
1019
1020         return flow;
1021
1022 fail_filter_insert:
1023         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1024
1025 fail_bad_value:
1026         rte_free(flow);
1027         sfc_adapter_unlock(sa);
1028
1029 fail_no_mem:
1030         return NULL;
1031 }
1032
1033 static int
1034 sfc_flow_remove(struct sfc_adapter *sa,
1035                 struct rte_flow *flow,
1036                 struct rte_flow_error *error)
1037 {
1038         int rc = 0;
1039
1040         SFC_ASSERT(sfc_adapter_is_locked(sa));
1041
1042         if (sa->state == SFC_ADAPTER_STARTED) {
1043                 rc = efx_filter_remove(sa->nic, &flow->spec);
1044                 if (rc != 0)
1045                         rte_flow_error_set(error, rc,
1046                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1047                                 "Failed to destroy flow rule");
1048         }
1049
1050         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1051         rte_free(flow);
1052
1053         return rc;
1054 }
1055
1056 static int
1057 sfc_flow_destroy(struct rte_eth_dev *dev,
1058                  struct rte_flow *flow,
1059                  struct rte_flow_error *error)
1060 {
1061         struct sfc_adapter *sa = dev->data->dev_private;
1062         struct rte_flow *flow_ptr;
1063         int rc = EINVAL;
1064
1065         sfc_adapter_lock(sa);
1066
1067         TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) {
1068                 if (flow_ptr == flow)
1069                         rc = 0;
1070         }
1071         if (rc != 0) {
1072                 rte_flow_error_set(error, rc,
1073                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1074                                    "Failed to find flow rule to destroy");
1075                 goto fail_bad_value;
1076         }
1077
1078         rc = sfc_flow_remove(sa, flow, error);
1079
1080 fail_bad_value:
1081         sfc_adapter_unlock(sa);
1082
1083         return -rc;
1084 }
1085
1086 static int
1087 sfc_flow_flush(struct rte_eth_dev *dev,
1088                struct rte_flow_error *error)
1089 {
1090         struct sfc_adapter *sa = dev->data->dev_private;
1091         struct rte_flow *flow;
1092         int rc = 0;
1093         int ret = 0;
1094
1095         sfc_adapter_lock(sa);
1096
1097         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1098                 rc = sfc_flow_remove(sa, flow, error);
1099                 if (rc != 0)
1100                         ret = rc;
1101         }
1102
1103         sfc_adapter_unlock(sa);
1104
1105         return -ret;
1106 }
1107
1108 const struct rte_flow_ops sfc_flow_ops = {
1109         .validate = sfc_flow_validate,
1110         .create = sfc_flow_create,
1111         .destroy = sfc_flow_destroy,
1112         .flush = sfc_flow_flush,
1113         .query = NULL,
1114 };
1115
1116 void
1117 sfc_flow_init(struct sfc_adapter *sa)
1118 {
1119         SFC_ASSERT(sfc_adapter_is_locked(sa));
1120
1121         TAILQ_INIT(&sa->filter.flow_list);
1122 }
1123
1124 void
1125 sfc_flow_fini(struct sfc_adapter *sa)
1126 {
1127         struct rte_flow *flow;
1128
1129         SFC_ASSERT(sfc_adapter_is_locked(sa));
1130
1131         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1132                 TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1133                 rte_free(flow);
1134         }
1135 }
1136
1137 void
1138 sfc_flow_stop(struct sfc_adapter *sa)
1139 {
1140         struct rte_flow *flow;
1141
1142         SFC_ASSERT(sfc_adapter_is_locked(sa));
1143
1144         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries)
1145                 efx_filter_remove(sa->nic, &flow->spec);
1146 }
1147
1148 int
1149 sfc_flow_start(struct sfc_adapter *sa)
1150 {
1151         struct rte_flow *flow;
1152         int rc = 0;
1153
1154         sfc_log_init(sa, "entry");
1155
1156         SFC_ASSERT(sfc_adapter_is_locked(sa));
1157
1158         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) {
1159                 rc = efx_filter_insert(sa->nic, &flow->spec);
1160                 if (rc != 0)
1161                         goto fail_bad_flow;
1162         }
1163
1164         sfc_log_init(sa, "done");
1165
1166 fail_bad_flow:
1167         return rc;
1168 }