drivers/net/sfc/sfc_flow.c

   1 /*-
   2  * Copyright (c) 2017 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * This software was jointly developed between OKTET Labs (under contract
   6  * for Solarflare) and Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright notice,
  12  *    this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright notice,
  14  *    this list of conditions and the following disclaimer in the documentation
  15  *    and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 #include <rte_tailq.h>
  31 #include <rte_common.h>
  32 #include <rte_ethdev.h>
  33 #include <rte_eth_ctrl.h>
  34 #include <rte_ether.h>
  35 #include <rte_flow.h>
  36 #include <rte_flow_driver.h>
  37
  38 #include "efx.h"
  39
  40 #include "sfc.h"
  41 #include "sfc_rx.h"
  42 #include "sfc_filter.h"
  43 #include "sfc_flow.h"
  44 #include "sfc_log.h"
  45
  46 /*
  47  * At now flow API is implemented in such a manner that each
  48  * flow rule is converted to a hardware filter.
  49  * All elements of flow rule (attributes, pattern items, actions)
  50  * correspond to one or more fields in the efx_filter_spec_s structure
  51  * that is responsible for the hardware filter.
  52  */
  53
  54 enum sfc_flow_item_layers {
  55         SFC_FLOW_ITEM_ANY_LAYER,
  56         SFC_FLOW_ITEM_START_LAYER,
  57         SFC_FLOW_ITEM_L2,
  58         SFC_FLOW_ITEM_L3,
  59         SFC_FLOW_ITEM_L4,
  60 };
  61
  62 typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item,
  63                                   efx_filter_spec_t *spec,
  64                                   struct rte_flow_error *error);
  65
  66 struct sfc_flow_item {
  67         enum rte_flow_item_type type;           /* Type of item */
  68         enum sfc_flow_item_layers layer;        /* Layer of item */
  69         enum sfc_flow_item_layers prev_layer;   /* Previous layer of item */
  70         sfc_flow_item_parse *parse;             /* Parsing function */
  71 };
  72
  73 static sfc_flow_item_parse sfc_flow_parse_void;
  74 static sfc_flow_item_parse sfc_flow_parse_eth;
  75 static sfc_flow_item_parse sfc_flow_parse_vlan;
  76 static sfc_flow_item_parse sfc_flow_parse_ipv4;
  77 static sfc_flow_item_parse sfc_flow_parse_ipv6;
  78 static sfc_flow_item_parse sfc_flow_parse_tcp;
  79 static sfc_flow_item_parse sfc_flow_parse_udp;
  80
  81 static boolean_t
  82 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
  83 {
  84         uint8_t sum = 0;
  85         unsigned int i;
  86
  87         for (i = 0; i < size; i++)
  88                 sum |= buf[i];
  89
  90         return (sum == 0) ? B_TRUE : B_FALSE;
  91 }
  92
  93 /*
  94  * Validate item and prepare structures spec and mask for parsing
  95  */
  96 static int
  97 sfc_flow_parse_init(const struct rte_flow_item *item,
  98                     const void **spec_ptr,
  99                     const void **mask_ptr,
 100                     const void *supp_mask,
 101                     const void *def_mask,
 102                     unsigned int size,
 103                     struct rte_flow_error *error)
 104 {
 105         const uint8_t *spec;
 106         const uint8_t *mask;
 107         const uint8_t *last;
 108         uint8_t match;
 109         uint8_t supp;
 110         unsigned int i;
 111
 112         if (item == NULL) {
 113                 rte_flow_error_set(error, EINVAL,
 114                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 115                                    "NULL item");
 116                 return -rte_errno;
 117         }
 118
 119         if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) {
 120                 rte_flow_error_set(error, EINVAL,
 121                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 122                                    "Mask or last is set without spec");
 123                 return -rte_errno;
 124         }
 125
 126         /*
 127          * If "mask" is not set, default mask is used,
 128          * but if default mask is NULL, "mask" should be set
 129          */
 130         if (item->mask == NULL) {
 131                 if (def_mask == NULL) {
 132                         rte_flow_error_set(error, EINVAL,
 133                                 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 134                                 "Mask should be specified");
 135                         return -rte_errno;
 136                 }
 137
 138                 mask = (const uint8_t *)def_mask;
 139         } else {
 140                 mask = (const uint8_t *)item->mask;
 141         }
 142
 143         spec = (const uint8_t *)item->spec;
 144         last = (const uint8_t *)item->last;
 145
 146         if (spec == NULL)
 147                 goto exit;
 148
 149         /*
 150          * If field values in "last" are either 0 or equal to the corresponding
 151          * values in "spec" then they are ignored
 152          */
 153         if (last != NULL &&
 154             !sfc_flow_is_zero(last, size) &&
 155             memcmp(last, spec, size) != 0) {
 156                 rte_flow_error_set(error, ENOTSUP,
 157                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 158                                    "Ranging is not supported");
 159                 return -rte_errno;
 160         }
 161
 162         if (supp_mask == NULL) {
 163                 rte_flow_error_set(error, EINVAL,
 164                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 165                         "Supported mask for item should be specified");
 166                 return -rte_errno;
 167         }
 168
 169         /* Check that mask and spec not asks for more match than supp_mask */
 170         for (i = 0; i < size; i++) {
 171                 match = spec[i] | mask[i];
 172                 supp = ((const uint8_t *)supp_mask)[i];
 173
 174                 if ((match | supp) != supp) {
 175                         rte_flow_error_set(error, ENOTSUP,
 176                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 177                                            "Item's field is not supported");
 178                         return -rte_errno;
 179                 }
 180         }
 181
 182 exit:
 183         *spec_ptr = spec;
 184         *mask_ptr = mask;
 185         return 0;
 186 }
 187
 188 /*
 189  * Protocol parsers.
 190  * Masking is not supported, so masks in items should be either
 191  * full or empty (zeroed) and set only for supported fields which
 192  * are specified in the supp_mask.
 193  */
 194
 195 static int
 196 sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item,
 197                     __rte_unused efx_filter_spec_t *efx_spec,
 198                     __rte_unused struct rte_flow_error *error)
 199 {
 200         return 0;
 201 }
 202
 203 /**
 204  * Convert Ethernet item to EFX filter specification.
 205  *
 206  * @param item[in]
 207  *   Item specification. Only source and destination addresses and
 208  *   Ethernet type fields are supported. In addition to full and
 209  *   empty masks of destination address, individual/group mask is
 210  *   also supported. If the mask is NULL, default mask will be used.
 211  *   Ranging is not supported.
 212  * @param efx_spec[in, out]
 213  *   EFX filter specification to update.
 214  * @param[out] error
 215  *   Perform verbose error reporting if not NULL.
 216  */
 217 static int
 218 sfc_flow_parse_eth(const struct rte_flow_item *item,
 219                    efx_filter_spec_t *efx_spec,
 220                    struct rte_flow_error *error)
 221 {
 222         int rc;
 223         const struct rte_flow_item_eth *spec = NULL;
 224         const struct rte_flow_item_eth *mask = NULL;
 225         const struct rte_flow_item_eth supp_mask = {
 226                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 227                 .src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 228                 .type = 0xffff,
 229         };
 230         const uint8_t ig_mask[EFX_MAC_ADDR_LEN] = {
 231                 0x01, 0x00, 0x00, 0x00, 0x00, 0x00
 232         };
 233
 234         rc = sfc_flow_parse_init(item,
 235                                  (const void **)&spec,
 236                                  (const void **)&mask,
 237                                  &supp_mask,
 238                                  &rte_flow_item_eth_mask,
 239                                  sizeof(struct rte_flow_item_eth),
 240                                  error);
 241         if (rc != 0)
 242                 return rc;
 243
 244         /* If "spec" is not set, could be any Ethernet */
 245         if (spec == NULL)
 246                 return 0;
 247
 248         if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) {
 249                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 250                 rte_memcpy(efx_spec->efs_loc_mac, spec->dst.addr_bytes,
 251                            EFX_MAC_ADDR_LEN);
 252         } else if (memcmp(mask->dst.addr_bytes, ig_mask,
 253                           EFX_MAC_ADDR_LEN) == 0) {
 254                 if (is_unicast_ether_addr(&spec->dst))
 255                         efx_spec->efs_match_flags |=
 256                                 EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 257                 else
 258                         efx_spec->efs_match_flags |=
 259                                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 260         } else if (!is_zero_ether_addr(&mask->dst)) {
 261                 goto fail_bad_mask;
 262         }
 263
 264         if (is_same_ether_addr(&mask->src, &supp_mask.src)) {
 265                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC;
 266                 rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes,
 267                            EFX_MAC_ADDR_LEN);
 268         } else if (!is_zero_ether_addr(&mask->src)) {
 269                 goto fail_bad_mask;
 270         }
 271
 272         /*
 273          * Ether type is in big-endian byte order in item and
 274          * in little-endian in efx_spec, so byte swap is used
 275          */
 276         if (mask->type == supp_mask.type) {
 277                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 278                 efx_spec->efs_ether_type = rte_bswap16(spec->type);
 279         } else if (mask->type != 0) {
 280                 goto fail_bad_mask;
 281         }
 282
 283         return 0;
 284
 285 fail_bad_mask:
 286         rte_flow_error_set(error, EINVAL,
 287                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 288                            "Bad mask in the ETH pattern item");
 289         return -rte_errno;
 290 }
 291
 292 /**
 293  * Convert VLAN item to EFX filter specification.
 294  *
 295  * @param item[in]
 296  *   Item specification. Only VID field is supported.
 297  *   The mask can not be NULL. Ranging is not supported.
 298  * @param efx_spec[in, out]
 299  *   EFX filter specification to update.
 300  * @param[out] error
 301  *   Perform verbose error reporting if not NULL.
 302  */
 303 static int
 304 sfc_flow_parse_vlan(const struct rte_flow_item *item,
 305                     efx_filter_spec_t *efx_spec,
 306                     struct rte_flow_error *error)
 307 {
 308         int rc;
 309         uint16_t vid;
 310         const struct rte_flow_item_vlan *spec = NULL;
 311         const struct rte_flow_item_vlan *mask = NULL;
 312         const struct rte_flow_item_vlan supp_mask = {
 313                 .tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX),
 314         };
 315
 316         rc = sfc_flow_parse_init(item,
 317                                  (const void **)&spec,
 318                                  (const void **)&mask,
 319                                  &supp_mask,
 320                                  NULL,
 321                                  sizeof(struct rte_flow_item_vlan),
 322                                  error);
 323         if (rc != 0)
 324                 return rc;
 325
 326         /*
 327          * VID is in big-endian byte order in item and
 328          * in little-endian in efx_spec, so byte swap is used.
 329          * If two VLAN items are included, the first matches
 330          * the outer tag and the next matches the inner tag.
 331          */
 332         if (mask->tci == supp_mask.tci) {
 333                 vid = rte_bswap16(spec->tci);
 334
 335                 if (!(efx_spec->efs_match_flags &
 336                       EFX_FILTER_MATCH_OUTER_VID)) {
 337                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 338                         efx_spec->efs_outer_vid = vid;
 339                 } else if (!(efx_spec->efs_match_flags &
 340                              EFX_FILTER_MATCH_INNER_VID)) {
 341                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID;
 342                         efx_spec->efs_inner_vid = vid;
 343                 } else {
 344                         rte_flow_error_set(error, EINVAL,
 345                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 346                                            "More than two VLAN items");
 347                         return -rte_errno;
 348                 }
 349         } else {
 350                 rte_flow_error_set(error, EINVAL,
 351                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 352                                    "VLAN ID in TCI match is required");
 353                 return -rte_errno;
 354         }
 355
 356         return 0;
 357 }
 358
 359 /**
 360  * Convert IPv4 item to EFX filter specification.
 361  *
 362  * @param item[in]
 363  *   Item specification. Only source and destination addresses and
 364  *   protocol fields are supported. If the mask is NULL, default
 365  *   mask will be used. Ranging is not supported.
 366  * @param efx_spec[in, out]
 367  *   EFX filter specification to update.
 368  * @param[out] error
 369  *   Perform verbose error reporting if not NULL.
 370  */
 371 static int
 372 sfc_flow_parse_ipv4(const struct rte_flow_item *item,
 373                     efx_filter_spec_t *efx_spec,
 374                     struct rte_flow_error *error)
 375 {
 376         int rc;
 377         const struct rte_flow_item_ipv4 *spec = NULL;
 378         const struct rte_flow_item_ipv4 *mask = NULL;
 379         const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4);
 380         const struct rte_flow_item_ipv4 supp_mask = {
 381                 .hdr = {
 382                         .src_addr = 0xffffffff,
 383                         .dst_addr = 0xffffffff,
 384                         .next_proto_id = 0xff,
 385                 }
 386         };
 387
 388         rc = sfc_flow_parse_init(item,
 389                                  (const void **)&spec,
 390                                  (const void **)&mask,
 391                                  &supp_mask,
 392                                  &rte_flow_item_ipv4_mask,
 393                                  sizeof(struct rte_flow_item_ipv4),
 394                                  error);
 395         if (rc != 0)
 396                 return rc;
 397
 398         /*
 399          * Filtering by IPv4 source and destination addresses requires
 400          * the appropriate ETHER_TYPE in hardware filters
 401          */
 402         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 403                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 404                 efx_spec->efs_ether_type = ether_type_ipv4;
 405         } else if (efx_spec->efs_ether_type != ether_type_ipv4) {
 406                 rte_flow_error_set(error, EINVAL,
 407                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 408                         "Ethertype in pattern with IPV4 item should be appropriate");
 409                 return -rte_errno;
 410         }
 411
 412         if (spec == NULL)
 413                 return 0;
 414
 415         /*
 416          * IPv4 addresses are in big-endian byte order in item and in
 417          * efx_spec
 418          */
 419         if (mask->hdr.src_addr == supp_mask.hdr.src_addr) {
 420                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 421                 efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr;
 422         } else if (mask->hdr.src_addr != 0) {
 423                 goto fail_bad_mask;
 424         }
 425
 426         if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) {
 427                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 428                 efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr;
 429         } else if (mask->hdr.dst_addr != 0) {
 430                 goto fail_bad_mask;
 431         }
 432
 433         if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) {
 434                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 435                 efx_spec->efs_ip_proto = spec->hdr.next_proto_id;
 436         } else if (mask->hdr.next_proto_id != 0) {
 437                 goto fail_bad_mask;
 438         }
 439
 440         return 0;
 441
 442 fail_bad_mask:
 443         rte_flow_error_set(error, EINVAL,
 444                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 445                            "Bad mask in the IPV4 pattern item");
 446         return -rte_errno;
 447 }
 448
 449 /**
 450  * Convert IPv6 item to EFX filter specification.
 451  *
 452  * @param item[in]
 453  *   Item specification. Only source and destination addresses and
 454  *   next header fields are supported. If the mask is NULL, default
 455  *   mask will be used. Ranging is not supported.
 456  * @param efx_spec[in, out]
 457  *   EFX filter specification to update.
 458  * @param[out] error
 459  *   Perform verbose error reporting if not NULL.
 460  */
 461 static int
 462 sfc_flow_parse_ipv6(const struct rte_flow_item *item,
 463                     efx_filter_spec_t *efx_spec,
 464                     struct rte_flow_error *error)
 465 {
 466         int rc;
 467         const struct rte_flow_item_ipv6 *spec = NULL;
 468         const struct rte_flow_item_ipv6 *mask = NULL;
 469         const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6);
 470         const struct rte_flow_item_ipv6 supp_mask = {
 471                 .hdr = {
 472                         .src_addr = { 0xff, 0xff, 0xff, 0xff,
 473                                       0xff, 0xff, 0xff, 0xff,
 474                                       0xff, 0xff, 0xff, 0xff,
 475                                       0xff, 0xff, 0xff, 0xff },
 476                         .dst_addr = { 0xff, 0xff, 0xff, 0xff,
 477                                       0xff, 0xff, 0xff, 0xff,
 478                                       0xff, 0xff, 0xff, 0xff,
 479                                       0xff, 0xff, 0xff, 0xff },
 480                         .proto = 0xff,
 481                 }
 482         };
 483
 484         rc = sfc_flow_parse_init(item,
 485                                  (const void **)&spec,
 486                                  (const void **)&mask,
 487                                  &supp_mask,
 488                                  &rte_flow_item_ipv6_mask,
 489                                  sizeof(struct rte_flow_item_ipv6),
 490                                  error);
 491         if (rc != 0)
 492                 return rc;
 493
 494         /*
 495          * Filtering by IPv6 source and destination addresses requires
 496          * the appropriate ETHER_TYPE in hardware filters
 497          */
 498         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 499                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 500                 efx_spec->efs_ether_type = ether_type_ipv6;
 501         } else if (efx_spec->efs_ether_type != ether_type_ipv6) {
 502                 rte_flow_error_set(error, EINVAL,
 503                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 504                         "Ethertype in pattern with IPV6 item should be appropriate");
 505                 return -rte_errno;
 506         }
 507
 508         if (spec == NULL)
 509                 return 0;
 510
 511         /*
 512          * IPv6 addresses are in big-endian byte order in item and in
 513          * efx_spec
 514          */
 515         if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr,
 516                    sizeof(mask->hdr.src_addr)) == 0) {
 517                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 518
 519                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) !=
 520                                  sizeof(spec->hdr.src_addr));
 521                 rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr,
 522                            sizeof(efx_spec->efs_rem_host));
 523         } else if (!sfc_flow_is_zero(mask->hdr.src_addr,
 524                                      sizeof(mask->hdr.src_addr))) {
 525                 goto fail_bad_mask;
 526         }
 527
 528         if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr,
 529                    sizeof(mask->hdr.dst_addr)) == 0) {
 530                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 531
 532                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) !=
 533                                  sizeof(spec->hdr.dst_addr));
 534                 rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr,
 535                            sizeof(efx_spec->efs_loc_host));
 536         } else if (!sfc_flow_is_zero(mask->hdr.dst_addr,
 537                                      sizeof(mask->hdr.dst_addr))) {
 538                 goto fail_bad_mask;
 539         }
 540
 541         if (mask->hdr.proto == supp_mask.hdr.proto) {
 542                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 543                 efx_spec->efs_ip_proto = spec->hdr.proto;
 544         } else if (mask->hdr.proto != 0) {
 545                 goto fail_bad_mask;
 546         }
 547
 548         return 0;
 549
 550 fail_bad_mask:
 551         rte_flow_error_set(error, EINVAL,
 552                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 553                            "Bad mask in the IPV6 pattern item");
 554         return -rte_errno;
 555 }
 556
 557 /**
 558  * Convert TCP item to EFX filter specification.
 559  *
 560  * @param item[in]
 561  *   Item specification. Only source and destination ports fields
 562  *   are supported. If the mask is NULL, default mask will be used.
 563  *   Ranging is not supported.
 564  * @param efx_spec[in, out]
 565  *   EFX filter specification to update.
 566  * @param[out] error
 567  *   Perform verbose error reporting if not NULL.
 568  */
 569 static int
 570 sfc_flow_parse_tcp(const struct rte_flow_item *item,
 571                    efx_filter_spec_t *efx_spec,
 572                    struct rte_flow_error *error)
 573 {
 574         int rc;
 575         const struct rte_flow_item_tcp *spec = NULL;
 576         const struct rte_flow_item_tcp *mask = NULL;
 577         const struct rte_flow_item_tcp supp_mask = {
 578                 .hdr = {
 579                         .src_port = 0xffff,
 580                         .dst_port = 0xffff,
 581                 }
 582         };
 583
 584         rc = sfc_flow_parse_init(item,
 585                                  (const void **)&spec,
 586                                  (const void **)&mask,
 587                                  &supp_mask,
 588                                  &rte_flow_item_tcp_mask,
 589                                  sizeof(struct rte_flow_item_tcp),
 590                                  error);
 591         if (rc != 0)
 592                 return rc;
 593
 594         /*
 595          * Filtering by TCP source and destination ports requires
 596          * the appropriate IP_PROTO in hardware filters
 597          */
 598         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 599                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 600                 efx_spec->efs_ip_proto = EFX_IPPROTO_TCP;
 601         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) {
 602                 rte_flow_error_set(error, EINVAL,
 603                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 604                         "IP proto in pattern with TCP item should be appropriate");
 605                 return -rte_errno;
 606         }
 607
 608         if (spec == NULL)
 609                 return 0;
 610
 611         /*
 612          * Source and destination ports are in big-endian byte order in item and
 613          * in little-endian in efx_spec, so byte swap is used
 614          */
 615         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 616                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 617                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 618         } else if (mask->hdr.src_port != 0) {
 619                 goto fail_bad_mask;
 620         }
 621
 622         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 623                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 624                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 625         } else if (mask->hdr.dst_port != 0) {
 626                 goto fail_bad_mask;
 627         }
 628
 629         return 0;
 630
 631 fail_bad_mask:
 632         rte_flow_error_set(error, EINVAL,
 633                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 634                            "Bad mask in the TCP pattern item");
 635         return -rte_errno;
 636 }
 637
 638 /**
 639  * Convert UDP item to EFX filter specification.
 640  *
 641  * @param item[in]
 642  *   Item specification. Only source and destination ports fields
 643  *   are supported. If the mask is NULL, default mask will be used.
 644  *   Ranging is not supported.
 645  * @param efx_spec[in, out]
 646  *   EFX filter specification to update.
 647  * @param[out] error
 648  *   Perform verbose error reporting if not NULL.
 649  */
 650 static int
 651 sfc_flow_parse_udp(const struct rte_flow_item *item,
 652                    efx_filter_spec_t *efx_spec,
 653                    struct rte_flow_error *error)
 654 {
 655         int rc;
 656         const struct rte_flow_item_udp *spec = NULL;
 657         const struct rte_flow_item_udp *mask = NULL;
 658         const struct rte_flow_item_udp supp_mask = {
 659                 .hdr = {
 660                         .src_port = 0xffff,
 661                         .dst_port = 0xffff,
 662                 }
 663         };
 664
 665         rc = sfc_flow_parse_init(item,
 666                                  (const void **)&spec,
 667                                  (const void **)&mask,
 668                                  &supp_mask,
 669                                  &rte_flow_item_udp_mask,
 670                                  sizeof(struct rte_flow_item_udp),
 671                                  error);
 672         if (rc != 0)
 673                 return rc;
 674
 675         /*
 676          * Filtering by UDP source and destination ports requires
 677          * the appropriate IP_PROTO in hardware filters
 678          */
 679         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 680                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 681                 efx_spec->efs_ip_proto = EFX_IPPROTO_UDP;
 682         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) {
 683                 rte_flow_error_set(error, EINVAL,
 684                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 685                         "IP proto in pattern with UDP item should be appropriate");
 686                 return -rte_errno;
 687         }
 688
 689         if (spec == NULL)
 690                 return 0;
 691
 692         /*
 693          * Source and destination ports are in big-endian byte order in item and
 694          * in little-endian in efx_spec, so byte swap is used
 695          */
 696         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 697                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 698                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 699         } else if (mask->hdr.src_port != 0) {
 700                 goto fail_bad_mask;
 701         }
 702
 703         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 704                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 705                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 706         } else if (mask->hdr.dst_port != 0) {
 707                 goto fail_bad_mask;
 708         }
 709
 710         return 0;
 711
 712 fail_bad_mask:
 713         rte_flow_error_set(error, EINVAL,
 714                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 715                            "Bad mask in the UDP pattern item");
 716         return -rte_errno;
 717 }
 718
 719 static const struct sfc_flow_item sfc_flow_items[] = {
 720         {
 721                 .type = RTE_FLOW_ITEM_TYPE_VOID,
 722                 .prev_layer = SFC_FLOW_ITEM_ANY_LAYER,
 723                 .layer = SFC_FLOW_ITEM_ANY_LAYER,
 724                 .parse = sfc_flow_parse_void,
 725         },
 726         {
 727                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 728                 .prev_layer = SFC_FLOW_ITEM_START_LAYER,
 729                 .layer = SFC_FLOW_ITEM_L2,
 730                 .parse = sfc_flow_parse_eth,
 731         },
 732         {
 733                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 734                 .prev_layer = SFC_FLOW_ITEM_L2,
 735                 .layer = SFC_FLOW_ITEM_L2,
 736                 .parse = sfc_flow_parse_vlan,
 737         },
 738         {
 739                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 740                 .prev_layer = SFC_FLOW_ITEM_L2,
 741                 .layer = SFC_FLOW_ITEM_L3,
 742                 .parse = sfc_flow_parse_ipv4,
 743         },
 744         {
 745                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 746                 .prev_layer = SFC_FLOW_ITEM_L2,
 747                 .layer = SFC_FLOW_ITEM_L3,
 748                 .parse = sfc_flow_parse_ipv6,
 749         },
 750         {
 751                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 752                 .prev_layer = SFC_FLOW_ITEM_L3,
 753                 .layer = SFC_FLOW_ITEM_L4,
 754                 .parse = sfc_flow_parse_tcp,
 755         },
 756         {
 757                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 758                 .prev_layer = SFC_FLOW_ITEM_L3,
 759                 .layer = SFC_FLOW_ITEM_L4,
 760                 .parse = sfc_flow_parse_udp,
 761         },
 762 };
 763
 764 /*
 765  * Protocol-independent flow API support
 766  */
 767 static int
 768 sfc_flow_parse_attr(const struct rte_flow_attr *attr,
 769                     struct rte_flow *flow,
 770                     struct rte_flow_error *error)
 771 {
 772         if (attr == NULL) {
 773                 rte_flow_error_set(error, EINVAL,
 774                                    RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 775                                    "NULL attribute");
 776                 return -rte_errno;
 777         }
 778         if (attr->group != 0) {
 779                 rte_flow_error_set(error, ENOTSUP,
 780                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
 781                                    "Groups are not supported");
 782                 return -rte_errno;
 783         }
 784         if (attr->priority != 0) {
 785                 rte_flow_error_set(error, ENOTSUP,
 786                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
 787                                    "Priorities are not supported");
 788                 return -rte_errno;
 789         }
 790         if (attr->egress != 0) {
 791                 rte_flow_error_set(error, ENOTSUP,
 792                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
 793                                    "Egress is not supported");
 794                 return -rte_errno;
 795         }
 796         if (attr->ingress == 0) {
 797                 rte_flow_error_set(error, ENOTSUP,
 798                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
 799                                    "Only ingress is supported");
 800                 return -rte_errno;
 801         }
 802
 803         flow->spec.efs_flags |= EFX_FILTER_FLAG_RX;
 804         flow->spec.efs_rss_context = EFX_FILTER_SPEC_RSS_CONTEXT_DEFAULT;
 805
 806         return 0;
 807 }
 808
 809 /* Get item from array sfc_flow_items */
 810 static const struct sfc_flow_item *
 811 sfc_flow_get_item(enum rte_flow_item_type type)
 812 {
 813         unsigned int i;
 814
 815         for (i = 0; i < RTE_DIM(sfc_flow_items); i++)
 816                 if (sfc_flow_items[i].type == type)
 817                         return &sfc_flow_items[i];
 818
 819         return NULL;
 820 }
 821
 822 static int
 823 sfc_flow_parse_pattern(const struct rte_flow_item pattern[],
 824                        struct rte_flow *flow,
 825                        struct rte_flow_error *error)
 826 {
 827         int rc;
 828         unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER;
 829         const struct sfc_flow_item *item;
 830
 831         if (pattern == NULL) {
 832                 rte_flow_error_set(error, EINVAL,
 833                                    RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
 834                                    "NULL pattern");
 835                 return -rte_errno;
 836         }
 837
 838         for (; pattern != NULL &&
 839                pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
 840                 item = sfc_flow_get_item(pattern->type);
 841                 if (item == NULL) {
 842                         rte_flow_error_set(error, ENOTSUP,
 843                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 844                                            "Unsupported pattern item");
 845                         return -rte_errno;
 846                 }
 847
 848                 /*
 849                  * Omitting one or several protocol layers at the beginning
 850                  * of pattern is supported
 851                  */
 852                 if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
 853                     prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
 854                     item->prev_layer != prev_layer) {
 855                         rte_flow_error_set(error, ENOTSUP,
 856                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 857                                            "Unexpected sequence of pattern items");
 858                         return -rte_errno;
 859                 }
 860
 861                 rc = item->parse(pattern, &flow->spec, error);
 862                 if (rc != 0)
 863                         return rc;
 864
 865                 if (item->layer != SFC_FLOW_ITEM_ANY_LAYER)
 866                         prev_layer = item->layer;
 867         }
 868
 869         if (pattern == NULL) {
 870                 rte_flow_error_set(error, EINVAL,
 871                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 872                                    "NULL item");
 873                 return -rte_errno;
 874         }
 875
 876         return 0;
 877 }
 878
 879 static int
 880 sfc_flow_parse_queue(struct sfc_adapter *sa,
 881                      const struct rte_flow_action_queue *queue,
 882                      struct rte_flow *flow)
 883 {
 884         struct sfc_rxq *rxq;
 885
 886         if (queue->index >= sa->rxq_count)
 887                 return -EINVAL;
 888
 889         rxq = sa->rxq_info[queue->index].rxq;
 890         flow->spec.efs_dmaq_id = (uint16_t)rxq->hw_index;
 891
 892         return 0;
 893 }
 894
 895 static int
 896 sfc_flow_parse_actions(struct sfc_adapter *sa,
 897                        const struct rte_flow_action actions[],
 898                        struct rte_flow *flow,
 899                        struct rte_flow_error *error)
 900 {
 901         int rc;
 902         boolean_t is_specified = B_FALSE;
 903
 904         if (actions == NULL) {
 905                 rte_flow_error_set(error, EINVAL,
 906                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
 907                                    "NULL actions");
 908                 return -rte_errno;
 909         }
 910
 911         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 912                 switch (actions->type) {
 913                 case RTE_FLOW_ACTION_TYPE_VOID:
 914                         break;
 915
 916                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 917                         rc = sfc_flow_parse_queue(sa, actions->conf, flow);
 918                         if (rc != 0) {
 919                                 rte_flow_error_set(error, EINVAL,
 920                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
 921                                         "Bad QUEUE action");
 922                                 return -rte_errno;
 923                         }
 924
 925                         is_specified = B_TRUE;
 926                         break;
 927
 928                 default:
 929                         rte_flow_error_set(error, ENOTSUP,
 930                                            RTE_FLOW_ERROR_TYPE_ACTION, actions,
 931                                            "Action is not supported");
 932                         return -rte_errno;
 933                 }
 934         }
 935
 936         if (!is_specified) {
 937                 rte_flow_error_set(error, EINVAL,
 938                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, actions,
 939                                    "Action is unspecified");
 940                 return -rte_errno;
 941         }
 942
 943         return 0;
 944 }
 945
 946 static int
 947 sfc_flow_parse(struct rte_eth_dev *dev,
 948                const struct rte_flow_attr *attr,
 949                const struct rte_flow_item pattern[],
 950                const struct rte_flow_action actions[],
 951                struct rte_flow *flow,
 952                struct rte_flow_error *error)
 953 {
 954         struct sfc_adapter *sa = dev->data->dev_private;
 955         int rc;
 956
 957         memset(&flow->spec, 0, sizeof(flow->spec));
 958
 959         rc = sfc_flow_parse_attr(attr, flow, error);
 960         if (rc != 0)
 961                 goto fail_bad_value;
 962
 963         rc = sfc_flow_parse_pattern(pattern, flow, error);
 964         if (rc != 0)
 965                 goto fail_bad_value;
 966
 967         rc = sfc_flow_parse_actions(sa, actions, flow, error);
 968         if (rc != 0)
 969                 goto fail_bad_value;
 970
 971         if (!sfc_filter_is_match_supported(sa, flow->spec.efs_match_flags)) {
 972                 rte_flow_error_set(error, ENOTSUP,
 973                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 974                                    "Flow rule pattern is not supported");
 975                 return -rte_errno;
 976         }
 977
 978 fail_bad_value:
 979         return rc;
 980 }
 981
 982 static int
 983 sfc_flow_validate(struct rte_eth_dev *dev,
 984                   const struct rte_flow_attr *attr,
 985                   const struct rte_flow_item pattern[],
 986                   const struct rte_flow_action actions[],
 987                   struct rte_flow_error *error)
 988 {
 989         struct rte_flow flow;
 990
 991         return sfc_flow_parse(dev, attr, pattern, actions, &flow, error);
 992 }
 993
 994 static struct rte_flow *
 995 sfc_flow_create(struct rte_eth_dev *dev,
 996                 const struct rte_flow_attr *attr,
 997                 const struct rte_flow_item pattern[],
 998                 const struct rte_flow_action actions[],
 999                 struct rte_flow_error *error)
1000 {
1001         struct sfc_adapter *sa = dev->data->dev_private;
1002         struct rte_flow *flow = NULL;
1003         int rc;
1004
1005         flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0);
1006         if (flow == NULL) {
1007                 rte_flow_error_set(error, ENOMEM,
1008                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1009                                    "Failed to allocate memory");
1010                 goto fail_no_mem;
1011         }
1012
1013         rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error);
1014         if (rc != 0)
1015                 goto fail_bad_value;
1016
1017         TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries);
1018
1019         sfc_adapter_lock(sa);
1020
1021         if (sa->state == SFC_ADAPTER_STARTED) {
1022                 rc = efx_filter_insert(sa->nic, &flow->spec);
1023                 if (rc != 0) {
1024                         rte_flow_error_set(error, rc,
1025                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1026                                 "Failed to insert filter");
1027                         goto fail_filter_insert;
1028                 }
1029         }
1030
1031         sfc_adapter_unlock(sa);
1032
1033         return flow;
1034
1035 fail_filter_insert:
1036         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1037
1038 fail_bad_value:
1039         rte_free(flow);
1040         sfc_adapter_unlock(sa);
1041
1042 fail_no_mem:
1043         return NULL;
1044 }
1045
1046 static int
1047 sfc_flow_remove(struct sfc_adapter *sa,
1048                 struct rte_flow *flow,
1049                 struct rte_flow_error *error)
1050 {
1051         int rc = 0;
1052
1053         SFC_ASSERT(sfc_adapter_is_locked(sa));
1054
1055         if (sa->state == SFC_ADAPTER_STARTED) {
1056                 rc = efx_filter_remove(sa->nic, &flow->spec);
1057                 if (rc != 0)
1058                         rte_flow_error_set(error, rc,
1059                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1060                                 "Failed to destroy flow rule");
1061         }
1062
1063         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1064         rte_free(flow);
1065
1066         return rc;
1067 }
1068
1069 static int
1070 sfc_flow_destroy(struct rte_eth_dev *dev,
1071                  struct rte_flow *flow,
1072                  struct rte_flow_error *error)
1073 {
1074         struct sfc_adapter *sa = dev->data->dev_private;
1075         struct rte_flow *flow_ptr;
1076         int rc = EINVAL;
1077
1078         sfc_adapter_lock(sa);
1079
1080         TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) {
1081                 if (flow_ptr == flow)
1082                         rc = 0;
1083         }
1084         if (rc != 0) {
1085                 rte_flow_error_set(error, rc,
1086                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1087                                    "Failed to find flow rule to destroy");
1088                 goto fail_bad_value;
1089         }
1090
1091         rc = sfc_flow_remove(sa, flow, error);
1092
1093 fail_bad_value:
1094         sfc_adapter_unlock(sa);
1095
1096         return -rc;
1097 }
1098
1099 static int
1100 sfc_flow_flush(struct rte_eth_dev *dev,
1101                struct rte_flow_error *error)
1102 {
1103         struct sfc_adapter *sa = dev->data->dev_private;
1104         struct rte_flow *flow;
1105         int rc = 0;
1106         int ret = 0;
1107
1108         sfc_adapter_lock(sa);
1109
1110         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1111                 rc = sfc_flow_remove(sa, flow, error);
1112                 if (rc != 0)
1113                         ret = rc;
1114         }
1115
1116         sfc_adapter_unlock(sa);
1117
1118         return -ret;
1119 }
1120
1121 const struct rte_flow_ops sfc_flow_ops = {
1122         .validate = sfc_flow_validate,
1123         .create = sfc_flow_create,
1124         .destroy = sfc_flow_destroy,
1125         .flush = sfc_flow_flush,
1126         .query = NULL,
1127 };
1128
1129 void
1130 sfc_flow_init(struct sfc_adapter *sa)
1131 {
1132         SFC_ASSERT(sfc_adapter_is_locked(sa));
1133
1134         TAILQ_INIT(&sa->filter.flow_list);
1135 }
1136
1137 void
1138 sfc_flow_fini(struct sfc_adapter *sa)
1139 {
1140         struct rte_flow *flow;
1141
1142         SFC_ASSERT(sfc_adapter_is_locked(sa));
1143
1144         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1145                 TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1146                 rte_free(flow);
1147         }
1148 }
1149
1150 void
1151 sfc_flow_stop(struct sfc_adapter *sa)
1152 {
1153         struct rte_flow *flow;
1154
1155         SFC_ASSERT(sfc_adapter_is_locked(sa));
1156
1157         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries)
1158                 efx_filter_remove(sa->nic, &flow->spec);
1159 }
1160
1161 int
1162 sfc_flow_start(struct sfc_adapter *sa)
1163 {
1164         struct rte_flow *flow;
1165         int rc = 0;
1166
1167         sfc_log_init(sa, "entry");
1168
1169         SFC_ASSERT(sfc_adapter_is_locked(sa));
1170
1171         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) {
1172                 rc = efx_filter_insert(sa->nic, &flow->spec);
1173                 if (rc != 0)
1174                         goto fail_bad_flow;
1175         }
1176
1177         sfc_log_init(sa, "done");
1178
1179 fail_bad_flow:
1180         return rc;
1181 }