drivers/net/tap/tap_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <errno.h>
  35 #include <string.h>
  36 #include <sys/queue.h>
  37
  38 #include <rte_byteorder.h>
  39 #include <rte_jhash.h>
  40 #include <rte_malloc.h>
  41 #include <rte_eth_tap.h>
  42 #include <tap_flow.h>
  43 #include <tap_autoconf.h>
  44 #include <tap_tcmsgs.h>
  45
  46 #ifndef HAVE_TC_FLOWER
  47 /*
  48  * For kernels < 4.2, this enum is not defined. Runtime checks will be made to
  49  * avoid sending TC messages the kernel cannot understand.
  50  */
  51 enum {
  52         TCA_FLOWER_UNSPEC,
  53         TCA_FLOWER_CLASSID,
  54         TCA_FLOWER_INDEV,
  55         TCA_FLOWER_ACT,
  56         TCA_FLOWER_KEY_ETH_DST,         /* ETH_ALEN */
  57         TCA_FLOWER_KEY_ETH_DST_MASK,    /* ETH_ALEN */
  58         TCA_FLOWER_KEY_ETH_SRC,         /* ETH_ALEN */
  59         TCA_FLOWER_KEY_ETH_SRC_MASK,    /* ETH_ALEN */
  60         TCA_FLOWER_KEY_ETH_TYPE,        /* be16 */
  61         TCA_FLOWER_KEY_IP_PROTO,        /* u8 */
  62         TCA_FLOWER_KEY_IPV4_SRC,        /* be32 */
  63         TCA_FLOWER_KEY_IPV4_SRC_MASK,   /* be32 */
  64         TCA_FLOWER_KEY_IPV4_DST,        /* be32 */
  65         TCA_FLOWER_KEY_IPV4_DST_MASK,   /* be32 */
  66         TCA_FLOWER_KEY_IPV6_SRC,        /* struct in6_addr */
  67         TCA_FLOWER_KEY_IPV6_SRC_MASK,   /* struct in6_addr */
  68         TCA_FLOWER_KEY_IPV6_DST,        /* struct in6_addr */
  69         TCA_FLOWER_KEY_IPV6_DST_MASK,   /* struct in6_addr */
  70         TCA_FLOWER_KEY_TCP_SRC,         /* be16 */
  71         TCA_FLOWER_KEY_TCP_DST,         /* be16 */
  72         TCA_FLOWER_KEY_UDP_SRC,         /* be16 */
  73         TCA_FLOWER_KEY_UDP_DST,         /* be16 */
  74 };
  75 #endif
  76 #ifndef HAVE_TC_VLAN_ID
  77 enum {
  78         /* TCA_FLOWER_FLAGS, */
  79         TCA_FLOWER_KEY_VLAN_ID = TCA_FLOWER_KEY_UDP_DST + 2, /* be16 */
  80         TCA_FLOWER_KEY_VLAN_PRIO,       /* u8   */
  81         TCA_FLOWER_KEY_VLAN_ETH_TYPE,   /* be16 */
  82 };
  83 #endif
  84
  85 #define ISOLATE_HANDLE 1
  86
  87 struct rte_flow {
  88         LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */
  89         struct rte_flow *remote_flow; /* associated remote flow */
  90         struct nlmsg msg;
  91 };
  92
  93 struct convert_data {
  94         uint16_t eth_type;
  95         uint16_t ip_proto;
  96         uint8_t vlan;
  97         struct rte_flow *flow;
  98 };
  99
 100 struct remote_rule {
 101         struct rte_flow_attr attr;
 102         struct rte_flow_item items[2];
 103         struct rte_flow_action actions[2];
 104         int mirred;
 105 };
 106
 107 static int tap_flow_create_eth(const struct rte_flow_item *item, void *data);
 108 static int tap_flow_create_vlan(const struct rte_flow_item *item, void *data);
 109 static int tap_flow_create_ipv4(const struct rte_flow_item *item, void *data);
 110 static int tap_flow_create_ipv6(const struct rte_flow_item *item, void *data);
 111 static int tap_flow_create_udp(const struct rte_flow_item *item, void *data);
 112 static int tap_flow_create_tcp(const struct rte_flow_item *item, void *data);
 113 static int
 114 tap_flow_validate(struct rte_eth_dev *dev,
 115                   const struct rte_flow_attr *attr,
 116                   const struct rte_flow_item items[],
 117                   const struct rte_flow_action actions[],
 118                   struct rte_flow_error *error);
 119
 120 static struct rte_flow *
 121 tap_flow_create(struct rte_eth_dev *dev,
 122                 const struct rte_flow_attr *attr,
 123                 const struct rte_flow_item items[],
 124                 const struct rte_flow_action actions[],
 125                 struct rte_flow_error *error);
 126
 127 static int
 128 tap_flow_destroy(struct rte_eth_dev *dev,
 129                  struct rte_flow *flow,
 130                  struct rte_flow_error *error);
 131
 132 static int
 133 tap_flow_isolate(struct rte_eth_dev *dev,
 134                  int set,
 135                  struct rte_flow_error *error);
 136
 137 static const struct rte_flow_ops tap_flow_ops = {
 138         .validate = tap_flow_validate,
 139         .create = tap_flow_create,
 140         .destroy = tap_flow_destroy,
 141         .flush = tap_flow_flush,
 142         .isolate = tap_flow_isolate,
 143 };
 144
 145 /* Static initializer for items. */
 146 #define ITEMS(...) \
 147         (const enum rte_flow_item_type []){ \
 148                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
 149         }
 150
 151 /* Structure to generate a simple graph of layers supported by the NIC. */
 152 struct tap_flow_items {
 153         /* Bit-mask corresponding to what is supported for this item. */
 154         const void *mask;
 155         const unsigned int mask_sz; /* Bit-mask size in bytes. */
 156         /*
 157          * Bit-mask corresponding to the default mask, if none is provided
 158          * along with the item.
 159          */
 160         const void *default_mask;
 161         /**
 162          * Conversion function from rte_flow to netlink attributes.
 163          *
 164          * @param item
 165          *   rte_flow item to convert.
 166          * @param data
 167          *   Internal structure to store the conversion.
 168          *
 169          * @return
 170          *   0 on success, negative value otherwise.
 171          */
 172         int (*convert)(const struct rte_flow_item *item, void *data);
 173         /** List of possible following items.  */
 174         const enum rte_flow_item_type *const items;
 175 };
 176
 177 /* Graph of supported items and associated actions. */
 178 static const struct tap_flow_items tap_flow_items[] = {
 179         [RTE_FLOW_ITEM_TYPE_END] = {
 180                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 181         },
 182         [RTE_FLOW_ITEM_TYPE_ETH] = {
 183                 .items = ITEMS(
 184                         RTE_FLOW_ITEM_TYPE_VLAN,
 185                         RTE_FLOW_ITEM_TYPE_IPV4,
 186                         RTE_FLOW_ITEM_TYPE_IPV6),
 187                 .mask = &(const struct rte_flow_item_eth){
 188                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 189                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 190                         .type = -1,
 191                 },
 192                 .mask_sz = sizeof(struct rte_flow_item_eth),
 193                 .default_mask = &rte_flow_item_eth_mask,
 194                 .convert = tap_flow_create_eth,
 195         },
 196         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 197                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
 198                                RTE_FLOW_ITEM_TYPE_IPV6),
 199                 .mask = &(const struct rte_flow_item_vlan){
 200                         .tpid = -1,
 201                         /* DEI matching is not supported */
 202 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 203                         .tci = 0xffef,
 204 #else
 205                         .tci = 0xefff,
 206 #endif
 207                 },
 208                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 209                 .default_mask = &rte_flow_item_vlan_mask,
 210                 .convert = tap_flow_create_vlan,
 211         },
 212         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 213                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 214                                RTE_FLOW_ITEM_TYPE_TCP),
 215                 .mask = &(const struct rte_flow_item_ipv4){
 216                         .hdr = {
 217                                 .src_addr = -1,
 218                                 .dst_addr = -1,
 219                                 .next_proto_id = -1,
 220                         },
 221                 },
 222                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 223                 .default_mask = &rte_flow_item_ipv4_mask,
 224                 .convert = tap_flow_create_ipv4,
 225         },
 226         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 227                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 228                                RTE_FLOW_ITEM_TYPE_TCP),
 229                 .mask = &(const struct rte_flow_item_ipv6){
 230                         .hdr = {
 231                                 .src_addr = {
 232                                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 233                                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 234                                 },
 235                                 .dst_addr = {
 236                                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 237                                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 238                                 },
 239                                 .proto = -1,
 240                         },
 241                 },
 242                 .mask_sz = sizeof(struct rte_flow_item_ipv6),
 243                 .default_mask = &rte_flow_item_ipv6_mask,
 244                 .convert = tap_flow_create_ipv6,
 245         },
 246         [RTE_FLOW_ITEM_TYPE_UDP] = {
 247                 .mask = &(const struct rte_flow_item_udp){
 248                         .hdr = {
 249                                 .src_port = -1,
 250                                 .dst_port = -1,
 251                         },
 252                 },
 253                 .mask_sz = sizeof(struct rte_flow_item_udp),
 254                 .default_mask = &rte_flow_item_udp_mask,
 255                 .convert = tap_flow_create_udp,
 256         },
 257         [RTE_FLOW_ITEM_TYPE_TCP] = {
 258                 .mask = &(const struct rte_flow_item_tcp){
 259                         .hdr = {
 260                                 .src_port = -1,
 261                                 .dst_port = -1,
 262                         },
 263                 },
 264                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 265                 .default_mask = &rte_flow_item_tcp_mask,
 266                 .convert = tap_flow_create_tcp,
 267         },
 268 };
 269
 270 /*
 271  *                TC rules, by growing priority
 272  *
 273  *        Remote netdevice                  Tap netdevice
 274  * +-------------+-------------+  +-------------+-------------+
 275  * |   Ingress   |   Egress    |  |   Ingress   |   Egress    |
 276  * |-------------|-------------|  |-------------|-------------|
 277  * |             |  \       /  |  |             |  REMOTE TX  | prio 1
 278  * |             |   \     /   |  |             |   \     /   | prio 2
 279  * |  EXPLICIT   |    \   /    |  |  EXPLICIT   |    \   /    |   .
 280  * |             |     \ /     |  |             |     \ /     |   .
 281  * |    RULES    |      X      |  |    RULES    |      X      |   .
 282  * |      .      |     / \     |  |      .      |     / \     |   .
 283  * |      .      |    /   \    |  |      .      |    /   \    |   .
 284  * |      .      |   /     \   |  |      .      |   /     \   |   .
 285  * |      .      |  /       \  |  |      .      |  /       \  |   .
 286  *
 287  *      ....           ....           ....           ....
 288  *
 289  * |      .      |  \       /  |  |      .      |  \       /  |   .
 290  * |      .      |   \     /   |  |      .      |   \     /   |   .
 291  * |             |    \   /    |  |             |    \   /    |
 292  * |  LOCAL_MAC  |     \ /     |  |    \   /    |     \ /     | last prio - 5
 293  * |   PROMISC   |      X      |  |     \ /     |      X      | last prio - 4
 294  * |   ALLMULTI  |     / \     |  |      X      |     / \     | last prio - 3
 295  * |  BROADCAST  |    /   \    |  |     / \     |    /   \    | last prio - 2
 296  * | BROADCASTV6 |   /     \   |  |    /   \    |   /     \   | last prio - 1
 297  * |     xx      |  /       \  |  |   ISOLATE   |  /       \  | last prio
 298  * +-------------+-------------+  +-------------+-------------+
 299  *
 300  * The implicit flow rules are stored in a list in with mandatorily the last two
 301  * being the ISOLATE and REMOTE_TX rules. e.g.:
 302  *
 303  * LOCAL_MAC -> BROADCAST -> BROADCASTV6 -> REMOTE_TX -> ISOLATE -> NULL
 304  *
 305  * That enables tap_flow_isolate() to remove implicit rules by popping the list
 306  * head and remove it as long as it applies on the remote netdevice. The
 307  * implicit rule for TX redirection is not removed, as isolate concerns only
 308  * incoming traffic.
 309  */
 310
 311 static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = {
 312         [TAP_REMOTE_LOCAL_MAC] = {
 313                 .attr = {
 314                         .group = MAX_GROUP,
 315                         .priority = PRIORITY_MASK - TAP_REMOTE_LOCAL_MAC,
 316                         .ingress = 1,
 317                 },
 318                 .items[0] = {
 319                         .type = RTE_FLOW_ITEM_TYPE_ETH,
 320                         .mask =  &(const struct rte_flow_item_eth){
 321                                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 322                         },
 323                 },
 324                 .items[1] = {
 325                         .type = RTE_FLOW_ITEM_TYPE_END,
 326                 },
 327                 .mirred = TCA_EGRESS_REDIR,
 328         },
 329         [TAP_REMOTE_BROADCAST] = {
 330                 .attr = {
 331                         .group = MAX_GROUP,
 332                         .priority = PRIORITY_MASK - TAP_REMOTE_BROADCAST,
 333                         .ingress = 1,
 334                 },
 335                 .items[0] = {
 336                         .type = RTE_FLOW_ITEM_TYPE_ETH,
 337                         .mask =  &(const struct rte_flow_item_eth){
 338                                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 339                         },
 340                         .spec = &(const struct rte_flow_item_eth){
 341                                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 342                         },
 343                 },
 344                 .items[1] = {
 345                         .type = RTE_FLOW_ITEM_TYPE_END,
 346                 },
 347                 .mirred = TCA_EGRESS_MIRROR,
 348         },
 349         [TAP_REMOTE_BROADCASTV6] = {
 350                 .attr = {
 351                         .group = MAX_GROUP,
 352                         .priority = PRIORITY_MASK - TAP_REMOTE_BROADCASTV6,
 353                         .ingress = 1,
 354                 },
 355                 .items[0] = {
 356                         .type = RTE_FLOW_ITEM_TYPE_ETH,
 357                         .mask =  &(const struct rte_flow_item_eth){
 358                                 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
 359                         },
 360                         .spec = &(const struct rte_flow_item_eth){
 361                                 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
 362                         },
 363                 },
 364                 .items[1] = {
 365                         .type = RTE_FLOW_ITEM_TYPE_END,
 366                 },
 367                 .mirred = TCA_EGRESS_MIRROR,
 368         },
 369         [TAP_REMOTE_PROMISC] = {
 370                 .attr = {
 371                         .group = MAX_GROUP,
 372                         .priority = PRIORITY_MASK - TAP_REMOTE_PROMISC,
 373                         .ingress = 1,
 374                 },
 375                 .items[0] = {
 376                         .type = RTE_FLOW_ITEM_TYPE_VOID,
 377                 },
 378                 .items[1] = {
 379                         .type = RTE_FLOW_ITEM_TYPE_END,
 380                 },
 381                 .mirred = TCA_EGRESS_MIRROR,
 382         },
 383         [TAP_REMOTE_ALLMULTI] = {
 384                 .attr = {
 385                         .group = MAX_GROUP,
 386                         .priority = PRIORITY_MASK - TAP_REMOTE_ALLMULTI,
 387                         .ingress = 1,
 388                 },
 389                 .items[0] = {
 390                         .type = RTE_FLOW_ITEM_TYPE_ETH,
 391                         .mask =  &(const struct rte_flow_item_eth){
 392                                 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
 393                         },
 394                         .spec = &(const struct rte_flow_item_eth){
 395                                 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
 396                         },
 397                 },
 398                 .items[1] = {
 399                         .type = RTE_FLOW_ITEM_TYPE_END,
 400                 },
 401                 .mirred = TCA_EGRESS_MIRROR,
 402         },
 403         [TAP_REMOTE_TX] = {
 404                 .attr = {
 405                         .group = 0,
 406                         .priority = TAP_REMOTE_TX,
 407                         .egress = 1,
 408                 },
 409                 .items[0] = {
 410                         .type = RTE_FLOW_ITEM_TYPE_VOID,
 411                 },
 412                 .items[1] = {
 413                         .type = RTE_FLOW_ITEM_TYPE_END,
 414                 },
 415                 .mirred = TCA_EGRESS_MIRROR,
 416         },
 417         [TAP_ISOLATE] = {
 418                 .attr = {
 419                         .group = MAX_GROUP,
 420                         .priority = PRIORITY_MASK - TAP_ISOLATE,
 421                         .ingress = 1,
 422                 },
 423                 .items[0] = {
 424                         .type = RTE_FLOW_ITEM_TYPE_VOID,
 425                 },
 426                 .items[1] = {
 427                         .type = RTE_FLOW_ITEM_TYPE_END,
 428                 },
 429         },
 430 };
 431
 432 /**
 433  * Make as much checks as possible on an Ethernet item, and if a flow is
 434  * provided, fill it appropriately with Ethernet info.
 435  *
 436  * @param[in] item
 437  *   Item specification.
 438  * @param[in, out] data
 439  *   Additional data structure to tell next layers we've been here.
 440  *
 441  * @return
 442  *   0 if checks are alright, -1 otherwise.
 443  */
 444 static int
 445 tap_flow_create_eth(const struct rte_flow_item *item, void *data)
 446 {
 447         struct convert_data *info = (struct convert_data *)data;
 448         const struct rte_flow_item_eth *spec = item->spec;
 449         const struct rte_flow_item_eth *mask = item->mask;
 450         struct rte_flow *flow = info->flow;
 451         struct nlmsg *msg;
 452
 453         /* use default mask if none provided */
 454         if (!mask)
 455                 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_ETH].default_mask;
 456         /* TC does not support eth_type masking. Only accept if exact match. */
 457         if (mask->type && mask->type != 0xffff)
 458                 return -1;
 459         if (!spec)
 460                 return 0;
 461         /* store eth_type for consistency if ipv4/6 pattern item comes next */
 462         if (spec->type & mask->type)
 463                 info->eth_type = spec->type;
 464         if (!flow)
 465                 return 0;
 466         msg = &flow->msg;
 467         if (!is_zero_ether_addr(&spec->dst)) {
 468                 nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN,
 469                            &spec->dst.addr_bytes);
 470                 nlattr_add(&msg->nh,
 471                            TCA_FLOWER_KEY_ETH_DST_MASK, ETHER_ADDR_LEN,
 472                            &mask->dst.addr_bytes);
 473         }
 474         if (!is_zero_ether_addr(&mask->src)) {
 475                 nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC, ETHER_ADDR_LEN,
 476                            &spec->src.addr_bytes);
 477                 nlattr_add(&msg->nh,
 478                            TCA_FLOWER_KEY_ETH_SRC_MASK, ETHER_ADDR_LEN,
 479                            &mask->src.addr_bytes);
 480         }
 481         return 0;
 482 }
 483
 484 /**
 485  * Make as much checks as possible on a VLAN item, and if a flow is provided,
 486  * fill it appropriately with VLAN info.
 487  *
 488  * @param[in] item
 489  *   Item specification.
 490  * @param[in, out] data
 491  *   Additional data structure to tell next layers we've been here.
 492  *
 493  * @return
 494  *   0 if checks are alright, -1 otherwise.
 495  */
 496 static int
 497 tap_flow_create_vlan(const struct rte_flow_item *item, void *data)
 498 {
 499         struct convert_data *info = (struct convert_data *)data;
 500         const struct rte_flow_item_vlan *spec = item->spec;
 501         const struct rte_flow_item_vlan *mask = item->mask;
 502         struct rte_flow *flow = info->flow;
 503         struct nlmsg *msg;
 504
 505         /* use default mask if none provided */
 506         if (!mask)
 507                 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_VLAN].default_mask;
 508         /* TC does not support tpid masking. Only accept if exact match. */
 509         if (mask->tpid && mask->tpid != 0xffff)
 510                 return -1;
 511         /* Double-tagging not supported. */
 512         if (spec && mask->tpid && spec->tpid != htons(ETH_P_8021Q))
 513                 return -1;
 514         info->vlan = 1;
 515         if (!flow)
 516                 return 0;
 517         msg = &flow->msg;
 518         msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_8021Q));
 519 #define VLAN_PRIO(tci) ((tci) >> 13)
 520 #define VLAN_ID(tci) ((tci) & 0xfff)
 521         if (!spec)
 522                 return 0;
 523         if (spec->tci) {
 524                 uint16_t tci = ntohs(spec->tci) & mask->tci;
 525                 uint16_t prio = VLAN_PRIO(tci);
 526                 uint8_t vid = VLAN_ID(tci);
 527
 528                 if (prio)
 529                         nlattr_add8(&msg->nh, TCA_FLOWER_KEY_VLAN_PRIO, prio);
 530                 if (vid)
 531                         nlattr_add16(&msg->nh, TCA_FLOWER_KEY_VLAN_ID, vid);
 532         }
 533         return 0;
 534 }
 535
 536 /**
 537  * Make as much checks as possible on an IPv4 item, and if a flow is provided,
 538  * fill it appropriately with IPv4 info.
 539  *
 540  * @param[in] item
 541  *   Item specification.
 542  * @param[in, out] data
 543  *   Additional data structure to tell next layers we've been here.
 544  *
 545  * @return
 546  *   0 if checks are alright, -1 otherwise.
 547  */
 548 static int
 549 tap_flow_create_ipv4(const struct rte_flow_item *item, void *data)
 550 {
 551         struct convert_data *info = (struct convert_data *)data;
 552         const struct rte_flow_item_ipv4 *spec = item->spec;
 553         const struct rte_flow_item_ipv4 *mask = item->mask;
 554         struct rte_flow *flow = info->flow;
 555         struct nlmsg *msg;
 556
 557         /* use default mask if none provided */
 558         if (!mask)
 559                 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV4].default_mask;
 560         /* check that previous eth type is compatible with ipv4 */
 561         if (info->eth_type && info->eth_type != htons(ETH_P_IP))
 562                 return -1;
 563         /* store ip_proto for consistency if udp/tcp pattern item comes next */
 564         if (spec)
 565                 info->ip_proto = spec->hdr.next_proto_id;
 566         if (!flow)
 567                 return 0;
 568         msg = &flow->msg;
 569         if (!info->eth_type)
 570                 info->eth_type = htons(ETH_P_IP);
 571         if (!spec)
 572                 return 0;
 573         if (spec->hdr.dst_addr) {
 574                 nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST,
 575                              spec->hdr.dst_addr);
 576                 nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK,
 577                              mask->hdr.dst_addr);
 578         }
 579         if (spec->hdr.src_addr) {
 580                 nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC,
 581                              spec->hdr.src_addr);
 582                 nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK,
 583                              mask->hdr.src_addr);
 584         }
 585         if (spec->hdr.next_proto_id)
 586                 nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO,
 587                             spec->hdr.next_proto_id);
 588         return 0;
 589 }
 590
 591 /**
 592  * Make as much checks as possible on an IPv6 item, and if a flow is provided,
 593  * fill it appropriately with IPv6 info.
 594  *
 595  * @param[in] item
 596  *   Item specification.
 597  * @param[in, out] data
 598  *   Additional data structure to tell next layers we've been here.
 599  *
 600  * @return
 601  *   0 if checks are alright, -1 otherwise.
 602  */
 603 static int
 604 tap_flow_create_ipv6(const struct rte_flow_item *item, void *data)
 605 {
 606         struct convert_data *info = (struct convert_data *)data;
 607         const struct rte_flow_item_ipv6 *spec = item->spec;
 608         const struct rte_flow_item_ipv6 *mask = item->mask;
 609         struct rte_flow *flow = info->flow;
 610         uint8_t empty_addr[16] = { 0 };
 611         struct nlmsg *msg;
 612
 613         /* use default mask if none provided */
 614         if (!mask)
 615                 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV6].default_mask;
 616         /* check that previous eth type is compatible with ipv6 */
 617         if (info->eth_type && info->eth_type != htons(ETH_P_IPV6))
 618                 return -1;
 619         /* store ip_proto for consistency if udp/tcp pattern item comes next */
 620         if (spec)
 621                 info->ip_proto = spec->hdr.proto;
 622         if (!flow)
 623                 return 0;
 624         msg = &flow->msg;
 625         if (!info->eth_type)
 626                 info->eth_type = htons(ETH_P_IPV6);
 627         if (!spec)
 628                 return 0;
 629         if (memcmp(spec->hdr.dst_addr, empty_addr, 16)) {
 630                 nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST,
 631                            sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr);
 632                 nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK,
 633                            sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr);
 634         }
 635         if (memcmp(spec->hdr.src_addr, empty_addr, 16)) {
 636                 nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC,
 637                            sizeof(spec->hdr.src_addr), &spec->hdr.src_addr);
 638                 nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK,
 639                            sizeof(mask->hdr.src_addr), &mask->hdr.src_addr);
 640         }
 641         if (spec->hdr.proto)
 642                 nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto);
 643         return 0;
 644 }
 645
 646 /**
 647  * Make as much checks as possible on a UDP item, and if a flow is provided,
 648  * fill it appropriately with UDP info.
 649  *
 650  * @param[in] item
 651  *   Item specification.
 652  * @param[in, out] data
 653  *   Additional data structure to tell next layers we've been here.
 654  *
 655  * @return
 656  *   0 if checks are alright, -1 otherwise.
 657  */
 658 static int
 659 tap_flow_create_udp(const struct rte_flow_item *item, void *data)
 660 {
 661         struct convert_data *info = (struct convert_data *)data;
 662         const struct rte_flow_item_udp *spec = item->spec;
 663         const struct rte_flow_item_udp *mask = item->mask;
 664         struct rte_flow *flow = info->flow;
 665         struct nlmsg *msg;
 666
 667         /* use default mask if none provided */
 668         if (!mask)
 669                 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_UDP].default_mask;
 670         /* check that previous ip_proto is compatible with udp */
 671         if (info->ip_proto && info->ip_proto != IPPROTO_UDP)
 672                 return -1;
 673         /* TC does not support UDP port masking. Only accept if exact match. */
 674         if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
 675             (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
 676                 return -1;
 677         if (!flow)
 678                 return 0;
 679         msg = &flow->msg;
 680         nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP);
 681         if (!spec)
 682                 return 0;
 683         if (spec->hdr.dst_port & mask->hdr.dst_port)
 684                 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST,
 685                              spec->hdr.dst_port);
 686         if (spec->hdr.src_port & mask->hdr.src_port)
 687                 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC,
 688                              spec->hdr.src_port);
 689         return 0;
 690 }
 691
 692 /**
 693  * Make as much checks as possible on a TCP item, and if a flow is provided,
 694  * fill it appropriately with TCP info.
 695  *
 696  * @param[in] item
 697  *   Item specification.
 698  * @param[in, out] data
 699  *   Additional data structure to tell next layers we've been here.
 700  *
 701  * @return
 702  *   0 if checks are alright, -1 otherwise.
 703  */
 704 static int
 705 tap_flow_create_tcp(const struct rte_flow_item *item, void *data)
 706 {
 707         struct convert_data *info = (struct convert_data *)data;
 708         const struct rte_flow_item_tcp *spec = item->spec;
 709         const struct rte_flow_item_tcp *mask = item->mask;
 710         struct rte_flow *flow = info->flow;
 711         struct nlmsg *msg;
 712
 713         /* use default mask if none provided */
 714         if (!mask)
 715                 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_TCP].default_mask;
 716         /* check that previous ip_proto is compatible with tcp */
 717         if (info->ip_proto && info->ip_proto != IPPROTO_TCP)
 718                 return -1;
 719         /* TC does not support TCP port masking. Only accept if exact match. */
 720         if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
 721             (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
 722                 return -1;
 723         if (!flow)
 724                 return 0;
 725         msg = &flow->msg;
 726         nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP);
 727         if (!spec)
 728                 return 0;
 729         if (spec->hdr.dst_port & mask->hdr.dst_port)
 730                 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST,
 731                              spec->hdr.dst_port);
 732         if (spec->hdr.src_port & mask->hdr.src_port)
 733                 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC,
 734                              spec->hdr.src_port);
 735         return 0;
 736 }
 737
 738 /**
 739  * Check support for a given item.
 740  *
 741  * @param[in] item
 742  *   Item specification.
 743  * @param size
 744  *   Bit-Mask size in bytes.
 745  * @param[in] supported_mask
 746  *   Bit-mask covering supported fields to compare with spec, last and mask in
 747  *   \item.
 748  * @param[in] default_mask
 749  *   Bit-mask default mask if none is provided in \item.
 750  *
 751  * @return
 752  *   0 on success.
 753  */
 754 static int
 755 tap_flow_item_validate(const struct rte_flow_item *item,
 756                        unsigned int size,
 757                        const uint8_t *supported_mask,
 758                        const uint8_t *default_mask)
 759 {
 760         int ret = 0;
 761
 762         /* An empty layer is allowed, as long as all fields are NULL */
 763         if (!item->spec && (item->mask || item->last))
 764                 return -1;
 765         /* Is the item spec compatible with what the NIC supports? */
 766         if (item->spec && !item->mask) {
 767                 unsigned int i;
 768                 const uint8_t *spec = item->spec;
 769
 770                 for (i = 0; i < size; ++i)
 771                         if ((spec[i] | supported_mask[i]) != supported_mask[i])
 772                                 return -1;
 773                 /* Is the default mask compatible with what the NIC supports? */
 774                 for (i = 0; i < size; i++)
 775                         if ((default_mask[i] | supported_mask[i]) !=
 776                             supported_mask[i])
 777                                 return -1;
 778         }
 779         /* Is the item last compatible with what the NIC supports? */
 780         if (item->last && !item->mask) {
 781                 unsigned int i;
 782                 const uint8_t *spec = item->last;
 783
 784                 for (i = 0; i < size; ++i)
 785                         if ((spec[i] | supported_mask[i]) != supported_mask[i])
 786                                 return -1;
 787         }
 788         /* Is the item mask compatible with what the NIC supports? */
 789         if (item->mask) {
 790                 unsigned int i;
 791                 const uint8_t *spec = item->mask;
 792
 793                 for (i = 0; i < size; ++i)
 794                         if ((spec[i] | supported_mask[i]) != supported_mask[i])
 795                                 return -1;
 796         }
 797         /**
 798          * Once masked, Are item spec and item last equal?
 799          * TC does not support range so anything else is invalid.
 800          */
 801         if (item->spec && item->last) {
 802                 uint8_t spec[size];
 803                 uint8_t last[size];
 804                 const uint8_t *apply = default_mask;
 805                 unsigned int i;
 806
 807                 if (item->mask)
 808                         apply = item->mask;
 809                 for (i = 0; i < size; ++i) {
 810                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 811                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 812                 }
 813                 ret = memcmp(spec, last, size);
 814         }
 815         return ret;
 816 }
 817
 818 /**
 819  * Transform a DROP/PASSTHRU action item in the provided flow for TC.
 820  *
 821  * @param[in, out] flow
 822  *   Flow to be filled.
 823  * @param[in] action
 824  *   Appropriate action to be set in the TCA_GACT_PARMS structure.
 825  *
 826  * @return
 827  *   0 if checks are alright, -1 otherwise.
 828  */
 829 static int
 830 add_action_gact(struct rte_flow *flow, int action)
 831 {
 832         struct nlmsg *msg = &flow->msg;
 833         size_t act_index = 1;
 834         struct tc_gact p = {
 835                 .action = action
 836         };
 837
 838         if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0)
 839                 return -1;
 840         if (nlattr_nested_start(msg, act_index++) < 0)
 841                 return -1;
 842         nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("gact"), "gact");
 843         if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
 844                 return -1;
 845         nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(p), &p);
 846         nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
 847         nlattr_nested_finish(msg); /* nested act_index */
 848         nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
 849         return 0;
 850 }
 851
 852 /**
 853  * Transform a MIRRED action item in the provided flow for TC.
 854  *
 855  * @param[in, out] flow
 856  *   Flow to be filled.
 857  * @param[in] ifindex
 858  *   Netdevice ifindex, where to mirror/redirect packet to.
 859  * @param[in] action_type
 860  *   Either TCA_EGRESS_REDIR for redirection or TCA_EGRESS_MIRROR for mirroring.
 861  *
 862  * @return
 863  *   0 if checks are alright, -1 otherwise.
 864  */
 865 static int
 866 add_action_mirred(struct rte_flow *flow, uint16_t ifindex, uint16_t action_type)
 867 {
 868         struct nlmsg *msg = &flow->msg;
 869         size_t act_index = 1;
 870         struct tc_mirred p = {
 871                 .eaction = action_type,
 872                 .ifindex = ifindex,
 873         };
 874
 875         if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0)
 876                 return -1;
 877         if (nlattr_nested_start(msg, act_index++) < 0)
 878                 return -1;
 879         nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("mirred"), "mirred");
 880         if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
 881                 return -1;
 882         if (action_type == TCA_EGRESS_MIRROR)
 883                 p.action = TC_ACT_PIPE;
 884         else /* REDIRECT */
 885                 p.action = TC_ACT_STOLEN;
 886         nlattr_add(&msg->nh, TCA_MIRRED_PARMS, sizeof(p), &p);
 887         nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
 888         nlattr_nested_finish(msg); /* nested act_index */
 889         nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
 890         return 0;
 891 }
 892
 893 /**
 894  * Transform a QUEUE action item in the provided flow for TC.
 895  *
 896  * @param[in, out] flow
 897  *   Flow to be filled.
 898  * @param[in] queue
 899  *   Queue id to use.
 900  *
 901  * @return
 902  *   0 if checks are alright, -1 otherwise.
 903  */
 904 static int
 905 add_action_skbedit(struct rte_flow *flow, uint16_t queue)
 906 {
 907         struct nlmsg *msg = &flow->msg;
 908         size_t act_index = 1;
 909         struct tc_skbedit p = {
 910                 .action = TC_ACT_PIPE
 911         };
 912
 913         if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0)
 914                 return -1;
 915         if (nlattr_nested_start(msg, act_index++) < 0)
 916                 return -1;
 917         nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("skbedit"), "skbedit");
 918         if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
 919                 return -1;
 920         nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS, sizeof(p), &p);
 921         nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING, queue);
 922         nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
 923         nlattr_nested_finish(msg); /* nested act_index */
 924         nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
 925         return 0;
 926 }
 927
 928 /**
 929  * Validate a flow supported by TC.
 930  * If flow param is not NULL, then also fill the netlink message inside.
 931  *
 932  * @param pmd
 933  *   Pointer to private structure.
 934  * @param[in] attr
 935  *   Flow rule attributes.
 936  * @param[in] pattern
 937  *   Pattern specification (list terminated by the END pattern item).
 938  * @param[in] actions
 939  *   Associated actions (list terminated by the END action).
 940  * @param[out] error
 941  *   Perform verbose error reporting if not NULL.
 942  * @param[in, out] flow
 943  *   Flow structure to update.
 944  * @param[in] mirred
 945  *   If set to TCA_EGRESS_REDIR, provided actions will be replaced with a
 946  *   redirection to the tap netdevice, and the TC rule will be configured
 947  *   on the remote netdevice in pmd.
 948  *   If set to TCA_EGRESS_MIRROR, provided actions will be replaced with a
 949  *   mirroring to the tap netdevice, and the TC rule will be configured
 950  *   on the remote netdevice in pmd. Matching packets will thus be duplicated.
 951  *   If set to 0, the standard behavior is to be used: set correct actions for
 952  *   the TC rule, and apply it on the tap netdevice.
 953  *
 954  * @return
 955  *   0 on success, a negative errno value otherwise and rte_errno is set.
 956  */
 957 static int
 958 priv_flow_process(struct pmd_internals *pmd,
 959                   const struct rte_flow_attr *attr,
 960                   const struct rte_flow_item items[],
 961                   const struct rte_flow_action actions[],
 962                   struct rte_flow_error *error,
 963                   struct rte_flow *flow,
 964                   int mirred)
 965 {
 966         const struct tap_flow_items *cur_item = tap_flow_items;
 967         struct convert_data data = {
 968                 .eth_type = 0,
 969                 .ip_proto = 0,
 970                 .flow = flow,
 971         };
 972         int action = 0; /* Only one action authorized for now */
 973
 974         if (attr->group > MAX_GROUP) {
 975                 rte_flow_error_set(
 976                         error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 977                         NULL, "group value too big: cannot exceed 15");
 978                 return -rte_errno;
 979         }
 980         if (attr->priority > MAX_PRIORITY) {
 981                 rte_flow_error_set(
 982                         error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 983                         NULL, "priority value too big");
 984                 return -rte_errno;
 985         } else if (flow) {
 986                 uint16_t group = attr->group << GROUP_SHIFT;
 987                 uint16_t prio = group | (attr->priority + PRIORITY_OFFSET);
 988                 flow->msg.t.tcm_info = TC_H_MAKE(prio << 16,
 989                                                  flow->msg.t.tcm_info);
 990         }
 991         if (flow) {
 992                 if (mirred) {
 993                         /*
 994                          * If attr->ingress, the rule applies on remote ingress
 995                          * to match incoming packets
 996                          * If attr->egress, the rule applies on tap ingress (as
 997                          * seen from the kernel) to deal with packets going out
 998                          * from the DPDK app.
 999                          */
1000                         flow->msg.t.tcm_parent = TC_H_MAKE(TC_H_INGRESS, 0);
1001                 } else {
1002                         /* Standard rule on tap egress (kernel standpoint). */
1003                         flow->msg.t.tcm_parent =
1004                                 TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0);
1005                 }
1006                 /* use flower filter type */
1007                 nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower");
1008                 if (nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0)
1009                         goto exit_item_not_supported;
1010         }
1011         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1012                 const struct tap_flow_items *token = NULL;
1013                 unsigned int i;
1014                 int err = 0;
1015
1016                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1017                         continue;
1018                 for (i = 0;
1019                      cur_item->items &&
1020                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
1021                      ++i) {
1022                         if (cur_item->items[i] == items->type) {
1023                                 token = &tap_flow_items[items->type];
1024                                 break;
1025                         }
1026                 }
1027                 if (!token)
1028                         goto exit_item_not_supported;
1029                 cur_item = token;
1030                 err = tap_flow_item_validate(
1031                         items, cur_item->mask_sz,
1032                         (const uint8_t *)cur_item->mask,
1033                         (const uint8_t *)cur_item->default_mask);
1034                 if (err)
1035                         goto exit_item_not_supported;
1036                 if (flow && cur_item->convert) {
1037                         if (!pmd->flower_vlan_support &&
1038                             cur_item->convert == tap_flow_create_vlan)
1039                                 goto exit_item_not_supported;
1040                         err = cur_item->convert(items, &data);
1041                         if (err)
1042                                 goto exit_item_not_supported;
1043                 }
1044         }
1045         if (flow) {
1046                 if (pmd->flower_vlan_support && data.vlan) {
1047                         nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1048                                      htons(ETH_P_8021Q));
1049                         nlattr_add16(&flow->msg.nh,
1050                                      TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1051                                      data.eth_type ?
1052                                      data.eth_type : htons(ETH_P_ALL));
1053                 } else if (data.eth_type) {
1054                         nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1055                                      data.eth_type);
1056                 }
1057         }
1058         if (mirred && flow) {
1059                 uint16_t if_index = pmd->if_index;
1060
1061                 /*
1062                  * If attr->egress && mirred, then this is a special
1063                  * case where the rule must be applied on the tap, to
1064                  * redirect packets coming from the DPDK App, out
1065                  * through the remote netdevice.
1066                  */
1067                 if (attr->egress)
1068                         if_index = pmd->remote_if_index;
1069                 if (add_action_mirred(flow, if_index, mirred) < 0)
1070                         goto exit_action_not_supported;
1071                 else
1072                         goto end;
1073         }
1074         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1075                 int err = 0;
1076
1077                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1078                         continue;
1079                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1080                         if (action)
1081                                 goto exit_action_not_supported;
1082                         action = 1;
1083                         if (flow)
1084                                 err = add_action_gact(flow, TC_ACT_SHOT);
1085                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_PASSTHRU) {
1086                         if (action)
1087                                 goto exit_action_not_supported;
1088                         action = 1;
1089                         if (flow)
1090                                 err = add_action_gact(flow, TC_ACT_UNSPEC);
1091                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1092                         const struct rte_flow_action_queue *queue =
1093                                 (const struct rte_flow_action_queue *)
1094                                 actions->conf;
1095                         if (action)
1096                                 goto exit_action_not_supported;
1097                         action = 1;
1098                         if (!queue || (queue->index >= pmd->nb_queues))
1099                                 goto exit_action_not_supported;
1100                         if (flow)
1101                                 err = add_action_skbedit(flow, queue->index);
1102                 } else {
1103                         goto exit_action_not_supported;
1104                 }
1105                 if (err)
1106                         goto exit_action_not_supported;
1107         }
1108 end:
1109         if (flow)
1110                 nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */
1111         return 0;
1112 exit_item_not_supported:
1113         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1114                            items, "item not supported");
1115         return -rte_errno;
1116 exit_action_not_supported:
1117         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1118                            actions, "action not supported");
1119         return -rte_errno;
1120 }
1121
1122
1123
1124 /**
1125  * Validate a flow.
1126  *
1127  * @see rte_flow_validate()
1128  * @see rte_flow_ops
1129  */
1130 static int
1131 tap_flow_validate(struct rte_eth_dev *dev,
1132                   const struct rte_flow_attr *attr,
1133                   const struct rte_flow_item items[],
1134                   const struct rte_flow_action actions[],
1135                   struct rte_flow_error *error)
1136 {
1137         struct pmd_internals *pmd = dev->data->dev_private;
1138
1139         return priv_flow_process(pmd, attr, items, actions, error, NULL, 0);
1140 }
1141
1142 /**
1143  * Set a unique handle in a flow.
1144  *
1145  * The kernel supports TC rules with equal priority, as long as they use the
1146  * same matching fields (e.g.: dst mac and ipv4) with different values (and
1147  * full mask to ensure no collision is possible).
1148  * In those rules, the handle (uint32_t) is the part that would identify
1149  * specifically each rule.
1150  *
1151  * On 32-bit architectures, the handle can simply be the flow's pointer address.
1152  * On 64-bit architectures, we rely on jhash(flow) to find a (sufficiently)
1153  * unique handle.
1154  *
1155  * @param[in, out] flow
1156  *   The flow that needs its handle set.
1157  */
1158 static void
1159 tap_flow_set_handle(struct rte_flow *flow)
1160 {
1161         uint32_t handle = 0;
1162
1163         if (sizeof(flow) > 4)
1164                 handle = rte_jhash(&flow, sizeof(flow), 1);
1165         else
1166                 handle = (uintptr_t)flow;
1167         /* must be at least 1 to avoid letting the kernel choose one for us */
1168         if (!handle)
1169                 handle = 1;
1170         flow->msg.t.tcm_handle = handle;
1171 }
1172
1173 /**
1174  * Create a flow.
1175  *
1176  * @see rte_flow_create()
1177  * @see rte_flow_ops
1178  */
1179 static struct rte_flow *
1180 tap_flow_create(struct rte_eth_dev *dev,
1181                 const struct rte_flow_attr *attr,
1182                 const struct rte_flow_item items[],
1183                 const struct rte_flow_action actions[],
1184                 struct rte_flow_error *error)
1185 {
1186         struct pmd_internals *pmd = dev->data->dev_private;
1187         struct rte_flow *remote_flow = NULL;
1188         struct rte_flow *flow = NULL;
1189         struct nlmsg *msg = NULL;
1190         int err;
1191
1192         if (!pmd->if_index) {
1193                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1194                                    NULL,
1195                                    "can't create rule, ifindex not found");
1196                 goto fail;
1197         }
1198         /*
1199          * No rules configured through standard rte_flow should be set on the
1200          * priorities used by implicit rules.
1201          */
1202         if ((attr->group == MAX_GROUP) &&
1203             attr->priority > (MAX_PRIORITY - TAP_REMOTE_MAX_IDX)) {
1204                 rte_flow_error_set(
1205                         error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1206                         NULL, "priority value too big");
1207                 goto fail;
1208         }
1209         flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1210         if (!flow) {
1211                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1212                                    NULL, "cannot allocate memory for rte_flow");
1213                 goto fail;
1214         }
1215         msg = &flow->msg;
1216         tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER,
1217                     NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1218         msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1219         tap_flow_set_handle(flow);
1220         if (priv_flow_process(pmd, attr, items, actions, error, flow, 0))
1221                 goto fail;
1222         err = nl_send(pmd->nlsk_fd, &msg->nh);
1223         if (err < 0) {
1224                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1225                                    NULL, "couldn't send request to kernel");
1226                 goto fail;
1227         }
1228         err = nl_recv_ack(pmd->nlsk_fd);
1229         if (err < 0) {
1230                 RTE_LOG(ERR, PMD,
1231                         "Kernel refused TC filter rule creation (%d): %s\n",
1232                         errno, strerror(errno));
1233                 rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_HANDLE,
1234                                    NULL, "overlapping rules");
1235                 goto fail;
1236         }
1237         LIST_INSERT_HEAD(&pmd->flows, flow, next);
1238         /**
1239          * If a remote device is configured, a TC rule with identical items for
1240          * matching must be set on that device, with a single action: redirect
1241          * to the local pmd->if_index.
1242          */
1243         if (pmd->remote_if_index) {
1244                 remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1245                 if (!remote_flow) {
1246                         rte_flow_error_set(
1247                                 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1248                                 "cannot allocate memory for rte_flow");
1249                         goto fail;
1250                 }
1251                 msg = &remote_flow->msg;
1252                 /* set the rule if_index for the remote netdevice */
1253                 tc_init_msg(
1254                         msg, pmd->remote_if_index, RTM_NEWTFILTER,
1255                         NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1256                 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1257                 tap_flow_set_handle(remote_flow);
1258                 if (priv_flow_process(pmd, attr, items, NULL,
1259                                       error, remote_flow, TCA_EGRESS_REDIR)) {
1260                         rte_flow_error_set(
1261                                 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1262                                 NULL, "rte flow rule validation failed");
1263                         goto fail;
1264                 }
1265                 err = nl_send(pmd->nlsk_fd, &msg->nh);
1266                 if (err < 0) {
1267                         rte_flow_error_set(
1268                                 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1269                                 NULL, "Failure sending nl request");
1270                         goto fail;
1271                 }
1272                 err = nl_recv_ack(pmd->nlsk_fd);
1273                 if (err < 0) {
1274                         RTE_LOG(ERR, PMD,
1275                                 "Kernel refused TC filter rule creation (%d): %s\n",
1276                                 errno, strerror(errno));
1277                         rte_flow_error_set(
1278                                 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1279                                 NULL, "overlapping rules");
1280                         goto fail;
1281                 }
1282                 flow->remote_flow = remote_flow;
1283         }
1284         return flow;
1285 fail:
1286         if (remote_flow)
1287                 rte_free(remote_flow);
1288         if (flow)
1289                 rte_free(flow);
1290         return NULL;
1291 }
1292
1293 /**
1294  * Destroy a flow using pointer to pmd_internal.
1295  *
1296  * @param[in, out] pmd
1297  *   Pointer to private structure.
1298  * @param[in] flow
1299  *   Pointer to the flow to destroy.
1300  * @param[in, out] error
1301  *   Pointer to the flow error handler
1302  *
1303  * @return 0 if the flow could be destroyed, -1 otherwise.
1304  */
1305 static int
1306 tap_flow_destroy_pmd(struct pmd_internals *pmd,
1307                      struct rte_flow *flow,
1308                      struct rte_flow_error *error)
1309 {
1310         struct rte_flow *remote_flow = flow->remote_flow;
1311         int ret = 0;
1312
1313         LIST_REMOVE(flow, next);
1314         flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1315         flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1316
1317         ret = nl_send(pmd->nlsk_fd, &flow->msg.nh);
1318         if (ret < 0) {
1319                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1320                                    NULL, "couldn't send request to kernel");
1321                 goto end;
1322         }
1323         ret = nl_recv_ack(pmd->nlsk_fd);
1324         /* If errno is ENOENT, the rule is already no longer in the kernel. */
1325         if (ret < 0 && errno == ENOENT)
1326                 ret = 0;
1327         if (ret < 0) {
1328                 RTE_LOG(ERR, PMD,
1329                         "Kernel refused TC filter rule deletion (%d): %s\n",
1330                         errno, strerror(errno));
1331                 rte_flow_error_set(
1332                         error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1333                         "couldn't receive kernel ack to our request");
1334                 goto end;
1335         }
1336         if (remote_flow) {
1337                 remote_flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1338                 remote_flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1339
1340                 ret = nl_send(pmd->nlsk_fd, &remote_flow->msg.nh);
1341                 if (ret < 0) {
1342                         rte_flow_error_set(
1343                                 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1344                                 NULL, "Failure sending nl request");
1345                         goto end;
1346                 }
1347                 ret = nl_recv_ack(pmd->nlsk_fd);
1348                 if (ret < 0 && errno == ENOENT)
1349                         ret = 0;
1350                 if (ret < 0) {
1351                         RTE_LOG(ERR, PMD,
1352                                 "Kernel refused TC filter rule deletion (%d): %s\n",
1353                                 errno, strerror(errno));
1354                         rte_flow_error_set(
1355                                 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1356                                 NULL, "Failure trying to receive nl ack");
1357                         goto end;
1358                 }
1359         }
1360 end:
1361         if (remote_flow)
1362                 rte_free(remote_flow);
1363         rte_free(flow);
1364         return ret;
1365 }
1366
1367 /**
1368  * Destroy a flow.
1369  *
1370  * @see rte_flow_destroy()
1371  * @see rte_flow_ops
1372  */
1373 static int
1374 tap_flow_destroy(struct rte_eth_dev *dev,
1375                  struct rte_flow *flow,
1376                  struct rte_flow_error *error)
1377 {
1378         struct pmd_internals *pmd = dev->data->dev_private;
1379
1380         return tap_flow_destroy_pmd(pmd, flow, error);
1381 }
1382
1383 /**
1384  * Enable/disable flow isolation.
1385  *
1386  * @see rte_flow_isolate()
1387  * @see rte_flow_ops
1388  */
1389 static int
1390 tap_flow_isolate(struct rte_eth_dev *dev,
1391                  int set,
1392                  struct rte_flow_error *error __rte_unused)
1393 {
1394         struct pmd_internals *pmd = dev->data->dev_private;
1395
1396         if (!pmd->flower_support)
1397                 return -rte_flow_error_set(
1398                         error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1399                         "rte_flow isolate requires TC flower kernel support");
1400         if (set)
1401                 pmd->flow_isolate = 1;
1402         else
1403                 pmd->flow_isolate = 0;
1404         /*
1405          * If netdevice is there, setup appropriate flow rules immediately.
1406          * Otherwise it will be set when bringing up the netdevice (tun_alloc).
1407          */
1408         if (!pmd->rxq[0].fd)
1409                 return 0;
1410         if (set) {
1411                 struct rte_flow *flow;
1412
1413                 while (1) {
1414                         flow = LIST_FIRST(&pmd->implicit_flows);
1415                         if (!flow)
1416                                 break;
1417                         /*
1418                          * Remove all implicit rules on the remote.
1419                          * Keep the local rule to redirect packets on TX.
1420                          * Keep also the last implicit local rule: ISOLATE.
1421                          */
1422                         if (flow->msg.t.tcm_ifindex == pmd->if_index)
1423                                 break;
1424                         if (tap_flow_destroy_pmd(pmd, flow, NULL) < 0)
1425                                 goto error;
1426                 }
1427                 /* Switch the TC rule according to pmd->flow_isolate */
1428                 if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1)
1429                         goto error;
1430         } else {
1431                 /* Switch the TC rule according to pmd->flow_isolate */
1432                 if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1)
1433                         goto error;
1434                 if (!pmd->remote_if_index)
1435                         return 0;
1436                 if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0)
1437                         goto error;
1438                 if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0)
1439                         goto error;
1440                 if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0)
1441                         goto error;
1442                 if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0)
1443                         goto error;
1444                 if (dev->data->promiscuous &&
1445                     tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC) < 0)
1446                         goto error;
1447                 if (dev->data->all_multicast &&
1448                     tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI) < 0)
1449                         goto error;
1450         }
1451         return 0;
1452 error:
1453         pmd->flow_isolate = 0;
1454         return -rte_flow_error_set(
1455                 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1456                 "TC rule creation failed");
1457 }
1458
1459 /**
1460  * Destroy all flows.
1461  *
1462  * @see rte_flow_flush()
1463  * @see rte_flow_ops
1464  */
1465 int
1466 tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
1467 {
1468         struct pmd_internals *pmd = dev->data->dev_private;
1469         struct rte_flow *flow;
1470
1471         while (!LIST_EMPTY(&pmd->flows)) {
1472                 flow = LIST_FIRST(&pmd->flows);
1473                 if (tap_flow_destroy(dev, flow, error) < 0)
1474                         return -1;
1475         }
1476         return 0;
1477 }
1478
1479 /**
1480  * Add an implicit flow rule on the remote device to make sure traffic gets to
1481  * the tap netdevice from there.
1482  *
1483  * @param pmd
1484  *   Pointer to private structure.
1485  * @param[in] idx
1486  *   The idx in the implicit_rte_flows array specifying which rule to apply.
1487  *
1488  * @return -1 if the rule couldn't be applied, 0 otherwise.
1489  */
1490 int tap_flow_implicit_create(struct pmd_internals *pmd,
1491                              enum implicit_rule_index idx)
1492 {
1493         uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE;
1494         struct rte_flow_action *actions = implicit_rte_flows[idx].actions;
1495         struct rte_flow_action isolate_actions[2] = {
1496                 [1] = {
1497                         .type = RTE_FLOW_ACTION_TYPE_END,
1498                 },
1499         };
1500         struct rte_flow_item *items = implicit_rte_flows[idx].items;
1501         struct rte_flow_attr *attr = &implicit_rte_flows[idx].attr;
1502         struct rte_flow_item_eth eth_local = { .type = 0 };
1503         uint16_t if_index = pmd->remote_if_index;
1504         struct rte_flow *remote_flow = NULL;
1505         struct nlmsg *msg = NULL;
1506         int err = 0;
1507         struct rte_flow_item items_local[2] = {
1508                 [0] = {
1509                         .type = items[0].type,
1510                         .spec = &eth_local,
1511                         .mask = items[0].mask,
1512                 },
1513                 [1] = {
1514                         .type = items[1].type,
1515                 }
1516         };
1517
1518         remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1519         if (!remote_flow) {
1520                 RTE_LOG(ERR, PMD, "Cannot allocate memory for rte_flow");
1521                 goto fail;
1522         }
1523         msg = &remote_flow->msg;
1524         if (idx == TAP_REMOTE_TX) {
1525                 if_index = pmd->if_index;
1526         } else if (idx == TAP_ISOLATE) {
1527                 if_index = pmd->if_index;
1528                 /* Don't be exclusive for this rule, it can be changed later. */
1529                 flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE;
1530                 isolate_actions[0].type = pmd->flow_isolate ?
1531                         RTE_FLOW_ACTION_TYPE_DROP :
1532                         RTE_FLOW_ACTION_TYPE_PASSTHRU;
1533                 actions = isolate_actions;
1534         } else if (idx == TAP_REMOTE_LOCAL_MAC) {
1535                 /*
1536                  * eth addr couldn't be set in implicit_rte_flows[] as it is not
1537                  * known at compile time.
1538                  */
1539                 memcpy(&eth_local.dst, &pmd->eth_addr, sizeof(pmd->eth_addr));
1540                 items = items_local;
1541         }
1542         tc_init_msg(msg, if_index, RTM_NEWTFILTER, flags);
1543         msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1544         /*
1545          * The ISOLATE rule is always present and must have a static handle, as
1546          * the action is changed whether the feature is enabled (DROP) or
1547          * disabled (PASSTHRU).
1548          */
1549         if (idx == TAP_ISOLATE)
1550                 remote_flow->msg.t.tcm_handle = ISOLATE_HANDLE;
1551         else
1552                 tap_flow_set_handle(remote_flow);
1553         if (priv_flow_process(pmd, attr, items, actions, NULL,
1554                               remote_flow, implicit_rte_flows[idx].mirred)) {
1555                 RTE_LOG(ERR, PMD, "rte flow rule validation failed\n");
1556                 goto fail;
1557         }
1558         err = nl_send(pmd->nlsk_fd, &msg->nh);
1559         if (err < 0) {
1560                 RTE_LOG(ERR, PMD, "Failure sending nl request");
1561                 goto fail;
1562         }
1563         err = nl_recv_ack(pmd->nlsk_fd);
1564         if (err < 0) {
1565                 RTE_LOG(ERR, PMD,
1566                         "Kernel refused TC filter rule creation (%d): %s\n",
1567                         errno, strerror(errno));
1568                 goto fail;
1569         }
1570         LIST_INSERT_HEAD(&pmd->implicit_flows, remote_flow, next);
1571         return 0;
1572 fail:
1573         if (remote_flow)
1574                 rte_free(remote_flow);
1575         return -1;
1576 }
1577
1578 /**
1579  * Remove specific implicit flow rule on the remote device.
1580  *
1581  * @param[in, out] pmd
1582  *   Pointer to private structure.
1583  * @param[in] idx
1584  *   The idx in the implicit_rte_flows array specifying which rule to remove.
1585  *
1586  * @return -1 if one of the implicit rules couldn't be created, 0 otherwise.
1587  */
1588 int tap_flow_implicit_destroy(struct pmd_internals *pmd,
1589                               enum implicit_rule_index idx)
1590 {
1591         struct rte_flow *remote_flow;
1592         int cur_prio = -1;
1593         int idx_prio = implicit_rte_flows[idx].attr.priority + PRIORITY_OFFSET;
1594
1595         for (remote_flow = LIST_FIRST(&pmd->implicit_flows);
1596              remote_flow;
1597              remote_flow = LIST_NEXT(remote_flow, next)) {
1598                 cur_prio = (remote_flow->msg.t.tcm_info >> 16) & PRIORITY_MASK;
1599                 if (cur_prio != idx_prio)
1600                         continue;
1601                 return tap_flow_destroy_pmd(pmd, remote_flow, NULL);
1602         }
1603         return 0;
1604 }
1605
1606 /**
1607  * Destroy all implicit flows.
1608  *
1609  * @see rte_flow_flush()
1610  */
1611 int
1612 tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error)
1613 {
1614         struct rte_flow *remote_flow;
1615
1616         while (!LIST_EMPTY(&pmd->implicit_flows)) {
1617                 remote_flow = LIST_FIRST(&pmd->implicit_flows);
1618                 if (tap_flow_destroy_pmd(pmd, remote_flow, error) < 0)
1619                         return -1;
1620         }
1621         return 0;
1622 }
1623
1624 /**
1625  * Manage filter operations.
1626  *
1627  * @param dev
1628  *   Pointer to Ethernet device structure.
1629  * @param filter_type
1630  *   Filter type.
1631  * @param filter_op
1632  *   Operation to perform.
1633  * @param arg
1634  *   Pointer to operation-specific structure.
1635  *
1636  * @return
1637  *   0 on success, negative errno value on failure.
1638  */
1639 int
1640 tap_dev_filter_ctrl(struct rte_eth_dev *dev,
1641                     enum rte_filter_type filter_type,
1642                     enum rte_filter_op filter_op,
1643                     void *arg)
1644 {
1645         struct pmd_internals *pmd = dev->data->dev_private;
1646
1647         if (!pmd->flower_support)
1648                 return -ENOTSUP;
1649         switch (filter_type) {
1650         case RTE_ETH_FILTER_GENERIC:
1651                 if (filter_op != RTE_ETH_FILTER_GET)
1652                         return -EINVAL;
1653                 *(const void **)arg = &tap_flow_ops;
1654                 return 0;
1655         default:
1656                 RTE_LOG(ERR, PMD, "%p: filter type (%d) not supported",
1657                         (void *)dev, filter_type);
1658         }
1659         return -EINVAL;
1660 }
1661