drivers/net/mlx5/mlx5_flow_tcf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <assert.h>
   7 #include <errno.h>
   8 #include <libmnl/libmnl.h>
   9 #include <linux/gen_stats.h>
  10 #include <linux/if_ether.h>
  11 #include <linux/netlink.h>
  12 #include <linux/pkt_cls.h>
  13 #include <linux/pkt_sched.h>
  14 #include <linux/rtnetlink.h>
  15 #include <linux/tc_act/tc_gact.h>
  16 #include <linux/tc_act/tc_mirred.h>
  17 #include <netinet/in.h>
  18 #include <stdalign.h>
  19 #include <stdbool.h>
  20 #include <stddef.h>
  21 #include <stdint.h>
  22 #include <stdlib.h>
  23 #include <sys/socket.h>
  24
  25 #include <rte_byteorder.h>
  26 #include <rte_errno.h>
  27 #include <rte_ether.h>
  28 #include <rte_flow.h>
  29 #include <rte_malloc.h>
  30 #include <rte_common.h>
  31
  32 #include "mlx5.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_autoconf.h"
  35
  36 #ifdef HAVE_TC_ACT_VLAN
  37
  38 #include <linux/tc_act/tc_vlan.h>
  39
  40 #else /* HAVE_TC_ACT_VLAN */
  41
  42 #define TCA_VLAN_ACT_POP 1
  43 #define TCA_VLAN_ACT_PUSH 2
  44 #define TCA_VLAN_ACT_MODIFY 3
  45 #define TCA_VLAN_PARMS 2
  46 #define TCA_VLAN_PUSH_VLAN_ID 3
  47 #define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
  48 #define TCA_VLAN_PAD 5
  49 #define TCA_VLAN_PUSH_VLAN_PRIORITY 6
  50
  51 struct tc_vlan {
  52         tc_gen;
  53         int v_action;
  54 };
  55
  56 #endif /* HAVE_TC_ACT_VLAN */
  57
  58 #ifdef HAVE_TC_ACT_PEDIT
  59
  60 #include <linux/tc_act/tc_pedit.h>
  61
  62 #else /* HAVE_TC_ACT_VLAN */
  63
  64 enum {
  65         TCA_PEDIT_UNSPEC,
  66         TCA_PEDIT_TM,
  67         TCA_PEDIT_PARMS,
  68         TCA_PEDIT_PAD,
  69         TCA_PEDIT_PARMS_EX,
  70         TCA_PEDIT_KEYS_EX,
  71         TCA_PEDIT_KEY_EX,
  72         __TCA_PEDIT_MAX
  73 };
  74
  75 enum {
  76         TCA_PEDIT_KEY_EX_HTYPE = 1,
  77         TCA_PEDIT_KEY_EX_CMD = 2,
  78         __TCA_PEDIT_KEY_EX_MAX
  79 };
  80
  81 enum pedit_header_type {
  82         TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK = 0,
  83         TCA_PEDIT_KEY_EX_HDR_TYPE_ETH = 1,
  84         TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 = 2,
  85         TCA_PEDIT_KEY_EX_HDR_TYPE_IP6 = 3,
  86         TCA_PEDIT_KEY_EX_HDR_TYPE_TCP = 4,
  87         TCA_PEDIT_KEY_EX_HDR_TYPE_UDP = 5,
  88         __PEDIT_HDR_TYPE_MAX,
  89 };
  90
  91 enum pedit_cmd {
  92         TCA_PEDIT_KEY_EX_CMD_SET = 0,
  93         TCA_PEDIT_KEY_EX_CMD_ADD = 1,
  94         __PEDIT_CMD_MAX,
  95 };
  96
  97 struct tc_pedit_key {
  98         __u32 mask; /* AND */
  99         __u32 val; /*XOR */
 100         __u32 off; /*offset */
 101         __u32 at;
 102         __u32 offmask;
 103         __u32 shift;
 104 };
 105
 106 __extension__
 107 struct tc_pedit_sel {
 108         tc_gen;
 109         unsigned char nkeys;
 110         unsigned char flags;
 111         struct tc_pedit_key keys[0];
 112 };
 113
 114 #endif /* HAVE_TC_ACT_VLAN */
 115
 116 #ifdef HAVE_TC_ACT_TUNNEL_KEY
 117
 118 #include <linux/tc_act/tc_tunnel_key.h>
 119
 120 #ifndef HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT
 121 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 122 #endif
 123
 124 #ifndef HAVE_TCA_TUNNEL_KEY_NO_CSUM
 125 #define TCA_TUNNEL_KEY_NO_CSUM 10
 126 #endif
 127
 128 #else /* HAVE_TC_ACT_TUNNEL_KEY */
 129
 130 #define TCA_ACT_TUNNEL_KEY 17
 131 #define TCA_TUNNEL_KEY_ACT_SET 1
 132 #define TCA_TUNNEL_KEY_ACT_RELEASE 2
 133 #define TCA_TUNNEL_KEY_PARMS 2
 134 #define TCA_TUNNEL_KEY_ENC_IPV4_SRC 3
 135 #define TCA_TUNNEL_KEY_ENC_IPV4_DST 4
 136 #define TCA_TUNNEL_KEY_ENC_IPV6_SRC 5
 137 #define TCA_TUNNEL_KEY_ENC_IPV6_DST 6
 138 #define TCA_TUNNEL_KEY_ENC_KEY_ID 7
 139 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 140 #define TCA_TUNNEL_KEY_NO_CSUM 10
 141
 142 struct tc_tunnel_key {
 143         tc_gen;
 144         int t_action;
 145 };
 146
 147 #endif /* HAVE_TC_ACT_TUNNEL_KEY */
 148
 149 /* Normally found in linux/netlink.h. */
 150 #ifndef NETLINK_CAP_ACK
 151 #define NETLINK_CAP_ACK 10
 152 #endif
 153
 154 /* Normally found in linux/pkt_sched.h. */
 155 #ifndef TC_H_MIN_INGRESS
 156 #define TC_H_MIN_INGRESS 0xfff2u
 157 #endif
 158
 159 /* Normally found in linux/pkt_cls.h. */
 160 #ifndef TCA_CLS_FLAGS_SKIP_SW
 161 #define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
 162 #endif
 163 #ifndef TCA_CLS_FLAGS_IN_HW
 164 #define TCA_CLS_FLAGS_IN_HW (1 << 2)
 165 #endif
 166 #ifndef HAVE_TCA_CHAIN
 167 #define TCA_CHAIN 11
 168 #endif
 169 #ifndef HAVE_TCA_FLOWER_ACT
 170 #define TCA_FLOWER_ACT 3
 171 #endif
 172 #ifndef HAVE_TCA_FLOWER_FLAGS
 173 #define TCA_FLOWER_FLAGS 22
 174 #endif
 175 #ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
 176 #define TCA_FLOWER_KEY_ETH_TYPE 8
 177 #endif
 178 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
 179 #define TCA_FLOWER_KEY_ETH_DST 4
 180 #endif
 181 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
 182 #define TCA_FLOWER_KEY_ETH_DST_MASK 5
 183 #endif
 184 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
 185 #define TCA_FLOWER_KEY_ETH_SRC 6
 186 #endif
 187 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
 188 #define TCA_FLOWER_KEY_ETH_SRC_MASK 7
 189 #endif
 190 #ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
 191 #define TCA_FLOWER_KEY_IP_PROTO 9
 192 #endif
 193 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
 194 #define TCA_FLOWER_KEY_IPV4_SRC 10
 195 #endif
 196 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
 197 #define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
 198 #endif
 199 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
 200 #define TCA_FLOWER_KEY_IPV4_DST 12
 201 #endif
 202 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
 203 #define TCA_FLOWER_KEY_IPV4_DST_MASK 13
 204 #endif
 205 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
 206 #define TCA_FLOWER_KEY_IPV6_SRC 14
 207 #endif
 208 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
 209 #define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
 210 #endif
 211 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
 212 #define TCA_FLOWER_KEY_IPV6_DST 16
 213 #endif
 214 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
 215 #define TCA_FLOWER_KEY_IPV6_DST_MASK 17
 216 #endif
 217 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
 218 #define TCA_FLOWER_KEY_TCP_SRC 18
 219 #endif
 220 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
 221 #define TCA_FLOWER_KEY_TCP_SRC_MASK 35
 222 #endif
 223 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
 224 #define TCA_FLOWER_KEY_TCP_DST 19
 225 #endif
 226 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
 227 #define TCA_FLOWER_KEY_TCP_DST_MASK 36
 228 #endif
 229 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
 230 #define TCA_FLOWER_KEY_UDP_SRC 20
 231 #endif
 232 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
 233 #define TCA_FLOWER_KEY_UDP_SRC_MASK 37
 234 #endif
 235 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
 236 #define TCA_FLOWER_KEY_UDP_DST 21
 237 #endif
 238 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
 239 #define TCA_FLOWER_KEY_UDP_DST_MASK 38
 240 #endif
 241 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
 242 #define TCA_FLOWER_KEY_VLAN_ID 23
 243 #endif
 244 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
 245 #define TCA_FLOWER_KEY_VLAN_PRIO 24
 246 #endif
 247 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
 248 #define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
 249 #endif
 250 #ifndef HAVE_TCA_FLOWER_KEY_ENC_KEY_ID
 251 #define TCA_FLOWER_KEY_ENC_KEY_ID 26
 252 #endif
 253 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC
 254 #define TCA_FLOWER_KEY_ENC_IPV4_SRC 27
 255 #endif
 256 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK
 257 #define TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK 28
 258 #endif
 259 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST
 260 #define TCA_FLOWER_KEY_ENC_IPV4_DST 29
 261 #endif
 262 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK
 263 #define TCA_FLOWER_KEY_ENC_IPV4_DST_MASK 30
 264 #endif
 265 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC
 266 #define TCA_FLOWER_KEY_ENC_IPV6_SRC 31
 267 #endif
 268 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK
 269 #define TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK 32
 270 #endif
 271 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST
 272 #define TCA_FLOWER_KEY_ENC_IPV6_DST 33
 273 #endif
 274 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK
 275 #define TCA_FLOWER_KEY_ENC_IPV6_DST_MASK 34
 276 #endif
 277 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT
 278 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT 43
 279 #endif
 280 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK
 281 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK 44
 282 #endif
 283 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT
 284 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT 45
 285 #endif
 286 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK
 287 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK 46
 288 #endif
 289 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS
 290 #define TCA_FLOWER_KEY_TCP_FLAGS 71
 291 #endif
 292 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK
 293 #define TCA_FLOWER_KEY_TCP_FLAGS_MASK 72
 294 #endif
 295 #ifndef HAVE_TC_ACT_GOTO_CHAIN
 296 #define TC_ACT_GOTO_CHAIN 0x20000000
 297 #endif
 298
 299 #ifndef IPV6_ADDR_LEN
 300 #define IPV6_ADDR_LEN 16
 301 #endif
 302
 303 #ifndef IPV4_ADDR_LEN
 304 #define IPV4_ADDR_LEN 4
 305 #endif
 306
 307 #ifndef TP_PORT_LEN
 308 #define TP_PORT_LEN 2 /* Transport Port (UDP/TCP) Length */
 309 #endif
 310
 311 #ifndef TTL_LEN
 312 #define TTL_LEN 1
 313 #endif
 314
 315 #ifndef TCA_ACT_MAX_PRIO
 316 #define TCA_ACT_MAX_PRIO 32
 317 #endif
 318
 319 /** UDP port range of VXLAN devices created by driver. */
 320 #define MLX5_VXLAN_PORT_MIN 30000
 321 #define MLX5_VXLAN_PORT_MAX 60000
 322 #define MLX5_VXLAN_DEVICE_PFX "vmlx_"
 323
 324 /** Tunnel action type, used for @p type in header structure. */
 325 enum flow_tcf_tunact_type {
 326         FLOW_TCF_TUNACT_VXLAN_DECAP,
 327         FLOW_TCF_TUNACT_VXLAN_ENCAP,
 328 };
 329
 330 /** Flags used for @p mask in tunnel action encap descriptors. */
 331 #define FLOW_TCF_ENCAP_ETH_SRC (1u << 0)
 332 #define FLOW_TCF_ENCAP_ETH_DST (1u << 1)
 333 #define FLOW_TCF_ENCAP_IPV4_SRC (1u << 2)
 334 #define FLOW_TCF_ENCAP_IPV4_DST (1u << 3)
 335 #define FLOW_TCF_ENCAP_IPV6_SRC (1u << 4)
 336 #define FLOW_TCF_ENCAP_IPV6_DST (1u << 5)
 337 #define FLOW_TCF_ENCAP_UDP_SRC (1u << 6)
 338 #define FLOW_TCF_ENCAP_UDP_DST (1u << 7)
 339 #define FLOW_TCF_ENCAP_VXLAN_VNI (1u << 8)
 340
 341 /**
 342  * Structure for holding netlink context.
 343  * Note the size of the message buffer which is MNL_SOCKET_BUFFER_SIZE.
 344  * Using this (8KB) buffer size ensures that netlink messages will never be
 345  * truncated.
 346  */
 347 struct mlx5_flow_tcf_context {
 348         struct mnl_socket *nl; /* NETLINK_ROUTE libmnl socket. */
 349         uint32_t seq; /* Message sequence number. */
 350         uint32_t buf_size; /* Message buffer size. */
 351         uint8_t *buf; /* Message buffer. */
 352 };
 353
 354 /**
 355  * Neigh rule structure. The neigh rule is applied via Netlink to
 356  * outer tunnel iface in order to provide destination MAC address
 357  * for the VXLAN encapsultion. The neigh rule is implicitly related
 358  * to the Flow itself and can be shared by multiple Flows.
 359  */
 360 struct tcf_neigh_rule {
 361         LIST_ENTRY(tcf_neigh_rule) next;
 362         uint32_t refcnt;
 363         struct ether_addr eth;
 364         uint16_t mask;
 365         union {
 366                 struct {
 367                         rte_be32_t dst;
 368                 } ipv4;
 369                 struct {
 370                         uint8_t dst[IPV6_ADDR_LEN];
 371                 } ipv6;
 372         };
 373 };
 374
 375 /**
 376  * Local rule structure. The local rule is applied via Netlink to
 377  * outer tunnel iface in order to provide local and peer IP addresses
 378  * of the VXLAN tunnel for encapsulation. The local rule is implicitly
 379  * related to the Flow itself and can be shared by multiple Flows.
 380  */
 381 struct tcf_local_rule {
 382         LIST_ENTRY(tcf_local_rule) next;
 383         uint32_t refcnt;
 384         uint16_t mask;
 385         union {
 386                 struct {
 387                         rte_be32_t dst;
 388                         rte_be32_t src;
 389                 } ipv4;
 390                 struct {
 391                         uint8_t dst[IPV6_ADDR_LEN];
 392                         uint8_t src[IPV6_ADDR_LEN];
 393                 } ipv6;
 394         };
 395 };
 396
 397 /** VXLAN virtual netdev. */
 398 struct tcf_vtep {
 399         LIST_ENTRY(tcf_vtep) next;
 400         LIST_HEAD(, tcf_neigh_rule) neigh;
 401         LIST_HEAD(, tcf_local_rule) local;
 402         uint32_t refcnt;
 403         unsigned int ifindex; /**< Own interface index. */
 404         unsigned int ifouter; /**< Index of device attached to. */
 405         uint16_t port;
 406         uint8_t created;
 407 };
 408
 409 /** Tunnel descriptor header, common for all tunnel types. */
 410 struct flow_tcf_tunnel_hdr {
 411         uint32_t type; /**< Tunnel action type. */
 412         struct tcf_vtep *vtep; /**< Virtual tunnel endpoint device. */
 413         unsigned int ifindex_org; /**< Original dst/src interface */
 414         unsigned int *ifindex_ptr; /**< Interface ptr in message. */
 415 };
 416
 417 struct flow_tcf_vxlan_decap {
 418         struct flow_tcf_tunnel_hdr hdr;
 419         uint16_t udp_port;
 420 };
 421
 422 struct flow_tcf_vxlan_encap {
 423         struct flow_tcf_tunnel_hdr hdr;
 424         uint32_t mask;
 425         struct {
 426                 struct ether_addr dst;
 427                 struct ether_addr src;
 428         } eth;
 429         union {
 430                 struct {
 431                         rte_be32_t dst;
 432                         rte_be32_t src;
 433                 } ipv4;
 434                 struct {
 435                         uint8_t dst[IPV6_ADDR_LEN];
 436                         uint8_t src[IPV6_ADDR_LEN];
 437                 } ipv6;
 438         };
 439 struct {
 440                 rte_be16_t src;
 441                 rte_be16_t dst;
 442         } udp;
 443         struct {
 444                 uint8_t vni[3];
 445         } vxlan;
 446 };
 447
 448 /** Structure used when extracting the values of a flow counters
 449  * from a netlink message.
 450  */
 451 struct flow_tcf_stats_basic {
 452         bool valid;
 453         struct gnet_stats_basic counters;
 454 };
 455
 456 /** Empty masks for known item types. */
 457 static const union {
 458         struct rte_flow_item_port_id port_id;
 459         struct rte_flow_item_eth eth;
 460         struct rte_flow_item_vlan vlan;
 461         struct rte_flow_item_ipv4 ipv4;
 462         struct rte_flow_item_ipv6 ipv6;
 463         struct rte_flow_item_tcp tcp;
 464         struct rte_flow_item_udp udp;
 465         struct rte_flow_item_vxlan vxlan;
 466 } flow_tcf_mask_empty;
 467
 468 /** Supported masks for known item types. */
 469 static const struct {
 470         struct rte_flow_item_port_id port_id;
 471         struct rte_flow_item_eth eth;
 472         struct rte_flow_item_vlan vlan;
 473         struct rte_flow_item_ipv4 ipv4;
 474         struct rte_flow_item_ipv6 ipv6;
 475         struct rte_flow_item_tcp tcp;
 476         struct rte_flow_item_udp udp;
 477         struct rte_flow_item_vxlan vxlan;
 478 } flow_tcf_mask_supported = {
 479         .port_id = {
 480                 .id = 0xffffffff,
 481         },
 482         .eth = {
 483                 .type = RTE_BE16(0xffff),
 484                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 485                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 486         },
 487         .vlan = {
 488                 /* PCP and VID only, no DEI. */
 489                 .tci = RTE_BE16(0xefff),
 490                 .inner_type = RTE_BE16(0xffff),
 491         },
 492         .ipv4.hdr = {
 493                 .next_proto_id = 0xff,
 494                 .src_addr = RTE_BE32(0xffffffff),
 495                 .dst_addr = RTE_BE32(0xffffffff),
 496         },
 497         .ipv6.hdr = {
 498                 .proto = 0xff,
 499                 .src_addr =
 500                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 501                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 502                 .dst_addr =
 503                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 504                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 505         },
 506         .tcp.hdr = {
 507                 .src_port = RTE_BE16(0xffff),
 508                 .dst_port = RTE_BE16(0xffff),
 509                 .tcp_flags = 0xff,
 510         },
 511         .udp.hdr = {
 512                 .src_port = RTE_BE16(0xffff),
 513                 .dst_port = RTE_BE16(0xffff),
 514         },
 515         .vxlan = {
 516                .vni = "\xff\xff\xff",
 517         },
 518 };
 519
 520 #define SZ_NLATTR_HDR MNL_ALIGN(sizeof(struct nlattr))
 521 #define SZ_NLATTR_NEST SZ_NLATTR_HDR
 522 #define SZ_NLATTR_DATA_OF(len) MNL_ALIGN(SZ_NLATTR_HDR + (len))
 523 #define SZ_NLATTR_TYPE_OF(typ) SZ_NLATTR_DATA_OF(sizeof(typ))
 524 #define SZ_NLATTR_STRZ_OF(str) SZ_NLATTR_DATA_OF(strlen(str) + 1)
 525
 526 #define PTOI_TABLE_SZ_MAX(dev) (mlx5_dev_to_port_id((dev)->device, NULL, 0) + 2)
 527
 528 /** DPDK port to network interface index (ifindex) conversion. */
 529 struct flow_tcf_ptoi {
 530         uint16_t port_id; /**< DPDK port ID. */
 531         unsigned int ifindex; /**< Network interface index. */
 532 };
 533
 534 /* Due to a limitation on driver/FW. */
 535 #define MLX5_TCF_GROUP_ID_MAX 3
 536
 537 /*
 538  * Due to a limitation on driver/FW, priority ranges from 1 to 16 in kernel.
 539  * Priority in rte_flow attribute starts from 0 and is added by 1 in
 540  * translation. This is subject to be changed to determine the max priority
 541  * based on trial-and-error like Verbs driver once the restriction is lifted or
 542  * the range is extended.
 543  */
 544 #define MLX5_TCF_GROUP_PRIORITY_MAX 15
 545
 546 #define MLX5_TCF_FATE_ACTIONS \
 547         (MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
 548          MLX5_FLOW_ACTION_JUMP)
 549
 550 #define MLX5_TCF_VLAN_ACTIONS \
 551         (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN | \
 552          MLX5_FLOW_ACTION_OF_SET_VLAN_VID | MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
 553
 554 #define MLX5_TCF_VXLAN_ACTIONS \
 555         (MLX5_FLOW_ACTION_VXLAN_ENCAP | MLX5_FLOW_ACTION_VXLAN_DECAP)
 556
 557 #define MLX5_TCF_PEDIT_ACTIONS \
 558         (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST | \
 559          MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST | \
 560          MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST | \
 561          MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL | \
 562          MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)
 563
 564 #define MLX5_TCF_CONFIG_ACTIONS \
 565         (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_JUMP | \
 566          MLX5_FLOW_ACTION_OF_PUSH_VLAN | MLX5_FLOW_ACTION_OF_SET_VLAN_VID | \
 567          MLX5_FLOW_ACTION_OF_SET_VLAN_PCP | \
 568          (MLX5_TCF_PEDIT_ACTIONS & ~MLX5_FLOW_ACTION_DEC_TTL))
 569
 570 #define MAX_PEDIT_KEYS 128
 571 #define SZ_PEDIT_KEY_VAL 4
 572
 573 #define NUM_OF_PEDIT_KEYS(sz) \
 574         (((sz) / SZ_PEDIT_KEY_VAL) + (((sz) % SZ_PEDIT_KEY_VAL) ? 1 : 0))
 575
 576 struct pedit_key_ex {
 577         enum pedit_header_type htype;
 578         enum pedit_cmd cmd;
 579 };
 580
 581 struct pedit_parser {
 582         struct tc_pedit_sel sel;
 583         struct tc_pedit_key keys[MAX_PEDIT_KEYS];
 584         struct pedit_key_ex keys_ex[MAX_PEDIT_KEYS];
 585 };
 586
 587 /**
 588  * Create space for using the implicitly created TC flow counter.
 589  *
 590  * @param[in] dev
 591  *   Pointer to the Ethernet device structure.
 592  *
 593  * @return
 594  *   A pointer to the counter data structure, NULL otherwise and
 595  *   rte_errno is set.
 596  */
 597 static struct mlx5_flow_counter *
 598 flow_tcf_counter_new(void)
 599 {
 600         struct mlx5_flow_counter *cnt;
 601
 602         /*
 603          * eswitch counter cannot be shared and its id is unknown.
 604          * currently returning all with id 0.
 605          * in the future maybe better to switch to unique numbers.
 606          */
 607         struct mlx5_flow_counter tmpl = {
 608                 .ref_cnt = 1,
 609         };
 610         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 611         if (!cnt) {
 612                 rte_errno = ENOMEM;
 613                 return NULL;
 614         }
 615         *cnt = tmpl;
 616         /* Implicit counter, do not add to list. */
 617         return cnt;
 618 }
 619
 620 /**
 621  * Set pedit key of MAC address
 622  *
 623  * @param[in] actions
 624  *   pointer to action specification
 625  * @param[in,out] p_parser
 626  *   pointer to pedit_parser
 627  */
 628 static void
 629 flow_tcf_pedit_key_set_mac(const struct rte_flow_action *actions,
 630                            struct pedit_parser *p_parser)
 631 {
 632         int idx = p_parser->sel.nkeys;
 633         uint32_t off = actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
 634                                         offsetof(struct ether_hdr, s_addr) :
 635                                         offsetof(struct ether_hdr, d_addr);
 636         const struct rte_flow_action_set_mac *conf =
 637                 (const struct rte_flow_action_set_mac *)actions->conf;
 638
 639         p_parser->keys[idx].off = off;
 640         p_parser->keys[idx].mask = ~UINT32_MAX;
 641         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 642         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 643         memcpy(&p_parser->keys[idx].val,
 644                 conf->mac_addr, SZ_PEDIT_KEY_VAL);
 645         idx++;
 646         p_parser->keys[idx].off = off + SZ_PEDIT_KEY_VAL;
 647         p_parser->keys[idx].mask = 0xFFFF0000;
 648         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 649         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 650         memcpy(&p_parser->keys[idx].val,
 651                 conf->mac_addr + SZ_PEDIT_KEY_VAL,
 652                 ETHER_ADDR_LEN - SZ_PEDIT_KEY_VAL);
 653         p_parser->sel.nkeys = (++idx);
 654 }
 655
 656 /**
 657  * Set pedit key of decrease/set ttl
 658  *
 659  * @param[in] actions
 660  *   pointer to action specification
 661  * @param[in,out] p_parser
 662  *   pointer to pedit_parser
 663  * @param[in] item_flags
 664  *   flags of all items presented
 665  */
 666 static void
 667 flow_tcf_pedit_key_set_dec_ttl(const struct rte_flow_action *actions,
 668                                 struct pedit_parser *p_parser,
 669                                 uint64_t item_flags)
 670 {
 671         int idx = p_parser->sel.nkeys;
 672
 673         p_parser->keys[idx].mask = 0xFFFFFF00;
 674         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) {
 675                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 676                 p_parser->keys[idx].off =
 677                         offsetof(struct ipv4_hdr, time_to_live);
 678         }
 679         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6) {
 680                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 681                 p_parser->keys[idx].off =
 682                         offsetof(struct ipv6_hdr, hop_limits);
 683         }
 684         if (actions->type == RTE_FLOW_ACTION_TYPE_DEC_TTL) {
 685                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_ADD;
 686                 p_parser->keys[idx].val = 0x000000FF;
 687         } else {
 688                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 689                 p_parser->keys[idx].val =
 690                         (__u32)((const struct rte_flow_action_set_ttl *)
 691                          actions->conf)->ttl_value;
 692         }
 693         p_parser->sel.nkeys = (++idx);
 694 }
 695
 696 /**
 697  * Set pedit key of transport (TCP/UDP) port value
 698  *
 699  * @param[in] actions
 700  *   pointer to action specification
 701  * @param[in,out] p_parser
 702  *   pointer to pedit_parser
 703  * @param[in] item_flags
 704  *   flags of all items presented
 705  */
 706 static void
 707 flow_tcf_pedit_key_set_tp_port(const struct rte_flow_action *actions,
 708                                 struct pedit_parser *p_parser,
 709                                 uint64_t item_flags)
 710 {
 711         int idx = p_parser->sel.nkeys;
 712
 713         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 714                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_UDP;
 715         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 716                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_TCP;
 717         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 718         /* offset of src/dst port is same for TCP and UDP */
 719         p_parser->keys[idx].off =
 720                 actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
 721                 offsetof(struct tcp_hdr, src_port) :
 722                 offsetof(struct tcp_hdr, dst_port);
 723         p_parser->keys[idx].mask = 0xFFFF0000;
 724         p_parser->keys[idx].val =
 725                 (__u32)((const struct rte_flow_action_set_tp *)
 726                                 actions->conf)->port;
 727         p_parser->sel.nkeys = (++idx);
 728 }
 729
 730 /**
 731  * Set pedit key of ipv6 address
 732  *
 733  * @param[in] actions
 734  *   pointer to action specification
 735  * @param[in,out] p_parser
 736  *   pointer to pedit_parser
 737  */
 738 static void
 739 flow_tcf_pedit_key_set_ipv6_addr(const struct rte_flow_action *actions,
 740                                  struct pedit_parser *p_parser)
 741 {
 742         int idx = p_parser->sel.nkeys;
 743         int keys = NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 744         int off_base =
 745                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
 746                 offsetof(struct ipv6_hdr, src_addr) :
 747                 offsetof(struct ipv6_hdr, dst_addr);
 748         const struct rte_flow_action_set_ipv6 *conf =
 749                 (const struct rte_flow_action_set_ipv6 *)actions->conf;
 750
 751         for (int i = 0; i < keys; i++, idx++) {
 752                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 753                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 754                 p_parser->keys[idx].off = off_base + i * SZ_PEDIT_KEY_VAL;
 755                 p_parser->keys[idx].mask = ~UINT32_MAX;
 756                 memcpy(&p_parser->keys[idx].val,
 757                         conf->ipv6_addr + i *  SZ_PEDIT_KEY_VAL,
 758                         SZ_PEDIT_KEY_VAL);
 759         }
 760         p_parser->sel.nkeys += keys;
 761 }
 762
 763 /**
 764  * Set pedit key of ipv4 address
 765  *
 766  * @param[in] actions
 767  *   pointer to action specification
 768  * @param[in,out] p_parser
 769  *   pointer to pedit_parser
 770  */
 771 static void
 772 flow_tcf_pedit_key_set_ipv4_addr(const struct rte_flow_action *actions,
 773                                  struct pedit_parser *p_parser)
 774 {
 775         int idx = p_parser->sel.nkeys;
 776
 777         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 778         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 779         p_parser->keys[idx].off =
 780                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
 781                 offsetof(struct ipv4_hdr, src_addr) :
 782                 offsetof(struct ipv4_hdr, dst_addr);
 783         p_parser->keys[idx].mask = ~UINT32_MAX;
 784         p_parser->keys[idx].val =
 785                 ((const struct rte_flow_action_set_ipv4 *)
 786                  actions->conf)->ipv4_addr;
 787         p_parser->sel.nkeys = (++idx);
 788 }
 789
 790 /**
 791  * Create the pedit's na attribute in netlink message
 792  * on pre-allocate message buffer
 793  *
 794  * @param[in,out] nl
 795  *   pointer to pre-allocated netlink message buffer
 796  * @param[in,out] actions
 797  *   pointer to pointer of actions specification.
 798  * @param[in,out] action_flags
 799  *   pointer to actions flags
 800  * @param[in] item_flags
 801  *   flags of all item presented
 802  */
 803 static void
 804 flow_tcf_create_pedit_mnl_msg(struct nlmsghdr *nl,
 805                               const struct rte_flow_action **actions,
 806                               uint64_t item_flags)
 807 {
 808         struct pedit_parser p_parser;
 809         struct nlattr *na_act_options;
 810         struct nlattr *na_pedit_keys;
 811
 812         memset(&p_parser, 0, sizeof(p_parser));
 813         mnl_attr_put_strz(nl, TCA_ACT_KIND, "pedit");
 814         na_act_options = mnl_attr_nest_start(nl, TCA_ACT_OPTIONS);
 815         /* all modify header actions should be in one tc-pedit action */
 816         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 817                 switch ((*actions)->type) {
 818                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 819                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 820                         flow_tcf_pedit_key_set_ipv4_addr(*actions, &p_parser);
 821                         break;
 822                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 823                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 824                         flow_tcf_pedit_key_set_ipv6_addr(*actions, &p_parser);
 825                         break;
 826                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 827                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 828                         flow_tcf_pedit_key_set_tp_port(*actions,
 829                                                         &p_parser, item_flags);
 830                         break;
 831                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 832                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 833                         flow_tcf_pedit_key_set_dec_ttl(*actions,
 834                                                         &p_parser, item_flags);
 835                         break;
 836                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 837                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 838                         flow_tcf_pedit_key_set_mac(*actions, &p_parser);
 839                         break;
 840                 default:
 841                         goto pedit_mnl_msg_done;
 842                 }
 843         }
 844 pedit_mnl_msg_done:
 845         p_parser.sel.action = TC_ACT_PIPE;
 846         mnl_attr_put(nl, TCA_PEDIT_PARMS_EX,
 847                      sizeof(p_parser.sel) +
 848                      p_parser.sel.nkeys * sizeof(struct tc_pedit_key),
 849                      &p_parser);
 850         na_pedit_keys =
 851                 mnl_attr_nest_start(nl, TCA_PEDIT_KEYS_EX | NLA_F_NESTED);
 852         for (int i = 0; i < p_parser.sel.nkeys; i++) {
 853                 struct nlattr *na_pedit_key =
 854                         mnl_attr_nest_start(nl,
 855                                             TCA_PEDIT_KEY_EX | NLA_F_NESTED);
 856                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_HTYPE,
 857                                  p_parser.keys_ex[i].htype);
 858                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_CMD,
 859                                  p_parser.keys_ex[i].cmd);
 860                 mnl_attr_nest_end(nl, na_pedit_key);
 861         }
 862         mnl_attr_nest_end(nl, na_pedit_keys);
 863         mnl_attr_nest_end(nl, na_act_options);
 864         (*actions)--;
 865 }
 866
 867 /**
 868  * Calculate max memory size of one TC-pedit actions.
 869  * One TC-pedit action can contain set of keys each defining
 870  * a rewrite element (rte_flow action)
 871  *
 872  * @param[in,out] actions
 873  *   actions specification.
 874  * @param[in,out] action_flags
 875  *   actions flags
 876  * @param[in,out] size
 877  *   accumulated size
 878  * @return
 879  *   Max memory size of one TC-pedit action
 880  */
 881 static int
 882 flow_tcf_get_pedit_actions_size(const struct rte_flow_action **actions,
 883                                 uint64_t *action_flags)
 884 {
 885         int pedit_size = 0;
 886         int keys = 0;
 887         uint64_t flags = 0;
 888
 889         pedit_size += SZ_NLATTR_NEST + /* na_act_index. */
 890                       SZ_NLATTR_STRZ_OF("pedit") +
 891                       SZ_NLATTR_NEST; /* TCA_ACT_OPTIONS. */
 892         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 893                 switch ((*actions)->type) {
 894                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 895                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 896                         flags |= MLX5_FLOW_ACTION_SET_IPV4_SRC;
 897                         break;
 898                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 899                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 900                         flags |= MLX5_FLOW_ACTION_SET_IPV4_DST;
 901                         break;
 902                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 903                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 904                         flags |= MLX5_FLOW_ACTION_SET_IPV6_SRC;
 905                         break;
 906                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 907                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 908                         flags |= MLX5_FLOW_ACTION_SET_IPV6_DST;
 909                         break;
 910                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 911                         /* TCP is as same as UDP */
 912                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 913                         flags |= MLX5_FLOW_ACTION_SET_TP_SRC;
 914                         break;
 915                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 916                         /* TCP is as same as UDP */
 917                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 918                         flags |= MLX5_FLOW_ACTION_SET_TP_DST;
 919                         break;
 920                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 921                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 922                         flags |= MLX5_FLOW_ACTION_SET_TTL;
 923                         break;
 924                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 925                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 926                         flags |= MLX5_FLOW_ACTION_DEC_TTL;
 927                         break;
 928                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 929                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 930                         flags |= MLX5_FLOW_ACTION_SET_MAC_SRC;
 931                         break;
 932                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 933                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 934                         flags |= MLX5_FLOW_ACTION_SET_MAC_DST;
 935                         break;
 936                 default:
 937                         goto get_pedit_action_size_done;
 938                 }
 939         }
 940 get_pedit_action_size_done:
 941         /* TCA_PEDIT_PARAMS_EX */
 942         pedit_size +=
 943                 SZ_NLATTR_DATA_OF(sizeof(struct tc_pedit_sel) +
 944                                   keys * sizeof(struct tc_pedit_key));
 945         pedit_size += SZ_NLATTR_NEST; /* TCA_PEDIT_KEYS */
 946         pedit_size += keys *
 947                       /* TCA_PEDIT_KEY_EX + HTYPE + CMD */
 948                       (SZ_NLATTR_NEST + SZ_NLATTR_DATA_OF(2) +
 949                        SZ_NLATTR_DATA_OF(2));
 950         (*action_flags) |= flags;
 951         (*actions)--;
 952         return pedit_size;
 953 }
 954
 955 /**
 956  * Retrieve mask for pattern item.
 957  *
 958  * This function does basic sanity checks on a pattern item in order to
 959  * return the most appropriate mask for it.
 960  *
 961  * @param[in] item
 962  *   Item specification.
 963  * @param[in] mask_default
 964  *   Default mask for pattern item as specified by the flow API.
 965  * @param[in] mask_supported
 966  *   Mask fields supported by the implementation.
 967  * @param[in] mask_empty
 968  *   Empty mask to return when there is no specification.
 969  * @param[out] error
 970  *   Perform verbose error reporting if not NULL.
 971  *
 972  * @return
 973  *   Either @p item->mask or one of the mask parameters on success, NULL
 974  *   otherwise and rte_errno is set.
 975  */
 976 static const void *
 977 flow_tcf_item_mask(const struct rte_flow_item *item, const void *mask_default,
 978                    const void *mask_supported, const void *mask_empty,
 979                    size_t mask_size, struct rte_flow_error *error)
 980 {
 981         const uint8_t *mask;
 982         size_t i;
 983
 984         /* item->last and item->mask cannot exist without item->spec. */
 985         if (!item->spec && (item->mask || item->last)) {
 986                 rte_flow_error_set(error, EINVAL,
 987                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 988                                    "\"mask\" or \"last\" field provided without"
 989                                    " a corresponding \"spec\"");
 990                 return NULL;
 991         }
 992         /* No spec, no mask, no problem. */
 993         if (!item->spec)
 994                 return mask_empty;
 995         mask = item->mask ? item->mask : mask_default;
 996         assert(mask);
 997         /*
 998          * Single-pass check to make sure that:
 999          * - Mask is supported, no bits are set outside mask_supported.
1000          * - Both item->spec and item->last are included in mask.
1001          */
1002         for (i = 0; i != mask_size; ++i) {
1003                 if (!mask[i])
1004                         continue;
1005                 if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
1006                     ((const uint8_t *)mask_supported)[i]) {
1007                         rte_flow_error_set(error, ENOTSUP,
1008                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1009                                            "unsupported field found"
1010                                            " in \"mask\"");
1011                         return NULL;
1012                 }
1013                 if (item->last &&
1014                     (((const uint8_t *)item->spec)[i] & mask[i]) !=
1015                     (((const uint8_t *)item->last)[i] & mask[i])) {
1016                         rte_flow_error_set(error, EINVAL,
1017                                            RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1018                                            item->last,
1019                                            "range between \"spec\" and \"last\""
1020                                            " not comprised in \"mask\"");
1021                         return NULL;
1022                 }
1023         }
1024         return mask;
1025 }
1026
1027 /**
1028  * Build a conversion table between port ID and ifindex.
1029  *
1030  * @param[in] dev
1031  *   Pointer to Ethernet device.
1032  * @param[out] ptoi
1033  *   Pointer to ptoi table.
1034  * @param[in] len
1035  *   Size of ptoi table provided.
1036  *
1037  * @return
1038  *   Size of ptoi table filled.
1039  */
1040 static unsigned int
1041 flow_tcf_build_ptoi_table(struct rte_eth_dev *dev, struct flow_tcf_ptoi *ptoi,
1042                           unsigned int len)
1043 {
1044         unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
1045         uint16_t port_id[n + 1];
1046         unsigned int i;
1047         unsigned int own = 0;
1048
1049         /* At least one port is needed when no switch domain is present. */
1050         if (!n) {
1051                 n = 1;
1052                 port_id[0] = dev->data->port_id;
1053         } else {
1054                 n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
1055         }
1056         if (n > len)
1057                 return 0;
1058         for (i = 0; i != n; ++i) {
1059                 struct rte_eth_dev_info dev_info;
1060
1061                 rte_eth_dev_info_get(port_id[i], &dev_info);
1062                 if (port_id[i] == dev->data->port_id)
1063                         own = i;
1064                 ptoi[i].port_id = port_id[i];
1065                 ptoi[i].ifindex = dev_info.if_index;
1066         }
1067         /* Ensure first entry of ptoi[] is the current device. */
1068         if (own) {
1069                 ptoi[n] = ptoi[0];
1070                 ptoi[0] = ptoi[own];
1071                 ptoi[own] = ptoi[n];
1072         }
1073         /* An entry with zero ifindex terminates ptoi[]. */
1074         ptoi[n].port_id = 0;
1075         ptoi[n].ifindex = 0;
1076         return n;
1077 }
1078
1079 /**
1080  * Verify the @p attr will be correctly understood by the E-switch.
1081  *
1082  * @param[in] attr
1083  *   Pointer to flow attributes
1084  * @param[out] error
1085  *   Pointer to error structure.
1086  *
1087  * @return
1088  *   0 on success, a negative errno value otherwise and rte_errno is set.
1089  */
1090 static int
1091 flow_tcf_validate_attributes(const struct rte_flow_attr *attr,
1092                              struct rte_flow_error *error)
1093 {
1094         /*
1095          * Supported attributes: groups, some priorities and ingress only.
1096          * group is supported only if kernel supports chain. Don't care about
1097          * transfer as it is the caller's problem.
1098          */
1099         if (attr->group > MLX5_TCF_GROUP_ID_MAX)
1100                 return rte_flow_error_set(error, ENOTSUP,
1101                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1102                                           "group ID larger than "
1103                                           RTE_STR(MLX5_TCF_GROUP_ID_MAX)
1104                                           " isn't supported");
1105         else if (attr->priority > MLX5_TCF_GROUP_PRIORITY_MAX)
1106                 return rte_flow_error_set(error, ENOTSUP,
1107                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1108                                           attr,
1109                                           "priority more than "
1110                                           RTE_STR(MLX5_TCF_GROUP_PRIORITY_MAX)
1111                                           " is not supported");
1112         if (!attr->ingress)
1113                 return rte_flow_error_set(error, EINVAL,
1114                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1115                                           attr, "only ingress is supported");
1116         if (attr->egress)
1117                 return rte_flow_error_set(error, ENOTSUP,
1118                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1119                                           attr, "egress is not supported");
1120         return 0;
1121 }
1122
1123 /**
1124  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_ETH item for E-Switch.
1125  * The routine checks the L2 fields to be used in encapsulation header.
1126  *
1127  * @param[in] item
1128  *   Pointer to the item structure.
1129  * @param[out] error
1130  *   Pointer to the error structure.
1131  *
1132  * @return
1133  *   0 on success, a negative errno value otherwise and rte_errno is set.
1134  **/
1135 static int
1136 flow_tcf_validate_vxlan_encap_eth(const struct rte_flow_item *item,
1137                                   struct rte_flow_error *error)
1138 {
1139         const struct rte_flow_item_eth *spec = item->spec;
1140         const struct rte_flow_item_eth *mask = item->mask;
1141
1142         if (!spec) {
1143                 /*
1144                  * Specification for L2 addresses can be empty
1145                  * because these ones are optional and not
1146                  * required directly by tc rule. Kernel tries
1147                  * to resolve these ones on its own
1148                  */
1149                 return 0;
1150         }
1151         if (!mask) {
1152                 /* If mask is not specified use the default one. */
1153                 mask = &rte_flow_item_eth_mask;
1154         }
1155         if (memcmp(&mask->dst,
1156                    &flow_tcf_mask_empty.eth.dst,
1157                    sizeof(flow_tcf_mask_empty.eth.dst))) {
1158                 if (memcmp(&mask->dst,
1159                            &rte_flow_item_eth_mask.dst,
1160                            sizeof(rte_flow_item_eth_mask.dst)))
1161                         return rte_flow_error_set
1162                                 (error, ENOTSUP,
1163                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1164                                  "no support for partial mask on"
1165                                  " \"eth.dst\" field");
1166         }
1167         if (memcmp(&mask->src,
1168                    &flow_tcf_mask_empty.eth.src,
1169                    sizeof(flow_tcf_mask_empty.eth.src))) {
1170                 if (memcmp(&mask->src,
1171                            &rte_flow_item_eth_mask.src,
1172                            sizeof(rte_flow_item_eth_mask.src)))
1173                         return rte_flow_error_set
1174                                 (error, ENOTSUP,
1175                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1176                                  "no support for partial mask on"
1177                                  " \"eth.src\" field");
1178         }
1179         if (mask->type != RTE_BE16(0x0000)) {
1180                 if (mask->type != RTE_BE16(0xffff))
1181                         return rte_flow_error_set
1182                                 (error, ENOTSUP,
1183                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1184                                  "no support for partial mask on"
1185                                  " \"eth.type\" field");
1186                 DRV_LOG(WARNING,
1187                         "outer ethernet type field"
1188                         " cannot be forced for vxlan"
1189                         " encapsulation, parameter ignored");
1190         }
1191         return 0;
1192 }
1193
1194 /**
1195  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV4 item for E-Switch.
1196  * The routine checks the IPv4 fields to be used in encapsulation header.
1197  *
1198  * @param[in] item
1199  *   Pointer to the item structure.
1200  * @param[out] error
1201  *   Pointer to the error structure.
1202  *
1203  * @return
1204  *   0 on success, a negative errno value otherwise and rte_errno is set.
1205  **/
1206 static int
1207 flow_tcf_validate_vxlan_encap_ipv4(const struct rte_flow_item *item,
1208                                    struct rte_flow_error *error)
1209 {
1210         const struct rte_flow_item_ipv4 *spec = item->spec;
1211         const struct rte_flow_item_ipv4 *mask = item->mask;
1212
1213         if (!spec) {
1214                 /*
1215                  * Specification for IP addresses cannot be empty
1216                  * because it is required by tunnel_key parameter.
1217                  */
1218                 return rte_flow_error_set(error, EINVAL,
1219                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1220                                           "NULL outer ipv4 address"
1221                                           " specification for vxlan"
1222                                           " encapsulation");
1223         }
1224         if (!mask)
1225                 mask = &rte_flow_item_ipv4_mask;
1226         if (mask->hdr.dst_addr != RTE_BE32(0x00000000)) {
1227                 if (mask->hdr.dst_addr != RTE_BE32(0xffffffff))
1228                         return rte_flow_error_set
1229                                 (error, ENOTSUP,
1230                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1231                                  "no support for partial mask on"
1232                                  " \"ipv4.hdr.dst_addr\" field"
1233                                  " for vxlan encapsulation");
1234                 /* More IPv4 address validations can be put here. */
1235         } else {
1236                 /*
1237                  * Kernel uses the destination IP address to determine
1238                  * the routing path and obtain the MAC destination
1239                  * address, so IP destination address must be
1240                  * specified in the tc rule.
1241                  */
1242                 return rte_flow_error_set(error, EINVAL,
1243                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1244                                           "outer ipv4 destination address"
1245                                           " must be specified for"
1246                                           " vxlan encapsulation");
1247         }
1248         if (mask->hdr.src_addr != RTE_BE32(0x00000000)) {
1249                 if (mask->hdr.src_addr != RTE_BE32(0xffffffff))
1250                         return rte_flow_error_set
1251                                 (error, ENOTSUP,
1252                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1253                                  "no support for partial mask on"
1254                                  " \"ipv4.hdr.src_addr\" field"
1255                                  " for vxlan encapsulation");
1256                 /* More IPv4 address validations can be put here. */
1257         } else {
1258                 /*
1259                  * Kernel uses the source IP address to select the
1260                  * interface for egress encapsulated traffic, so
1261                  * it must be specified in the tc rule.
1262                  */
1263                 return rte_flow_error_set(error, EINVAL,
1264                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1265                                           "outer ipv4 source address"
1266                                           " must be specified for"
1267                                           " vxlan encapsulation");
1268         }
1269         return 0;
1270 }
1271
1272 /**
1273  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV6 item for E-Switch.
1274  * The routine checks the IPv6 fields to be used in encapsulation header.
1275  *
1276  * @param[in] item
1277  *   Pointer to the item structure.
1278  * @param[out] error
1279  *   Pointer to the error structure.
1280  *
1281  * @return
1282  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1283  **/
1284 static int
1285 flow_tcf_validate_vxlan_encap_ipv6(const struct rte_flow_item *item,
1286                                    struct rte_flow_error *error)
1287 {
1288         const struct rte_flow_item_ipv6 *spec = item->spec;
1289         const struct rte_flow_item_ipv6 *mask = item->mask;
1290
1291         if (!spec) {
1292                 /*
1293                  * Specification for IP addresses cannot be empty
1294                  * because it is required by tunnel_key parameter.
1295                  */
1296                 return rte_flow_error_set(error, EINVAL,
1297                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1298                                           "NULL outer ipv6 address"
1299                                           " specification for"
1300                                           " vxlan encapsulation");
1301         }
1302         if (!mask)
1303                 mask = &rte_flow_item_ipv6_mask;
1304         if (memcmp(&mask->hdr.dst_addr,
1305                    &flow_tcf_mask_empty.ipv6.hdr.dst_addr,
1306                    IPV6_ADDR_LEN)) {
1307                 if (memcmp(&mask->hdr.dst_addr,
1308                            &rte_flow_item_ipv6_mask.hdr.dst_addr,
1309                            IPV6_ADDR_LEN))
1310                         return rte_flow_error_set
1311                                         (error, ENOTSUP,
1312                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1313                                          "no support for partial mask on"
1314                                          " \"ipv6.hdr.dst_addr\" field"
1315                                          " for vxlan encapsulation");
1316                 /* More IPv6 address validations can be put here. */
1317         } else {
1318                 /*
1319                  * Kernel uses the destination IP address to determine
1320                  * the routing path and obtain the MAC destination
1321                  * address (heigh or gate), so IP destination address
1322                  * must be specified within the tc rule.
1323                  */
1324                 return rte_flow_error_set(error, EINVAL,
1325                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1326                                           "outer ipv6 destination address"
1327                                           " must be specified for"
1328                                           " vxlan encapsulation");
1329         }
1330         if (memcmp(&mask->hdr.src_addr,
1331                    &flow_tcf_mask_empty.ipv6.hdr.src_addr,
1332                    IPV6_ADDR_LEN)) {
1333                 if (memcmp(&mask->hdr.src_addr,
1334                            &rte_flow_item_ipv6_mask.hdr.src_addr,
1335                            IPV6_ADDR_LEN))
1336                         return rte_flow_error_set
1337                                         (error, ENOTSUP,
1338                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1339                                          "no support for partial mask on"
1340                                          " \"ipv6.hdr.src_addr\" field"
1341                                          " for vxlan encapsulation");
1342                 /* More L3 address validation can be put here. */
1343         } else {
1344                 /*
1345                  * Kernel uses the source IP address to select the
1346                  * interface for egress encapsulated traffic, so
1347                  * it must be specified in the tc rule.
1348                  */
1349                 return rte_flow_error_set(error, EINVAL,
1350                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1351                                           "outer L3 source address"
1352                                           " must be specified for"
1353                                           " vxlan encapsulation");
1354         }
1355         return 0;
1356 }
1357
1358 /**
1359  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_UDP item for E-Switch.
1360  * The routine checks the UDP fields to be used in encapsulation header.
1361  *
1362  * @param[in] item
1363  *   Pointer to the item structure.
1364  * @param[out] error
1365  *   Pointer to the error structure.
1366  *
1367  * @return
1368  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1369  **/
1370 static int
1371 flow_tcf_validate_vxlan_encap_udp(const struct rte_flow_item *item,
1372                                   struct rte_flow_error *error)
1373 {
1374         const struct rte_flow_item_udp *spec = item->spec;
1375         const struct rte_flow_item_udp *mask = item->mask;
1376
1377         if (!spec) {
1378                 /*
1379                  * Specification for UDP ports cannot be empty
1380                  * because it is required by tunnel_key parameter.
1381                  */
1382                 return rte_flow_error_set(error, EINVAL,
1383                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1384                                           "NULL UDP port specification "
1385                                           " for vxlan encapsulation");
1386         }
1387         if (!mask)
1388                 mask = &rte_flow_item_udp_mask;
1389         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1390                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1391                         return rte_flow_error_set
1392                                         (error, ENOTSUP,
1393                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1394                                          "no support for partial mask on"
1395                                          " \"udp.hdr.dst_port\" field"
1396                                          " for vxlan encapsulation");
1397                 if (!spec->hdr.dst_port)
1398                         return rte_flow_error_set
1399                                         (error, EINVAL,
1400                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
1401                                          "outer UDP remote port cannot be"
1402                                          " 0 for vxlan encapsulation");
1403         } else {
1404                 return rte_flow_error_set(error, EINVAL,
1405                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1406                                           "outer UDP remote port"
1407                                           " must be specified for"
1408                                           " vxlan encapsulation");
1409         }
1410         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1411                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1412                         return rte_flow_error_set
1413                                         (error, ENOTSUP,
1414                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1415                                          "no support for partial mask on"
1416                                          " \"udp.hdr.src_port\" field"
1417                                          " for vxlan encapsulation");
1418                 DRV_LOG(WARNING,
1419                         "outer UDP source port cannot be"
1420                         " forced for vxlan encapsulation,"
1421                         " parameter ignored");
1422         }
1423         return 0;
1424 }
1425
1426 /**
1427  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_VXLAN item for E-Switch.
1428  * The routine checks the VNIP fields to be used in encapsulation header.
1429  *
1430  * @param[in] item
1431  *   Pointer to the item structure.
1432  * @param[out] error
1433  *   Pointer to the error structure.
1434  *
1435  * @return
1436  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1437  **/
1438 static int
1439 flow_tcf_validate_vxlan_encap_vni(const struct rte_flow_item *item,
1440                                   struct rte_flow_error *error)
1441 {
1442         const struct rte_flow_item_vxlan *spec = item->spec;
1443         const struct rte_flow_item_vxlan *mask = item->mask;
1444
1445         if (!spec) {
1446                 /* Outer VNI is required by tunnel_key parameter. */
1447                 return rte_flow_error_set(error, EINVAL,
1448                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1449                                           "NULL VNI specification"
1450                                           " for vxlan encapsulation");
1451         }
1452         if (!mask)
1453                 mask = &rte_flow_item_vxlan_mask;
1454         if (!mask->vni[0] && !mask->vni[1] && !mask->vni[2])
1455                 return rte_flow_error_set(error, EINVAL,
1456                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1457                                           "outer VNI must be specified "
1458                                           "for vxlan encapsulation");
1459         if (mask->vni[0] != 0xff ||
1460             mask->vni[1] != 0xff ||
1461             mask->vni[2] != 0xff)
1462                 return rte_flow_error_set(error, ENOTSUP,
1463                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1464                                           "no support for partial mask on"
1465                                           " \"vxlan.vni\" field");
1466
1467         if (!spec->vni[0] && !spec->vni[1] && !spec->vni[2])
1468                 return rte_flow_error_set(error, EINVAL,
1469                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1470                                           "vxlan vni cannot be 0");
1471         return 0;
1472 }
1473
1474 /**
1475  * Validate VXLAN_ENCAP action item list for E-Switch.
1476  * The routine checks items to be used in encapsulation header.
1477  *
1478  * @param[in] action
1479  *   Pointer to the VXLAN_ENCAP action structure.
1480  * @param[out] error
1481  *   Pointer to the error structure.
1482  *
1483  * @return
1484  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1485  **/
1486 static int
1487 flow_tcf_validate_vxlan_encap(const struct rte_flow_action *action,
1488                               struct rte_flow_error *error)
1489 {
1490         const struct rte_flow_item *items;
1491         int ret;
1492         uint32_t item_flags = 0;
1493
1494         if (!action->conf)
1495                 return rte_flow_error_set(error, EINVAL,
1496                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1497                                           "Missing vxlan tunnel"
1498                                           " action configuration");
1499         items = ((const struct rte_flow_action_vxlan_encap *)
1500                                         action->conf)->definition;
1501         if (!items)
1502                 return rte_flow_error_set(error, EINVAL,
1503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1504                                           "Missing vxlan tunnel"
1505                                           " encapsulation parameters");
1506         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1507                 switch (items->type) {
1508                 case RTE_FLOW_ITEM_TYPE_VOID:
1509                         break;
1510                 case RTE_FLOW_ITEM_TYPE_ETH:
1511                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1512                                                           error);
1513                         if (ret < 0)
1514                                 return ret;
1515                         ret = flow_tcf_validate_vxlan_encap_eth(items, error);
1516                         if (ret < 0)
1517                                 return ret;
1518                         item_flags |= MLX5_FLOW_LAYER_OUTER_L2;
1519                         break;
1520                 break;
1521                 case RTE_FLOW_ITEM_TYPE_IPV4:
1522                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1523                                                            error);
1524                         if (ret < 0)
1525                                 return ret;
1526                         ret = flow_tcf_validate_vxlan_encap_ipv4(items, error);
1527                         if (ret < 0)
1528                                 return ret;
1529                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1530                         break;
1531                 case RTE_FLOW_ITEM_TYPE_IPV6:
1532                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1533                                                            error);
1534                         if (ret < 0)
1535                                 return ret;
1536                         ret = flow_tcf_validate_vxlan_encap_ipv6(items, error);
1537                         if (ret < 0)
1538                                 return ret;
1539                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1540                         break;
1541                 case RTE_FLOW_ITEM_TYPE_UDP:
1542                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1543                                                            0xFF, error);
1544                         if (ret < 0)
1545                                 return ret;
1546                         ret = flow_tcf_validate_vxlan_encap_udp(items, error);
1547                         if (ret < 0)
1548                                 return ret;
1549                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
1550                         break;
1551                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1552                         ret = mlx5_flow_validate_item_vxlan(items,
1553                                                             item_flags, error);
1554                         if (ret < 0)
1555                                 return ret;
1556                         ret = flow_tcf_validate_vxlan_encap_vni(items, error);
1557                         if (ret < 0)
1558                                 return ret;
1559                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1560                         break;
1561                 default:
1562                         return rte_flow_error_set
1563                                         (error, ENOTSUP,
1564                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1565                                          "vxlan encap item not supported");
1566                 }
1567         }
1568         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1569                 return rte_flow_error_set(error, EINVAL,
1570                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1571                                           "no outer IP layer found"
1572                                           " for vxlan encapsulation");
1573         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1574                 return rte_flow_error_set(error, EINVAL,
1575                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1576                                           "no outer UDP layer found"
1577                                           " for vxlan encapsulation");
1578         if (!(item_flags & MLX5_FLOW_LAYER_VXLAN))
1579                 return rte_flow_error_set(error, EINVAL,
1580                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1581                                           "no VXLAN VNI found"
1582                                           " for vxlan encapsulation");
1583         return 0;
1584 }
1585
1586 /**
1587  * Validate outer RTE_FLOW_ITEM_TYPE_UDP item if tunnel item
1588  * RTE_FLOW_ITEM_TYPE_VXLAN is present in item list.
1589  *
1590  * @param[in] udp
1591  *   Outer UDP layer item (if any, NULL otherwise).
1592  * @param[out] error
1593  *   Pointer to the error structure.
1594  *
1595  * @return
1596  *   0 on success, a negative errno value otherwise and rte_errno is set.
1597  **/
1598 static int
1599 flow_tcf_validate_vxlan_decap_udp(const struct rte_flow_item *udp,
1600                                   struct rte_flow_error *error)
1601 {
1602         const struct rte_flow_item_udp *spec = udp->spec;
1603         const struct rte_flow_item_udp *mask = udp->mask;
1604
1605         if (!spec)
1606                 /*
1607                  * Specification for UDP ports cannot be empty
1608                  * because it is required as decap parameter.
1609                  */
1610                 return rte_flow_error_set(error, EINVAL,
1611                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1612                                           "NULL UDP port specification"
1613                                           " for VXLAN decapsulation");
1614         if (!mask)
1615                 mask = &rte_flow_item_udp_mask;
1616         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1617                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1618                         return rte_flow_error_set
1619                                         (error, ENOTSUP,
1620                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1621                                          "no support for partial mask on"
1622                                          " \"udp.hdr.dst_port\" field");
1623                 if (!spec->hdr.dst_port)
1624                         return rte_flow_error_set
1625                                         (error, EINVAL,
1626                                          RTE_FLOW_ERROR_TYPE_ITEM, udp,
1627                                          "zero decap local UDP port");
1628         } else {
1629                 return rte_flow_error_set(error, EINVAL,
1630                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1631                                           "outer UDP destination port must be "
1632                                           "specified for vxlan decapsulation");
1633         }
1634         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1635                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1636                         return rte_flow_error_set
1637                                         (error, ENOTSUP,
1638                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1639                                          "no support for partial mask on"
1640                                          " \"udp.hdr.src_port\" field");
1641                 DRV_LOG(WARNING,
1642                         "outer UDP local port cannot be "
1643                         "forced for VXLAN encapsulation, "
1644                         "parameter ignored");
1645         }
1646         return 0;
1647 }
1648
1649 /**
1650  * Validate flow for E-Switch.
1651  *
1652  * @param[in] priv
1653  *   Pointer to the priv structure.
1654  * @param[in] attr
1655  *   Pointer to the flow attributes.
1656  * @param[in] items
1657  *   Pointer to the list of items.
1658  * @param[in] actions
1659  *   Pointer to the list of actions.
1660  * @param[out] error
1661  *   Pointer to the error structure.
1662  *
1663  * @return
1664  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1665  */
1666 static int
1667 flow_tcf_validate(struct rte_eth_dev *dev,
1668                   const struct rte_flow_attr *attr,
1669                   const struct rte_flow_item items[],
1670                   const struct rte_flow_action actions[],
1671                   struct rte_flow_error *error)
1672 {
1673         union {
1674                 const struct rte_flow_item_port_id *port_id;
1675                 const struct rte_flow_item_eth *eth;
1676                 const struct rte_flow_item_vlan *vlan;
1677                 const struct rte_flow_item_ipv4 *ipv4;
1678                 const struct rte_flow_item_ipv6 *ipv6;
1679                 const struct rte_flow_item_tcp *tcp;
1680                 const struct rte_flow_item_udp *udp;
1681                 const struct rte_flow_item_vxlan *vxlan;
1682         } spec, mask;
1683         union {
1684                 const struct rte_flow_action_port_id *port_id;
1685                 const struct rte_flow_action_jump *jump;
1686                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
1687                 const struct rte_flow_action_of_set_vlan_vid *
1688                         of_set_vlan_vid;
1689                 const struct rte_flow_action_of_set_vlan_pcp *
1690                         of_set_vlan_pcp;
1691                 const struct rte_flow_action_vxlan_encap *vxlan_encap;
1692                 const struct rte_flow_action_set_ipv4 *set_ipv4;
1693                 const struct rte_flow_action_set_ipv6 *set_ipv6;
1694         } conf;
1695         const struct rte_flow_item *outer_udp = NULL;
1696         uint64_t item_flags = 0;
1697         uint64_t action_flags = 0;
1698         uint8_t next_protocol = -1;
1699         unsigned int tcm_ifindex = 0;
1700         uint8_t pedit_validated = 0;
1701         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
1702         struct rte_eth_dev *port_id_dev = NULL;
1703         bool in_port_id_set;
1704         int ret;
1705
1706         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
1707                                                 PTOI_TABLE_SZ_MAX(dev)));
1708         ret = flow_tcf_validate_attributes(attr, error);
1709         if (ret < 0)
1710                 return ret;
1711         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1712                 unsigned int i;
1713                 uint64_t current_action_flag = 0;
1714
1715                 switch (actions->type) {
1716                 case RTE_FLOW_ACTION_TYPE_VOID:
1717                         break;
1718                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
1719                         current_action_flag = MLX5_FLOW_ACTION_PORT_ID;
1720                         if (!actions->conf)
1721                                 break;
1722                         conf.port_id = actions->conf;
1723                         if (conf.port_id->original)
1724                                 i = 0;
1725                         else
1726                                 for (i = 0; ptoi[i].ifindex; ++i)
1727                                         if (ptoi[i].port_id == conf.port_id->id)
1728                                                 break;
1729                         if (!ptoi[i].ifindex)
1730                                 return rte_flow_error_set
1731                                         (error, ENODEV,
1732                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1733                                          conf.port_id,
1734                                          "missing data to convert port ID to"
1735                                          " ifindex");
1736                         port_id_dev = &rte_eth_devices[conf.port_id->id];
1737                         break;
1738                 case RTE_FLOW_ACTION_TYPE_JUMP:
1739                         current_action_flag = MLX5_FLOW_ACTION_JUMP;
1740                         if (!actions->conf)
1741                                 break;
1742                         conf.jump = actions->conf;
1743                         if (attr->group >= conf.jump->group)
1744                                 return rte_flow_error_set
1745                                         (error, ENOTSUP,
1746                                          RTE_FLOW_ERROR_TYPE_ACTION,
1747                                          actions,
1748                                          "can jump only to a group forward");
1749                         break;
1750                 case RTE_FLOW_ACTION_TYPE_DROP:
1751                         current_action_flag = MLX5_FLOW_ACTION_DROP;
1752                         break;
1753                 case RTE_FLOW_ACTION_TYPE_COUNT:
1754                         break;
1755                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
1756                         current_action_flag = MLX5_FLOW_ACTION_OF_POP_VLAN;
1757                         break;
1758                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1759                         rte_be16_t ethertype;
1760
1761                         current_action_flag = MLX5_FLOW_ACTION_OF_PUSH_VLAN;
1762                         if (!actions->conf)
1763                                 break;
1764                         conf.of_push_vlan = actions->conf;
1765                         ethertype = conf.of_push_vlan->ethertype;
1766                         if (ethertype != RTE_BE16(ETH_P_8021Q) &&
1767                             ethertype != RTE_BE16(ETH_P_8021AD))
1768                                 return rte_flow_error_set
1769                                         (error, EINVAL,
1770                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1771                                          "vlan push TPID must be "
1772                                          "802.1Q or 802.1AD");
1773                         break;
1774                 }
1775                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
1776                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1777                                 return rte_flow_error_set
1778                                         (error, ENOTSUP,
1779                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1780                                          "vlan modify is not supported,"
1781                                          " set action must follow push action");
1782                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
1783                         break;
1784                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
1785                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1786                                 return rte_flow_error_set
1787                                         (error, ENOTSUP,
1788                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1789                                          "vlan modify is not supported,"
1790                                          " set action must follow push action");
1791                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
1792                         break;
1793                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1794                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_DECAP;
1795                         break;
1796                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1797                         ret = flow_tcf_validate_vxlan_encap(actions, error);
1798                         if (ret < 0)
1799                                 return ret;
1800                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_ENCAP;
1801                         break;
1802                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
1803                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_SRC;
1804                         break;
1805                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
1806                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_DST;
1807                         break;
1808                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
1809                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_SRC;
1810                         break;
1811                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
1812                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_DST;
1813                         break;
1814                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
1815                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_SRC;
1816                         break;
1817                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
1818                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_DST;
1819                         break;
1820                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
1821                         current_action_flag = MLX5_FLOW_ACTION_SET_TTL;
1822                         break;
1823                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
1824                         current_action_flag = MLX5_FLOW_ACTION_DEC_TTL;
1825                         break;
1826                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
1827                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_SRC;
1828                         break;
1829                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
1830                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_DST;
1831                         break;
1832                 default:
1833                         return rte_flow_error_set(error, ENOTSUP,
1834                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1835                                                   actions,
1836                                                   "action not supported");
1837                 }
1838                 if (current_action_flag & MLX5_TCF_CONFIG_ACTIONS) {
1839                         if (!actions->conf)
1840                                 return rte_flow_error_set
1841                                         (error, EINVAL,
1842                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1843                                          actions,
1844                                          "action configuration not set");
1845                 }
1846                 if ((current_action_flag & MLX5_TCF_PEDIT_ACTIONS) &&
1847                     pedit_validated)
1848                         return rte_flow_error_set(error, ENOTSUP,
1849                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1850                                                   actions,
1851                                                   "set actions should be "
1852                                                   "listed successively");
1853                 if ((current_action_flag & ~MLX5_TCF_PEDIT_ACTIONS) &&
1854                     (action_flags & MLX5_TCF_PEDIT_ACTIONS))
1855                         pedit_validated = 1;
1856                 if ((current_action_flag & MLX5_TCF_FATE_ACTIONS) &&
1857                     (action_flags & MLX5_TCF_FATE_ACTIONS))
1858                         return rte_flow_error_set(error, EINVAL,
1859                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1860                                                   actions,
1861                                                   "can't have multiple fate"
1862                                                   " actions");
1863                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1864                     (action_flags & MLX5_TCF_VXLAN_ACTIONS))
1865                         return rte_flow_error_set(error, EINVAL,
1866                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1867                                                   actions,
1868                                                   "can't have multiple vxlan"
1869                                                   " actions");
1870                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1871                     (action_flags & MLX5_TCF_VLAN_ACTIONS))
1872                         return rte_flow_error_set(error, ENOTSUP,
1873                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1874                                                   actions,
1875                                                   "can't have vxlan and vlan"
1876                                                   " actions in the same rule");
1877                 action_flags |= current_action_flag;
1878         }
1879         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1880                 unsigned int i;
1881
1882                 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
1883                     items->type != RTE_FLOW_ITEM_TYPE_ETH)
1884                         return rte_flow_error_set(error, ENOTSUP,
1885                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1886                                                   items,
1887                                                   "only L2 inner item"
1888                                                   " is supported");
1889                 switch (items->type) {
1890                 case RTE_FLOW_ITEM_TYPE_VOID:
1891                         break;
1892                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
1893                         mask.port_id = flow_tcf_item_mask
1894                                 (items, &rte_flow_item_port_id_mask,
1895                                  &flow_tcf_mask_supported.port_id,
1896                                  &flow_tcf_mask_empty.port_id,
1897                                  sizeof(flow_tcf_mask_supported.port_id),
1898                                  error);
1899                         if (!mask.port_id)
1900                                 return -rte_errno;
1901                         if (mask.port_id == &flow_tcf_mask_empty.port_id) {
1902                                 in_port_id_set = 1;
1903                                 break;
1904                         }
1905                         spec.port_id = items->spec;
1906                         if (mask.port_id->id && mask.port_id->id != 0xffffffff)
1907                                 return rte_flow_error_set
1908                                         (error, ENOTSUP,
1909                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1910                                          mask.port_id,
1911                                          "no support for partial mask on"
1912                                          " \"id\" field");
1913                         if (!mask.port_id->id)
1914                                 i = 0;
1915                         else
1916                                 for (i = 0; ptoi[i].ifindex; ++i)
1917                                         if (ptoi[i].port_id == spec.port_id->id)
1918                                                 break;
1919                         if (!ptoi[i].ifindex)
1920                                 return rte_flow_error_set
1921                                         (error, ENODEV,
1922                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1923                                          spec.port_id,
1924                                          "missing data to convert port ID to"
1925                                          " ifindex");
1926                         if (in_port_id_set && ptoi[i].ifindex != tcm_ifindex)
1927                                 return rte_flow_error_set
1928                                         (error, ENOTSUP,
1929                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1930                                          spec.port_id,
1931                                          "cannot match traffic for"
1932                                          " several port IDs through"
1933                                          " a single flow rule");
1934                         tcm_ifindex = ptoi[i].ifindex;
1935                         in_port_id_set = 1;
1936                         break;
1937                 case RTE_FLOW_ITEM_TYPE_ETH:
1938                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1939                                                           error);
1940                         if (ret < 0)
1941                                 return ret;
1942                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
1943                                         MLX5_FLOW_LAYER_INNER_L2 :
1944                                         MLX5_FLOW_LAYER_OUTER_L2;
1945                         /* TODO:
1946                          * Redundant check due to different supported mask.
1947                          * Same for the rest of items.
1948                          */
1949                         mask.eth = flow_tcf_item_mask
1950                                 (items, &rte_flow_item_eth_mask,
1951                                  &flow_tcf_mask_supported.eth,
1952                                  &flow_tcf_mask_empty.eth,
1953                                  sizeof(flow_tcf_mask_supported.eth),
1954                                  error);
1955                         if (!mask.eth)
1956                                 return -rte_errno;
1957                         if (mask.eth->type && mask.eth->type !=
1958                             RTE_BE16(0xffff))
1959                                 return rte_flow_error_set
1960                                         (error, ENOTSUP,
1961                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1962                                          mask.eth,
1963                                          "no support for partial mask on"
1964                                          " \"type\" field");
1965                         break;
1966                 case RTE_FLOW_ITEM_TYPE_VLAN:
1967                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1968                                                            error);
1969                         if (ret < 0)
1970                                 return ret;
1971                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
1972                         mask.vlan = flow_tcf_item_mask
1973                                 (items, &rte_flow_item_vlan_mask,
1974                                  &flow_tcf_mask_supported.vlan,
1975                                  &flow_tcf_mask_empty.vlan,
1976                                  sizeof(flow_tcf_mask_supported.vlan),
1977                                  error);
1978                         if (!mask.vlan)
1979                                 return -rte_errno;
1980                         if ((mask.vlan->tci & RTE_BE16(0xe000) &&
1981                              (mask.vlan->tci & RTE_BE16(0xe000)) !=
1982                               RTE_BE16(0xe000)) ||
1983                             (mask.vlan->tci & RTE_BE16(0x0fff) &&
1984                              (mask.vlan->tci & RTE_BE16(0x0fff)) !=
1985                               RTE_BE16(0x0fff)) ||
1986                             (mask.vlan->inner_type &&
1987                              mask.vlan->inner_type != RTE_BE16(0xffff)))
1988                                 return rte_flow_error_set
1989                                         (error, ENOTSUP,
1990                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1991                                          mask.vlan,
1992                                          "no support for partial masks on"
1993                                          " \"tci\" (PCP and VID parts) and"
1994                                          " \"inner_type\" fields");
1995                         break;
1996                 case RTE_FLOW_ITEM_TYPE_IPV4:
1997                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1998                                                            error);
1999                         if (ret < 0)
2000                                 return ret;
2001                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2002                         mask.ipv4 = flow_tcf_item_mask
2003                                 (items, &rte_flow_item_ipv4_mask,
2004                                  &flow_tcf_mask_supported.ipv4,
2005                                  &flow_tcf_mask_empty.ipv4,
2006                                  sizeof(flow_tcf_mask_supported.ipv4),
2007                                  error);
2008                         if (!mask.ipv4)
2009                                 return -rte_errno;
2010                         if (mask.ipv4->hdr.next_proto_id &&
2011                             mask.ipv4->hdr.next_proto_id != 0xff)
2012                                 return rte_flow_error_set
2013                                         (error, ENOTSUP,
2014                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2015                                          mask.ipv4,
2016                                          "no support for partial mask on"
2017                                          " \"hdr.next_proto_id\" field");
2018                         else if (mask.ipv4->hdr.next_proto_id)
2019                                 next_protocol =
2020                                         ((const struct rte_flow_item_ipv4 *)
2021                                          (items->spec))->hdr.next_proto_id;
2022                         break;
2023                 case RTE_FLOW_ITEM_TYPE_IPV6:
2024                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2025                                                            error);
2026                         if (ret < 0)
2027                                 return ret;
2028                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2029                         mask.ipv6 = flow_tcf_item_mask
2030                                 (items, &rte_flow_item_ipv6_mask,
2031                                  &flow_tcf_mask_supported.ipv6,
2032                                  &flow_tcf_mask_empty.ipv6,
2033                                  sizeof(flow_tcf_mask_supported.ipv6),
2034                                  error);
2035                         if (!mask.ipv6)
2036                                 return -rte_errno;
2037                         if (mask.ipv6->hdr.proto &&
2038                             mask.ipv6->hdr.proto != 0xff)
2039                                 return rte_flow_error_set
2040                                         (error, ENOTSUP,
2041                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2042                                          mask.ipv6,
2043                                          "no support for partial mask on"
2044                                          " \"hdr.proto\" field");
2045                         else if (mask.ipv6->hdr.proto)
2046                                 next_protocol =
2047                                         ((const struct rte_flow_item_ipv6 *)
2048                                          (items->spec))->hdr.proto;
2049                         break;
2050                 case RTE_FLOW_ITEM_TYPE_UDP:
2051                         ret = mlx5_flow_validate_item_udp(items, item_flags,
2052                                                           next_protocol, error);
2053                         if (ret < 0)
2054                                 return ret;
2055                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
2056                         mask.udp = flow_tcf_item_mask
2057                                 (items, &rte_flow_item_udp_mask,
2058                                  &flow_tcf_mask_supported.udp,
2059                                  &flow_tcf_mask_empty.udp,
2060                                  sizeof(flow_tcf_mask_supported.udp),
2061                                  error);
2062                         if (!mask.udp)
2063                                 return -rte_errno;
2064                         /*
2065                          * Save the presumed outer UDP item for extra check
2066                          * if the tunnel item will be found later in the list.
2067                          */
2068                         if (!(item_flags & MLX5_FLOW_LAYER_TUNNEL))
2069                                 outer_udp = items;
2070                         break;
2071                 case RTE_FLOW_ITEM_TYPE_TCP:
2072                         ret = mlx5_flow_validate_item_tcp
2073                                              (items, item_flags,
2074                                               next_protocol,
2075                                               &flow_tcf_mask_supported.tcp,
2076                                               error);
2077                         if (ret < 0)
2078                                 return ret;
2079                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
2080                         mask.tcp = flow_tcf_item_mask
2081                                 (items, &rte_flow_item_tcp_mask,
2082                                  &flow_tcf_mask_supported.tcp,
2083                                  &flow_tcf_mask_empty.tcp,
2084                                  sizeof(flow_tcf_mask_supported.tcp),
2085                                  error);
2086                         if (!mask.tcp)
2087                                 return -rte_errno;
2088                         break;
2089                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2090                         if (!(action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP))
2091                                 return rte_flow_error_set
2092                                         (error, ENOTSUP,
2093                                          RTE_FLOW_ERROR_TYPE_ITEM,
2094                                          items,
2095                                          "vni pattern should be followed by"
2096                                          " vxlan decapsulation action");
2097                         ret = mlx5_flow_validate_item_vxlan(items,
2098                                                             item_flags, error);
2099                         if (ret < 0)
2100                                 return ret;
2101                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
2102                         mask.vxlan = flow_tcf_item_mask
2103                                 (items, &rte_flow_item_vxlan_mask,
2104                                  &flow_tcf_mask_supported.vxlan,
2105                                  &flow_tcf_mask_empty.vxlan,
2106                                  sizeof(flow_tcf_mask_supported.vxlan), error);
2107                         if (!mask.vxlan)
2108                                 return -rte_errno;
2109                         if (mask.vxlan->vni[0] != 0xff ||
2110                             mask.vxlan->vni[1] != 0xff ||
2111                             mask.vxlan->vni[2] != 0xff)
2112                                 return rte_flow_error_set
2113                                         (error, ENOTSUP,
2114                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2115                                          mask.vxlan,
2116                                          "no support for partial or "
2117                                          "empty mask on \"vxlan.vni\" field");
2118                         /*
2119                          * The VNI item assumes the VXLAN tunnel, it requires
2120                          * at least the outer destination UDP port must be
2121                          * specified without wildcards to allow kernel select
2122                          * the virtual VXLAN device by port. Also outer IPv4
2123                          * or IPv6 item must be specified (wilcards or even
2124                          * zero mask are allowed) to let driver know the tunnel
2125                          * IP version and process UDP traffic correctly.
2126                          */
2127                         if (!(item_flags &
2128                              (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2129                               MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2130                                 return rte_flow_error_set
2131                                                  (error, EINVAL,
2132                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2133                                                   NULL,
2134                                                   "no outer IP pattern found"
2135                                                   " for vxlan tunnel");
2136                         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2137                                 return rte_flow_error_set
2138                                                  (error, EINVAL,
2139                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2140                                                   NULL,
2141                                                   "no outer UDP pattern found"
2142                                                   " for vxlan tunnel");
2143                         /*
2144                          * All items preceding the tunnel item become outer
2145                          * ones and we should do extra validation for them
2146                          * due to tc limitations for tunnel outer parameters.
2147                          * Currently only outer UDP item requres extra check,
2148                          * use the saved pointer instead of item list rescan.
2149                          */
2150                         assert(outer_udp);
2151                         ret = flow_tcf_validate_vxlan_decap_udp
2152                                                 (outer_udp, error);
2153                         if (ret < 0)
2154                                 return ret;
2155                         /* Reset L4 protocol for inner parameters. */
2156                         next_protocol = 0xff;
2157                         break;
2158                 default:
2159                         return rte_flow_error_set(error, ENOTSUP,
2160                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2161                                                   items, "item not supported");
2162                 }
2163         }
2164         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2165             (action_flags & MLX5_FLOW_ACTION_DROP))
2166                 return rte_flow_error_set(error, ENOTSUP,
2167                                           RTE_FLOW_ERROR_TYPE_ACTION,
2168                                           actions,
2169                                           "set action is not compatible with "
2170                                           "drop action");
2171         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2172             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2173                 return rte_flow_error_set(error, ENOTSUP,
2174                                           RTE_FLOW_ERROR_TYPE_ACTION,
2175                                           actions,
2176                                           "set action must be followed by "
2177                                           "port_id action");
2178         if (action_flags &
2179            (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST)) {
2180                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4))
2181                         return rte_flow_error_set(error, EINVAL,
2182                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2183                                                   actions,
2184                                                   "no ipv4 item found in"
2185                                                   " pattern");
2186         }
2187         if (action_flags &
2188            (MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST)) {
2189                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6))
2190                         return rte_flow_error_set(error, EINVAL,
2191                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2192                                                   actions,
2193                                                   "no ipv6 item found in"
2194                                                   " pattern");
2195         }
2196         if (action_flags &
2197            (MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST)) {
2198                 if (!(item_flags &
2199                      (MLX5_FLOW_LAYER_OUTER_L4_UDP |
2200                       MLX5_FLOW_LAYER_OUTER_L4_TCP)))
2201                         return rte_flow_error_set(error, EINVAL,
2202                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2203                                                   actions,
2204                                                   "no TCP/UDP item found in"
2205                                                   " pattern");
2206         }
2207         /*
2208          * FW syndrome (0xA9C090):
2209          *     set_flow_table_entry: push vlan action fte in fdb can ONLY be
2210          *     forward to the uplink.
2211          */
2212         if ((action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2213             (action_flags & MLX5_FLOW_ACTION_PORT_ID) &&
2214             ((struct priv *)port_id_dev->data->dev_private)->representor)
2215                 return rte_flow_error_set(error, ENOTSUP,
2216                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2217                                           "vlan push can only be applied"
2218                                           " when forwarding to uplink port");
2219         /*
2220          * FW syndrome (0x294609):
2221          *     set_flow_table_entry: modify/pop/push actions in fdb flow table
2222          *     are supported only while forwarding to vport.
2223          */
2224         if ((action_flags & MLX5_TCF_VLAN_ACTIONS) &&
2225             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2226                 return rte_flow_error_set(error, ENOTSUP,
2227                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2228                                           "vlan actions are supported"
2229                                           " only with port_id action");
2230         if ((action_flags & MLX5_TCF_VXLAN_ACTIONS) &&
2231             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2232                 return rte_flow_error_set(error, ENOTSUP,
2233                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2234                                           "vxlan actions are supported"
2235                                           " only with port_id action");
2236         if (!(action_flags & MLX5_TCF_FATE_ACTIONS))
2237                 return rte_flow_error_set(error, EINVAL,
2238                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2239                                           "no fate action is found");
2240         if (action_flags &
2241            (MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL)) {
2242                 if (!(item_flags &
2243                      (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2244                       MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2245                         return rte_flow_error_set(error, EINVAL,
2246                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2247                                                   actions,
2248                                                   "no IP found in pattern");
2249         }
2250         if (action_flags &
2251             (MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)) {
2252                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L2))
2253                         return rte_flow_error_set(error, ENOTSUP,
2254                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2255                                                   actions,
2256                                                   "no ethernet found in"
2257                                                   " pattern");
2258         }
2259         if ((action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) &&
2260             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
2261                 return rte_flow_error_set(error, EINVAL,
2262                                           RTE_FLOW_ERROR_TYPE_ACTION,
2263                                           NULL,
2264                                           "no VNI pattern found"
2265                                           " for vxlan decap action");
2266         if ((action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) &&
2267             (item_flags & MLX5_FLOW_LAYER_TUNNEL))
2268                 return rte_flow_error_set(error, EINVAL,
2269                                           RTE_FLOW_ERROR_TYPE_ACTION,
2270                                           NULL,
2271                                           "vxlan encap not supported"
2272                                           " for tunneled traffic");
2273         return 0;
2274 }
2275
2276 /**
2277  * Calculate maximum size of memory for flow items of Linux TC flower.
2278  *
2279  * @param[in] attr
2280  *   Pointer to the flow attributes.
2281  * @param[in] items
2282  *   Pointer to the list of items.
2283  *
2284  * @return
2285  *   Maximum size of memory for items.
2286  */
2287 static int
2288 flow_tcf_get_items_size(const struct rte_flow_attr *attr,
2289                         const struct rte_flow_item items[])
2290 {
2291         int size = 0;
2292
2293         size += SZ_NLATTR_STRZ_OF("flower") +
2294                 SZ_NLATTR_NEST + /* TCA_OPTIONS. */
2295                 SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CLS_FLAGS_SKIP_SW. */
2296         if (attr->group > 0)
2297                 size += SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CHAIN. */
2298         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2299                 switch (items->type) {
2300                 case RTE_FLOW_ITEM_TYPE_VOID:
2301                         break;
2302                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2303                         break;
2304                 case RTE_FLOW_ITEM_TYPE_ETH:
2305                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2306                                 SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) * 4;
2307                                 /* dst/src MAC addr and mask. */
2308                         break;
2309                 case RTE_FLOW_ITEM_TYPE_VLAN:
2310                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2311                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2312                                 /* VLAN Ether type. */
2313                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* VLAN prio. */
2314                                 SZ_NLATTR_TYPE_OF(uint16_t); /* VLAN ID. */
2315                         break;
2316                 case RTE_FLOW_ITEM_TYPE_IPV4:
2317                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2318                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2319                                 SZ_NLATTR_TYPE_OF(uint32_t) * 4;
2320                                 /* dst/src IP addr and mask. */
2321                         break;
2322                 case RTE_FLOW_ITEM_TYPE_IPV6:
2323                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2324                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2325                                 SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 4;
2326                                 /* dst/src IP addr and mask. */
2327                         break;
2328                 case RTE_FLOW_ITEM_TYPE_UDP:
2329                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2330                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2331                                 /* dst/src port and mask. */
2332                         break;
2333                 case RTE_FLOW_ITEM_TYPE_TCP:
2334                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2335                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2336                                 /* dst/src port and mask. */
2337                         break;
2338                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2339                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2340                         break;
2341                 default:
2342                         DRV_LOG(WARNING,
2343                                 "unsupported item %p type %d,"
2344                                 " items must be validated before flow creation",
2345                                 (const void *)items, items->type);
2346                         break;
2347                 }
2348         }
2349         return size;
2350 }
2351
2352 /**
2353  * Calculate size of memory to store the VXLAN encapsultion
2354  * related items in the Netlink message buffer. Items list
2355  * is specified by RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action.
2356  * The item list should be validated.
2357  *
2358  * @param[in] action
2359  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2360  *   List of pattern items to scan data from.
2361  *
2362  * @return
2363  *   The size the part of Netlink message buffer to store the
2364  *   VXLAN encapsulation item attributes.
2365  */
2366 static int
2367 flow_tcf_vxlan_encap_size(const struct rte_flow_action *action)
2368 {
2369         const struct rte_flow_item *items;
2370         int size = 0;
2371
2372         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2373         assert(action->conf);
2374
2375         items = ((const struct rte_flow_action_vxlan_encap *)
2376                                         action->conf)->definition;
2377         assert(items);
2378         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2379                 switch (items->type) {
2380                 case RTE_FLOW_ITEM_TYPE_VOID:
2381                         break;
2382                 case RTE_FLOW_ITEM_TYPE_ETH:
2383                         /* This item does not require message buffer. */
2384                         break;
2385                 case RTE_FLOW_ITEM_TYPE_IPV4:
2386                         size += SZ_NLATTR_DATA_OF(IPV4_ADDR_LEN) * 2;
2387                         break;
2388                 case RTE_FLOW_ITEM_TYPE_IPV6:
2389                         size += SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 2;
2390                         break;
2391                 case RTE_FLOW_ITEM_TYPE_UDP: {
2392                         const struct rte_flow_item_udp *udp = items->mask;
2393
2394                         size += SZ_NLATTR_TYPE_OF(uint16_t);
2395                         if (!udp || udp->hdr.src_port != RTE_BE16(0x0000))
2396                                 size += SZ_NLATTR_TYPE_OF(uint16_t);
2397                         break;
2398                 }
2399                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2400                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2401                         break;
2402                 default:
2403                         assert(false);
2404                         DRV_LOG(WARNING,
2405                                 "unsupported item %p type %d,"
2406                                 " items must be validated"
2407                                 " before flow creation",
2408                                 (const void *)items, items->type);
2409                         return 0;
2410                 }
2411         }
2412         return size;
2413 }
2414
2415 /**
2416  * Calculate maximum size of memory for flow actions of Linux TC flower and
2417  * extract specified actions.
2418  *
2419  * @param[in] actions
2420  *   Pointer to the list of actions.
2421  * @param[out] action_flags
2422  *   Pointer to the detected actions.
2423  *
2424  * @return
2425  *   Maximum size of memory for actions.
2426  */
2427 static int
2428 flow_tcf_get_actions_and_size(const struct rte_flow_action actions[],
2429                               uint64_t *action_flags)
2430 {
2431         int size = 0;
2432         uint64_t flags = 0;
2433
2434         size += SZ_NLATTR_NEST; /* TCA_FLOWER_ACT. */
2435         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2436                 switch (actions->type) {
2437                 case RTE_FLOW_ACTION_TYPE_VOID:
2438                         break;
2439                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
2440                         size += SZ_NLATTR_NEST + /* na_act_index. */
2441                                 SZ_NLATTR_STRZ_OF("mirred") +
2442                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2443                                 SZ_NLATTR_TYPE_OF(struct tc_mirred);
2444                         flags |= MLX5_FLOW_ACTION_PORT_ID;
2445                         break;
2446                 case RTE_FLOW_ACTION_TYPE_JUMP:
2447                         size += SZ_NLATTR_NEST + /* na_act_index. */
2448                                 SZ_NLATTR_STRZ_OF("gact") +
2449                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2450                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2451                         flags |= MLX5_FLOW_ACTION_JUMP;
2452                         break;
2453                 case RTE_FLOW_ACTION_TYPE_DROP:
2454                         size += SZ_NLATTR_NEST + /* na_act_index. */
2455                                 SZ_NLATTR_STRZ_OF("gact") +
2456                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2457                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2458                         flags |= MLX5_FLOW_ACTION_DROP;
2459                         break;
2460                 case RTE_FLOW_ACTION_TYPE_COUNT:
2461                         break;
2462                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
2463                         flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
2464                         goto action_of_vlan;
2465                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
2466                         flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
2467                         goto action_of_vlan;
2468                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
2469                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
2470                         goto action_of_vlan;
2471                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
2472                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
2473                         goto action_of_vlan;
2474 action_of_vlan:
2475                         size += SZ_NLATTR_NEST + /* na_act_index. */
2476                                 SZ_NLATTR_STRZ_OF("vlan") +
2477                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2478                                 SZ_NLATTR_TYPE_OF(struct tc_vlan) +
2479                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2480                                 /* VLAN protocol. */
2481                                 SZ_NLATTR_TYPE_OF(uint16_t) + /* VLAN ID. */
2482                                 SZ_NLATTR_TYPE_OF(uint8_t); /* VLAN prio. */
2483                         break;
2484                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2485                         size += SZ_NLATTR_NEST + /* na_act_index. */
2486                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2487                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2488                                 SZ_NLATTR_TYPE_OF(uint8_t);
2489                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2490                         size += flow_tcf_vxlan_encap_size(actions) +
2491                                 RTE_ALIGN_CEIL /* preceding encap params. */
2492                                 (sizeof(struct flow_tcf_vxlan_encap),
2493                                 MNL_ALIGNTO);
2494                         flags |= MLX5_FLOW_ACTION_VXLAN_ENCAP;
2495                         break;
2496                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
2497                         size += SZ_NLATTR_NEST + /* na_act_index. */
2498                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2499                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2500                                 SZ_NLATTR_TYPE_OF(uint8_t);
2501                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2502                         size += RTE_ALIGN_CEIL /* preceding decap params. */
2503                                 (sizeof(struct flow_tcf_vxlan_decap),
2504                                 MNL_ALIGNTO);
2505                         flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2506                         break;
2507                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
2508                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
2509                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
2510                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
2511                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
2512                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
2513                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
2514                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
2515                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
2516                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
2517                         size += flow_tcf_get_pedit_actions_size(&actions,
2518                                                                 &flags);
2519                         break;
2520                 default:
2521                         DRV_LOG(WARNING,
2522                                 "unsupported action %p type %d,"
2523                                 " items must be validated before flow creation",
2524                                 (const void *)actions, actions->type);
2525                         break;
2526                 }
2527         }
2528         *action_flags = flags;
2529         return size;
2530 }
2531
2532 /**
2533  * Brand rtnetlink buffer with unique handle.
2534  *
2535  * This handle should be unique for a given network interface to avoid
2536  * collisions.
2537  *
2538  * @param nlh
2539  *   Pointer to Netlink message.
2540  * @param handle
2541  *   Unique 32-bit handle to use.
2542  */
2543 static void
2544 flow_tcf_nl_brand(struct nlmsghdr *nlh, uint32_t handle)
2545 {
2546         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
2547
2548         tcm->tcm_handle = handle;
2549         DRV_LOG(DEBUG, "Netlink msg %p is branded with handle %x",
2550                 (void *)nlh, handle);
2551 }
2552
2553 /**
2554  * Prepare a flow object for Linux TC flower. It calculates the maximum size of
2555  * memory required, allocates the memory, initializes Netlink message headers
2556  * and set unique TC message handle.
2557  *
2558  * @param[in] attr
2559  *   Pointer to the flow attributes.
2560  * @param[in] items
2561  *   Pointer to the list of items.
2562  * @param[in] actions
2563  *   Pointer to the list of actions.
2564  * @param[out] error
2565  *   Pointer to the error structure.
2566  *
2567  * @return
2568  *   Pointer to mlx5_flow object on success,
2569  *   otherwise NULL and rte_ernno is set.
2570  */
2571 static struct mlx5_flow *
2572 flow_tcf_prepare(const struct rte_flow_attr *attr,
2573                  const struct rte_flow_item items[],
2574                  const struct rte_flow_action actions[],
2575                  struct rte_flow_error *error)
2576 {
2577         size_t size = RTE_ALIGN_CEIL
2578                         (sizeof(struct mlx5_flow),
2579                          alignof(struct flow_tcf_tunnel_hdr)) +
2580                       MNL_ALIGN(sizeof(struct nlmsghdr)) +
2581                       MNL_ALIGN(sizeof(struct tcmsg));
2582         struct mlx5_flow *dev_flow;
2583         uint64_t action_flags = 0;
2584         struct nlmsghdr *nlh;
2585         struct tcmsg *tcm;
2586         uint8_t *sp, *tun = NULL;
2587
2588         size += flow_tcf_get_items_size(attr, items);
2589         size += flow_tcf_get_actions_and_size(actions, &action_flags);
2590         dev_flow = rte_zmalloc(__func__, size, MNL_ALIGNTO);
2591         if (!dev_flow) {
2592                 rte_flow_error_set(error, ENOMEM,
2593                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2594                                    "not enough memory to create E-Switch flow");
2595                 return NULL;
2596         }
2597         sp = (uint8_t *)(dev_flow + 1);
2598         if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) {
2599                 sp = RTE_PTR_ALIGN
2600                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2601                 tun = sp;
2602                 sp += RTE_ALIGN_CEIL
2603                         (sizeof(struct flow_tcf_vxlan_encap),
2604                         MNL_ALIGNTO);
2605 #ifndef NDEBUG
2606                 size -= RTE_ALIGN_CEIL
2607                         (sizeof(struct flow_tcf_vxlan_encap),
2608                         MNL_ALIGNTO);
2609 #endif
2610         } else if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2611                 sp = RTE_PTR_ALIGN
2612                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2613                 tun = sp;
2614                 sp += RTE_ALIGN_CEIL
2615                         (sizeof(struct flow_tcf_vxlan_decap),
2616                         MNL_ALIGNTO);
2617 #ifndef NDEBUG
2618                 size -= RTE_ALIGN_CEIL
2619                         (sizeof(struct flow_tcf_vxlan_decap),
2620                         MNL_ALIGNTO);
2621 #endif
2622         } else {
2623                 sp = RTE_PTR_ALIGN(sp, MNL_ALIGNTO);
2624         }
2625         nlh = mnl_nlmsg_put_header(sp);
2626         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
2627         *dev_flow = (struct mlx5_flow){
2628                 .tcf = (struct mlx5_flow_tcf){
2629 #ifndef NDEBUG
2630                         .nlsize = size - RTE_ALIGN_CEIL
2631                                 (sizeof(struct mlx5_flow),
2632                                  alignof(struct flow_tcf_tunnel_hdr)),
2633 #endif
2634                         .tunnel = (struct flow_tcf_tunnel_hdr *)tun,
2635                         .nlh = nlh,
2636                         .tcm = tcm,
2637                 },
2638         };
2639         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP)
2640                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_DECAP;
2641         else if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP)
2642                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_ENCAP;
2643         /*
2644          * Generate a reasonably unique handle based on the address of the
2645          * target buffer.
2646          *
2647          * This is straightforward on 32-bit systems where the flow pointer can
2648          * be used directly. Otherwise, its least significant part is taken
2649          * after shifting it by the previous power of two of the pointed buffer
2650          * size.
2651          */
2652         if (sizeof(dev_flow) <= 4)
2653                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow);
2654         else
2655                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow >>
2656                                        rte_log2_u32(rte_align32prevpow2(size)));
2657         return dev_flow;
2658 }
2659
2660 /**
2661  * Make adjustments for supporting count actions.
2662  *
2663  * @param[in] dev
2664  *   Pointer to the Ethernet device structure.
2665  * @param[in] dev_flow
2666  *   Pointer to mlx5_flow.
2667  * @param[out] error
2668  *   Pointer to error structure.
2669  *
2670  * @return
2671  *   0 On success else a negative errno value is returned and rte_errno is set.
2672  */
2673 static int
2674 flow_tcf_translate_action_count(struct rte_eth_dev *dev __rte_unused,
2675                                   struct mlx5_flow *dev_flow,
2676                                   struct rte_flow_error *error)
2677 {
2678         struct rte_flow *flow = dev_flow->flow;
2679
2680         if (!flow->counter) {
2681                 flow->counter = flow_tcf_counter_new();
2682                 if (!flow->counter)
2683                         return rte_flow_error_set(error, rte_errno,
2684                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2685                                                   NULL,
2686                                                   "cannot get counter"
2687                                                   " context.");
2688         }
2689         return 0;
2690 }
2691
2692 /**
2693  * Convert VXLAN VNI to 32-bit integer.
2694  *
2695  * @param[in] vni
2696  *   VXLAN VNI in 24-bit wire format.
2697  *
2698  * @return
2699  *   VXLAN VNI as a 32-bit integer value in network endian.
2700  */
2701 static inline rte_be32_t
2702 vxlan_vni_as_be32(const uint8_t vni[3])
2703 {
2704         union {
2705                 uint8_t vni[4];
2706                 rte_be32_t dword;
2707         } ret = {
2708                 .vni = { 0, vni[0], vni[1], vni[2] },
2709         };
2710         return ret.dword;
2711 }
2712
2713 /**
2714  * Helper function to process RTE_FLOW_ITEM_TYPE_ETH entry in configuration
2715  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the MAC address fields
2716  * in the encapsulation parameters structure. The item must be prevalidated,
2717  * no any validation checks performed by function.
2718  *
2719  * @param[in] spec
2720  *   RTE_FLOW_ITEM_TYPE_ETH entry specification.
2721  * @param[in] mask
2722  *   RTE_FLOW_ITEM_TYPE_ETH entry mask.
2723  * @param[out] encap
2724  *   Structure to fill the gathered MAC address data.
2725  */
2726 static void
2727 flow_tcf_parse_vxlan_encap_eth(const struct rte_flow_item_eth *spec,
2728                                const struct rte_flow_item_eth *mask,
2729                                struct flow_tcf_vxlan_encap *encap)
2730 {
2731         /* Item must be validated before. No redundant checks. */
2732         assert(spec);
2733         if (!mask || !memcmp(&mask->dst,
2734                              &rte_flow_item_eth_mask.dst,
2735                              sizeof(rte_flow_item_eth_mask.dst))) {
2736                 /*
2737                  * Ethernet addresses are not supported by
2738                  * tc as tunnel_key parameters. Destination
2739                  * address is needed to form encap packet
2740                  * header and retrieved by kernel from
2741                  * implicit sources (ARP table, etc),
2742                  * address masks are not supported at all.
2743                  */
2744                 encap->eth.dst = spec->dst;
2745                 encap->mask |= FLOW_TCF_ENCAP_ETH_DST;
2746         }
2747         if (!mask || !memcmp(&mask->src,
2748                              &rte_flow_item_eth_mask.src,
2749                              sizeof(rte_flow_item_eth_mask.src))) {
2750                 /*
2751                  * Ethernet addresses are not supported by
2752                  * tc as tunnel_key parameters. Source ethernet
2753                  * address is ignored anyway.
2754                  */
2755                 encap->eth.src = spec->src;
2756                 encap->mask |= FLOW_TCF_ENCAP_ETH_SRC;
2757         }
2758 }
2759
2760 /**
2761  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV4 entry in configuration
2762  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV4 address fields
2763  * in the encapsulation parameters structure. The item must be prevalidated,
2764  * no any validation checks performed by function.
2765  *
2766  * @param[in] spec
2767  *   RTE_FLOW_ITEM_TYPE_IPV4 entry specification.
2768  * @param[out] encap
2769  *   Structure to fill the gathered IPV4 address data.
2770  */
2771 static void
2772 flow_tcf_parse_vxlan_encap_ipv4(const struct rte_flow_item_ipv4 *spec,
2773                                 struct flow_tcf_vxlan_encap *encap)
2774 {
2775         /* Item must be validated before. No redundant checks. */
2776         assert(spec);
2777         encap->ipv4.dst = spec->hdr.dst_addr;
2778         encap->ipv4.src = spec->hdr.src_addr;
2779         encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC |
2780                        FLOW_TCF_ENCAP_IPV4_DST;
2781 }
2782
2783 /**
2784  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV6 entry in configuration
2785  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV6 address fields
2786  * in the encapsulation parameters structure. The item must be prevalidated,
2787  * no any validation checks performed by function.
2788  *
2789  * @param[in] spec
2790  *   RTE_FLOW_ITEM_TYPE_IPV6 entry specification.
2791  * @param[out] encap
2792  *   Structure to fill the gathered IPV6 address data.
2793  */
2794 static void
2795 flow_tcf_parse_vxlan_encap_ipv6(const struct rte_flow_item_ipv6 *spec,
2796                                 struct flow_tcf_vxlan_encap *encap)
2797 {
2798         /* Item must be validated before. No redundant checks. */
2799         assert(spec);
2800         memcpy(encap->ipv6.dst, spec->hdr.dst_addr, IPV6_ADDR_LEN);
2801         memcpy(encap->ipv6.src, spec->hdr.src_addr, IPV6_ADDR_LEN);
2802         encap->mask |= FLOW_TCF_ENCAP_IPV6_SRC |
2803                        FLOW_TCF_ENCAP_IPV6_DST;
2804 }
2805
2806 /**
2807  * Helper function to process RTE_FLOW_ITEM_TYPE_UDP entry in configuration
2808  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the UDP port fields
2809  * in the encapsulation parameters structure. The item must be prevalidated,
2810  * no any validation checks performed by function.
2811  *
2812  * @param[in] spec
2813  *   RTE_FLOW_ITEM_TYPE_UDP entry specification.
2814  * @param[in] mask
2815  *   RTE_FLOW_ITEM_TYPE_UDP entry mask.
2816  * @param[out] encap
2817  *   Structure to fill the gathered UDP port data.
2818  */
2819 static void
2820 flow_tcf_parse_vxlan_encap_udp(const struct rte_flow_item_udp *spec,
2821                                const struct rte_flow_item_udp *mask,
2822                                struct flow_tcf_vxlan_encap *encap)
2823 {
2824         assert(spec);
2825         encap->udp.dst = spec->hdr.dst_port;
2826         encap->mask |= FLOW_TCF_ENCAP_UDP_DST;
2827         if (!mask || mask->hdr.src_port != RTE_BE16(0x0000)) {
2828                 encap->udp.src = spec->hdr.src_port;
2829                 encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC;
2830         }
2831 }
2832
2833 /**
2834  * Helper function to process RTE_FLOW_ITEM_TYPE_VXLAN entry in configuration
2835  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the VNI fields
2836  * in the encapsulation parameters structure. The item must be prevalidated,
2837  * no any validation checks performed by function.
2838  *
2839  * @param[in] spec
2840  *   RTE_FLOW_ITEM_TYPE_VXLAN entry specification.
2841  * @param[out] encap
2842  *   Structure to fill the gathered VNI address data.
2843  */
2844 static void
2845 flow_tcf_parse_vxlan_encap_vni(const struct rte_flow_item_vxlan *spec,
2846                                struct flow_tcf_vxlan_encap *encap)
2847 {
2848         /* Item must be validated before. Do not redundant checks. */
2849         assert(spec);
2850         memcpy(encap->vxlan.vni, spec->vni, sizeof(encap->vxlan.vni));
2851         encap->mask |= FLOW_TCF_ENCAP_VXLAN_VNI;
2852 }
2853
2854 /**
2855  * Populate consolidated encapsulation object from list of pattern items.
2856  *
2857  * Helper function to process configuration of action such as
2858  * RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. The item list should be
2859  * validated, there is no way to return an meaningful error.
2860  *
2861  * @param[in] action
2862  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2863  *   List of pattern items to gather data from.
2864  * @param[out] src
2865  *   Structure to fill gathered data.
2866  */
2867 static void
2868 flow_tcf_vxlan_encap_parse(const struct rte_flow_action *action,
2869                            struct flow_tcf_vxlan_encap *encap)
2870 {
2871         union {
2872                 const struct rte_flow_item_eth *eth;
2873                 const struct rte_flow_item_ipv4 *ipv4;
2874                 const struct rte_flow_item_ipv6 *ipv6;
2875                 const struct rte_flow_item_udp *udp;
2876                 const struct rte_flow_item_vxlan *vxlan;
2877         } spec, mask;
2878         const struct rte_flow_item *items;
2879
2880         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2881         assert(action->conf);
2882
2883         items = ((const struct rte_flow_action_vxlan_encap *)
2884                                         action->conf)->definition;
2885         assert(items);
2886         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2887                 switch (items->type) {
2888                 case RTE_FLOW_ITEM_TYPE_VOID:
2889                         break;
2890                 case RTE_FLOW_ITEM_TYPE_ETH:
2891                         mask.eth = items->mask;
2892                         spec.eth = items->spec;
2893                         flow_tcf_parse_vxlan_encap_eth(spec.eth, mask.eth,
2894                                                        encap);
2895                         break;
2896                 case RTE_FLOW_ITEM_TYPE_IPV4:
2897                         spec.ipv4 = items->spec;
2898                         flow_tcf_parse_vxlan_encap_ipv4(spec.ipv4, encap);
2899                         break;
2900                 case RTE_FLOW_ITEM_TYPE_IPV6:
2901                         spec.ipv6 = items->spec;
2902                         flow_tcf_parse_vxlan_encap_ipv6(spec.ipv6, encap);
2903                         break;
2904                 case RTE_FLOW_ITEM_TYPE_UDP:
2905                         mask.udp = items->mask;
2906                         spec.udp = items->spec;
2907                         flow_tcf_parse_vxlan_encap_udp(spec.udp, mask.udp,
2908                                                        encap);
2909                         break;
2910                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2911                         spec.vxlan = items->spec;
2912                         flow_tcf_parse_vxlan_encap_vni(spec.vxlan, encap);
2913                         break;
2914                 default:
2915                         assert(false);
2916                         DRV_LOG(WARNING,
2917                                 "unsupported item %p type %d,"
2918                                 " items must be validated"
2919                                 " before flow creation",
2920                                 (const void *)items, items->type);
2921                         encap->mask = 0;
2922                         return;
2923                 }
2924         }
2925 }
2926
2927 /**
2928  * Translate flow for Linux TC flower and construct Netlink message.
2929  *
2930  * @param[in] priv
2931  *   Pointer to the priv structure.
2932  * @param[in, out] flow
2933  *   Pointer to the sub flow.
2934  * @param[in] attr
2935  *   Pointer to the flow attributes.
2936  * @param[in] items
2937  *   Pointer to the list of items.
2938  * @param[in] actions
2939  *   Pointer to the list of actions.
2940  * @param[out] error
2941  *   Pointer to the error structure.
2942  *
2943  * @return
2944  *   0 on success, a negative errno value otherwise and rte_ernno is set.
2945  */
2946 static int
2947 flow_tcf_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
2948                    const struct rte_flow_attr *attr,
2949                    const struct rte_flow_item items[],
2950                    const struct rte_flow_action actions[],
2951                    struct rte_flow_error *error)
2952 {
2953         union {
2954                 const struct rte_flow_item_port_id *port_id;
2955                 const struct rte_flow_item_eth *eth;
2956                 const struct rte_flow_item_vlan *vlan;
2957                 const struct rte_flow_item_ipv4 *ipv4;
2958                 const struct rte_flow_item_ipv6 *ipv6;
2959                 const struct rte_flow_item_tcp *tcp;
2960                 const struct rte_flow_item_udp *udp;
2961                 const struct rte_flow_item_vxlan *vxlan;
2962         } spec, mask;
2963         union {
2964                 const struct rte_flow_action_port_id *port_id;
2965                 const struct rte_flow_action_jump *jump;
2966                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
2967                 const struct rte_flow_action_of_set_vlan_vid *
2968                         of_set_vlan_vid;
2969                 const struct rte_flow_action_of_set_vlan_pcp *
2970                         of_set_vlan_pcp;
2971         } conf;
2972         union {
2973                 struct flow_tcf_tunnel_hdr *hdr;
2974                 struct flow_tcf_vxlan_decap *vxlan;
2975         } decap = {
2976                 .hdr = NULL,
2977         };
2978         union {
2979                 struct flow_tcf_tunnel_hdr *hdr;
2980                 struct flow_tcf_vxlan_encap *vxlan;
2981         } encap = {
2982                 .hdr = NULL,
2983         };
2984         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
2985         struct nlmsghdr *nlh = dev_flow->tcf.nlh;
2986         struct tcmsg *tcm = dev_flow->tcf.tcm;
2987         uint32_t na_act_index_cur;
2988         bool eth_type_set = 0;
2989         bool vlan_present = 0;
2990         bool vlan_eth_type_set = 0;
2991         bool ip_proto_set = 0;
2992         struct nlattr *na_flower;
2993         struct nlattr *na_flower_act;
2994         struct nlattr *na_vlan_id = NULL;
2995         struct nlattr *na_vlan_priority = NULL;
2996         uint64_t item_flags = 0;
2997         int ret;
2998
2999         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
3000                                                 PTOI_TABLE_SZ_MAX(dev)));
3001         if (dev_flow->tcf.tunnel) {
3002                 switch (dev_flow->tcf.tunnel->type) {
3003                 case FLOW_TCF_TUNACT_VXLAN_DECAP:
3004                         decap.vxlan = dev_flow->tcf.vxlan_decap;
3005                         break;
3006                 case FLOW_TCF_TUNACT_VXLAN_ENCAP:
3007                         encap.vxlan = dev_flow->tcf.vxlan_encap;
3008                         break;
3009                 /* New tunnel actions can be added here. */
3010                 default:
3011                         assert(false);
3012                         break;
3013                 }
3014         }
3015         nlh = dev_flow->tcf.nlh;
3016         tcm = dev_flow->tcf.tcm;
3017         /* Prepare API must have been called beforehand. */
3018         assert(nlh != NULL && tcm != NULL);
3019         tcm->tcm_family = AF_UNSPEC;
3020         tcm->tcm_ifindex = ptoi[0].ifindex;
3021         tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
3022         /*
3023          * Priority cannot be zero to prevent the kernel from picking one
3024          * automatically.
3025          */
3026         tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
3027                                   RTE_BE16(ETH_P_ALL));
3028         if (attr->group > 0)
3029                 mnl_attr_put_u32(nlh, TCA_CHAIN, attr->group);
3030         mnl_attr_put_strz(nlh, TCA_KIND, "flower");
3031         na_flower = mnl_attr_nest_start(nlh, TCA_OPTIONS);
3032         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3033                 unsigned int i;
3034
3035                 switch (items->type) {
3036                 case RTE_FLOW_ITEM_TYPE_VOID:
3037                         break;
3038                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3039                         mask.port_id = flow_tcf_item_mask
3040                                 (items, &rte_flow_item_port_id_mask,
3041                                  &flow_tcf_mask_supported.port_id,
3042                                  &flow_tcf_mask_empty.port_id,
3043                                  sizeof(flow_tcf_mask_supported.port_id),
3044                                  error);
3045                         assert(mask.port_id);
3046                         if (mask.port_id == &flow_tcf_mask_empty.port_id)
3047                                 break;
3048                         spec.port_id = items->spec;
3049                         if (!mask.port_id->id)
3050                                 i = 0;
3051                         else
3052                                 for (i = 0; ptoi[i].ifindex; ++i)
3053                                         if (ptoi[i].port_id == spec.port_id->id)
3054                                                 break;
3055                         assert(ptoi[i].ifindex);
3056                         tcm->tcm_ifindex = ptoi[i].ifindex;
3057                         break;
3058                 case RTE_FLOW_ITEM_TYPE_ETH:
3059                         item_flags |= (item_flags & MLX5_FLOW_LAYER_VXLAN) ?
3060                                       MLX5_FLOW_LAYER_INNER_L2 :
3061                                       MLX5_FLOW_LAYER_OUTER_L2;
3062                         mask.eth = flow_tcf_item_mask
3063                                 (items, &rte_flow_item_eth_mask,
3064                                  &flow_tcf_mask_supported.eth,
3065                                  &flow_tcf_mask_empty.eth,
3066                                  sizeof(flow_tcf_mask_supported.eth),
3067                                  error);
3068                         assert(mask.eth);
3069                         if (mask.eth == &flow_tcf_mask_empty.eth)
3070                                 break;
3071                         spec.eth = items->spec;
3072                         if (decap.vxlan &&
3073                             !(item_flags & MLX5_FLOW_LAYER_VXLAN)) {
3074                                 DRV_LOG(WARNING,
3075                                         "outer L2 addresses cannot be forced"
3076                                         " for vxlan decapsulation, parameter"
3077                                         " ignored");
3078                                 break;
3079                         }
3080                         if (mask.eth->type) {
3081                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3082                                                  spec.eth->type);
3083                                 eth_type_set = 1;
3084                         }
3085                         if (!is_zero_ether_addr(&mask.eth->dst)) {
3086                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST,
3087                                              ETHER_ADDR_LEN,
3088                                              spec.eth->dst.addr_bytes);
3089                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST_MASK,
3090                                              ETHER_ADDR_LEN,
3091                                              mask.eth->dst.addr_bytes);
3092                         }
3093                         if (!is_zero_ether_addr(&mask.eth->src)) {
3094                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC,
3095                                              ETHER_ADDR_LEN,
3096                                              spec.eth->src.addr_bytes);
3097                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC_MASK,
3098                                              ETHER_ADDR_LEN,
3099                                              mask.eth->src.addr_bytes);
3100                         }
3101                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3102                         break;
3103                 case RTE_FLOW_ITEM_TYPE_VLAN:
3104                         assert(!encap.hdr);
3105                         assert(!decap.hdr);
3106                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
3107                         mask.vlan = flow_tcf_item_mask
3108                                 (items, &rte_flow_item_vlan_mask,
3109                                  &flow_tcf_mask_supported.vlan,
3110                                  &flow_tcf_mask_empty.vlan,
3111                                  sizeof(flow_tcf_mask_supported.vlan),
3112                                  error);
3113                         assert(mask.vlan);
3114                         if (!eth_type_set)
3115                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3116                                                  RTE_BE16(ETH_P_8021Q));
3117                         eth_type_set = 1;
3118                         vlan_present = 1;
3119                         if (mask.vlan == &flow_tcf_mask_empty.vlan)
3120                                 break;
3121                         spec.vlan = items->spec;
3122                         if (mask.vlan->inner_type) {
3123                                 mnl_attr_put_u16(nlh,
3124                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE,
3125                                                  spec.vlan->inner_type);
3126                                 vlan_eth_type_set = 1;
3127                         }
3128                         if (mask.vlan->tci & RTE_BE16(0xe000))
3129                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_VLAN_PRIO,
3130                                                 (rte_be_to_cpu_16
3131                                                  (spec.vlan->tci) >> 13) & 0x7);
3132                         if (mask.vlan->tci & RTE_BE16(0x0fff))
3133                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_VLAN_ID,
3134                                                  rte_be_to_cpu_16
3135                                                  (spec.vlan->tci &
3136                                                   RTE_BE16(0x0fff)));
3137                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3138                         break;
3139                 case RTE_FLOW_ITEM_TYPE_IPV4:
3140                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3141                         mask.ipv4 = flow_tcf_item_mask
3142                                 (items, &rte_flow_item_ipv4_mask,
3143                                  &flow_tcf_mask_supported.ipv4,
3144                                  &flow_tcf_mask_empty.ipv4,
3145                                  sizeof(flow_tcf_mask_supported.ipv4),
3146                                  error);
3147                         assert(mask.ipv4);
3148                         spec.ipv4 = items->spec;
3149                         if (!decap.vxlan) {
3150                                 if (!eth_type_set ||
3151                                     (!vlan_eth_type_set && vlan_present))
3152                                         mnl_attr_put_u16
3153                                                 (nlh,
3154                                                  vlan_present ?
3155                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3156                                                  TCA_FLOWER_KEY_ETH_TYPE,
3157                                                  RTE_BE16(ETH_P_IP));
3158                                 eth_type_set = 1;
3159                                 vlan_eth_type_set = 1;
3160                                 if (mask.ipv4 == &flow_tcf_mask_empty.ipv4)
3161                                         break;
3162                                 if (mask.ipv4->hdr.next_proto_id) {
3163                                         mnl_attr_put_u8
3164                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3165                                                  spec.ipv4->hdr.next_proto_id);
3166                                         ip_proto_set = 1;
3167                                 }
3168                         } else {
3169                                 assert(mask.ipv4 != &flow_tcf_mask_empty.ipv4);
3170                         }
3171                         if (mask.ipv4->hdr.src_addr) {
3172                                 mnl_attr_put_u32
3173                                         (nlh, decap.vxlan ?
3174                                          TCA_FLOWER_KEY_ENC_IPV4_SRC :
3175                                          TCA_FLOWER_KEY_IPV4_SRC,
3176                                          spec.ipv4->hdr.src_addr);
3177                                 mnl_attr_put_u32
3178                                         (nlh, decap.vxlan ?
3179                                          TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK :
3180                                          TCA_FLOWER_KEY_IPV4_SRC_MASK,
3181                                          mask.ipv4->hdr.src_addr);
3182                         }
3183                         if (mask.ipv4->hdr.dst_addr) {
3184                                 mnl_attr_put_u32
3185                                         (nlh, decap.vxlan ?
3186                                          TCA_FLOWER_KEY_ENC_IPV4_DST :
3187                                          TCA_FLOWER_KEY_IPV4_DST,
3188                                          spec.ipv4->hdr.dst_addr);
3189                                 mnl_attr_put_u32
3190                                         (nlh, decap.vxlan ?
3191                                          TCA_FLOWER_KEY_ENC_IPV4_DST_MASK :
3192                                          TCA_FLOWER_KEY_IPV4_DST_MASK,
3193                                          mask.ipv4->hdr.dst_addr);
3194                         }
3195                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3196                         break;
3197                 case RTE_FLOW_ITEM_TYPE_IPV6:
3198                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3199                         mask.ipv6 = flow_tcf_item_mask
3200                                 (items, &rte_flow_item_ipv6_mask,
3201                                  &flow_tcf_mask_supported.ipv6,
3202                                  &flow_tcf_mask_empty.ipv6,
3203                                  sizeof(flow_tcf_mask_supported.ipv6),
3204                                  error);
3205                         assert(mask.ipv6);
3206                         spec.ipv6 = items->spec;
3207                         if (!decap.vxlan) {
3208                                 if (!eth_type_set ||
3209                                     (!vlan_eth_type_set && vlan_present))
3210                                         mnl_attr_put_u16
3211                                                 (nlh,
3212                                                  vlan_present ?
3213                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3214                                                  TCA_FLOWER_KEY_ETH_TYPE,
3215                                                  RTE_BE16(ETH_P_IPV6));
3216                                 eth_type_set = 1;
3217                                 vlan_eth_type_set = 1;
3218                                 if (mask.ipv6 == &flow_tcf_mask_empty.ipv6)
3219                                         break;
3220                                 if (mask.ipv6->hdr.proto) {
3221                                         mnl_attr_put_u8
3222                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3223                                                  spec.ipv6->hdr.proto);
3224                                         ip_proto_set = 1;
3225                                 }
3226                         } else {
3227                                 assert(mask.ipv6 != &flow_tcf_mask_empty.ipv6);
3228                         }
3229                         if (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.src_addr)) {
3230                                 mnl_attr_put(nlh, decap.vxlan ?
3231                                              TCA_FLOWER_KEY_ENC_IPV6_SRC :
3232                                              TCA_FLOWER_KEY_IPV6_SRC,
3233                                              IPV6_ADDR_LEN,
3234                                              spec.ipv6->hdr.src_addr);
3235                                 mnl_attr_put(nlh, decap.vxlan ?
3236                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK :
3237                                              TCA_FLOWER_KEY_IPV6_SRC_MASK,
3238                                              IPV6_ADDR_LEN,
3239                                              mask.ipv6->hdr.src_addr);
3240                         }
3241                         if (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.dst_addr)) {
3242                                 mnl_attr_put(nlh, decap.vxlan ?
3243                                              TCA_FLOWER_KEY_ENC_IPV6_DST :
3244                                              TCA_FLOWER_KEY_IPV6_DST,
3245                                              IPV6_ADDR_LEN,
3246                                              spec.ipv6->hdr.dst_addr);
3247                                 mnl_attr_put(nlh, decap.vxlan ?
3248                                              TCA_FLOWER_KEY_ENC_IPV6_DST_MASK :
3249                                              TCA_FLOWER_KEY_IPV6_DST_MASK,
3250                                              IPV6_ADDR_LEN,
3251                                              mask.ipv6->hdr.dst_addr);
3252                         }
3253                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3254                         break;
3255                 case RTE_FLOW_ITEM_TYPE_UDP:
3256                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3257                         mask.udp = flow_tcf_item_mask
3258                                 (items, &rte_flow_item_udp_mask,
3259                                  &flow_tcf_mask_supported.udp,
3260                                  &flow_tcf_mask_empty.udp,
3261                                  sizeof(flow_tcf_mask_supported.udp),
3262                                  error);
3263                         assert(mask.udp);
3264                         spec.udp = items->spec;
3265                         if (!decap.vxlan) {
3266                                 if (!ip_proto_set)
3267                                         mnl_attr_put_u8
3268                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3269                                                 IPPROTO_UDP);
3270                                 if (mask.udp == &flow_tcf_mask_empty.udp)
3271                                         break;
3272                         } else {
3273                                 assert(mask.udp != &flow_tcf_mask_empty.udp);
3274                                 decap.vxlan->udp_port =
3275                                         rte_be_to_cpu_16
3276                                                 (spec.udp->hdr.dst_port);
3277                         }
3278                         if (mask.udp->hdr.src_port) {
3279                                 mnl_attr_put_u16
3280                                         (nlh, decap.vxlan ?
3281                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT :
3282                                          TCA_FLOWER_KEY_UDP_SRC,
3283                                          spec.udp->hdr.src_port);
3284                                 mnl_attr_put_u16
3285                                         (nlh, decap.vxlan ?
3286                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK :
3287                                          TCA_FLOWER_KEY_UDP_SRC_MASK,
3288                                          mask.udp->hdr.src_port);
3289                         }
3290                         if (mask.udp->hdr.dst_port) {
3291                                 mnl_attr_put_u16
3292                                         (nlh, decap.vxlan ?
3293                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT :
3294                                          TCA_FLOWER_KEY_UDP_DST,
3295                                          spec.udp->hdr.dst_port);
3296                                 mnl_attr_put_u16
3297                                         (nlh, decap.vxlan ?
3298                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK :
3299                                          TCA_FLOWER_KEY_UDP_DST_MASK,
3300                                          mask.udp->hdr.dst_port);
3301                         }
3302                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3303                         break;
3304                 case RTE_FLOW_ITEM_TYPE_TCP:
3305                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3306                         mask.tcp = flow_tcf_item_mask
3307                                 (items, &rte_flow_item_tcp_mask,
3308                                  &flow_tcf_mask_supported.tcp,
3309                                  &flow_tcf_mask_empty.tcp,
3310                                  sizeof(flow_tcf_mask_supported.tcp),
3311                                  error);
3312                         assert(mask.tcp);
3313                         if (!ip_proto_set)
3314                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3315                                                 IPPROTO_TCP);
3316                         if (mask.tcp == &flow_tcf_mask_empty.tcp)
3317                                 break;
3318                         spec.tcp = items->spec;
3319                         if (mask.tcp->hdr.src_port) {
3320                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_SRC,
3321                                                  spec.tcp->hdr.src_port);
3322                                 mnl_attr_put_u16(nlh,
3323                                                  TCA_FLOWER_KEY_TCP_SRC_MASK,
3324                                                  mask.tcp->hdr.src_port);
3325                         }
3326                         if (mask.tcp->hdr.dst_port) {
3327                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_DST,
3328                                                  spec.tcp->hdr.dst_port);
3329                                 mnl_attr_put_u16(nlh,
3330                                                  TCA_FLOWER_KEY_TCP_DST_MASK,
3331                                                  mask.tcp->hdr.dst_port);
3332                         }
3333                         if (mask.tcp->hdr.tcp_flags) {
3334                                 mnl_attr_put_u16
3335                                         (nlh,
3336                                          TCA_FLOWER_KEY_TCP_FLAGS,
3337                                          rte_cpu_to_be_16
3338                                                 (spec.tcp->hdr.tcp_flags));
3339                                 mnl_attr_put_u16
3340                                         (nlh,
3341                                          TCA_FLOWER_KEY_TCP_FLAGS_MASK,
3342                                          rte_cpu_to_be_16
3343                                                 (mask.tcp->hdr.tcp_flags));
3344                         }
3345                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3346                         break;
3347                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3348                         assert(decap.vxlan);
3349                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
3350                         spec.vxlan = items->spec;
3351                         mnl_attr_put_u32(nlh,
3352                                          TCA_FLOWER_KEY_ENC_KEY_ID,
3353                                          vxlan_vni_as_be32(spec.vxlan->vni));
3354                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3355                         break;
3356                 default:
3357                         return rte_flow_error_set(error, ENOTSUP,
3358                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3359                                                   NULL, "item not supported");
3360                 }
3361         }
3362         na_flower_act = mnl_attr_nest_start(nlh, TCA_FLOWER_ACT);
3363         na_act_index_cur = 1;
3364         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3365                 struct nlattr *na_act_index;
3366                 struct nlattr *na_act;
3367                 unsigned int vlan_act;
3368                 unsigned int i;
3369
3370                 switch (actions->type) {
3371                 case RTE_FLOW_ACTION_TYPE_VOID:
3372                         break;
3373                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3374                         conf.port_id = actions->conf;
3375                         if (conf.port_id->original)
3376                                 i = 0;
3377                         else
3378                                 for (i = 0; ptoi[i].ifindex; ++i)
3379                                         if (ptoi[i].port_id == conf.port_id->id)
3380                                                 break;
3381                         assert(ptoi[i].ifindex);
3382                         na_act_index =
3383                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3384                         assert(na_act_index);
3385                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "mirred");
3386                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3387                         assert(na_act);
3388                         if (encap.hdr) {
3389                                 assert(dev_flow->tcf.tunnel);
3390                                 dev_flow->tcf.tunnel->ifindex_ptr =
3391                                         &((struct tc_mirred *)
3392                                         mnl_attr_get_payload
3393                                         (mnl_nlmsg_get_payload_tail
3394                                                 (nlh)))->ifindex;
3395                         }
3396                         mnl_attr_put(nlh, TCA_MIRRED_PARMS,
3397                                      sizeof(struct tc_mirred),
3398                                      &(struct tc_mirred){
3399                                         .action = TC_ACT_STOLEN,
3400                                         .eaction = TCA_EGRESS_REDIR,
3401                                         .ifindex = ptoi[i].ifindex,
3402                                      });
3403                         mnl_attr_nest_end(nlh, na_act);
3404                         mnl_attr_nest_end(nlh, na_act_index);
3405                         break;
3406                 case RTE_FLOW_ACTION_TYPE_JUMP:
3407                         conf.jump = actions->conf;
3408                         na_act_index =
3409                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3410                         assert(na_act_index);
3411                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3412                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3413                         assert(na_act);
3414                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3415                                      sizeof(struct tc_gact),
3416                                      &(struct tc_gact){
3417                                         .action = TC_ACT_GOTO_CHAIN |
3418                                                   conf.jump->group,
3419                                      });
3420                         mnl_attr_nest_end(nlh, na_act);
3421                         mnl_attr_nest_end(nlh, na_act_index);
3422                         break;
3423                 case RTE_FLOW_ACTION_TYPE_DROP:
3424                         na_act_index =
3425                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3426                         assert(na_act_index);
3427                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3428                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3429                         assert(na_act);
3430                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3431                                      sizeof(struct tc_gact),
3432                                      &(struct tc_gact){
3433                                         .action = TC_ACT_SHOT,
3434                                      });
3435                         mnl_attr_nest_end(nlh, na_act);
3436                         mnl_attr_nest_end(nlh, na_act_index);
3437                         break;
3438                 case RTE_FLOW_ACTION_TYPE_COUNT:
3439                         /*
3440                          * Driver adds the count action implicitly for
3441                          * each rule it creates.
3442                          */
3443                         ret = flow_tcf_translate_action_count(dev,
3444                                                               dev_flow, error);
3445                         if (ret < 0)
3446                                 return ret;
3447                         break;
3448                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3449                         conf.of_push_vlan = NULL;
3450                         vlan_act = TCA_VLAN_ACT_POP;
3451                         goto action_of_vlan;
3452                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3453                         conf.of_push_vlan = actions->conf;
3454                         vlan_act = TCA_VLAN_ACT_PUSH;
3455                         goto action_of_vlan;
3456                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3457                         conf.of_set_vlan_vid = actions->conf;
3458                         if (na_vlan_id)
3459                                 goto override_na_vlan_id;
3460                         vlan_act = TCA_VLAN_ACT_MODIFY;
3461                         goto action_of_vlan;
3462                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3463                         conf.of_set_vlan_pcp = actions->conf;
3464                         if (na_vlan_priority)
3465                                 goto override_na_vlan_priority;
3466                         vlan_act = TCA_VLAN_ACT_MODIFY;
3467                         goto action_of_vlan;
3468 action_of_vlan:
3469                         na_act_index =
3470                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3471                         assert(na_act_index);
3472                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "vlan");
3473                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3474                         assert(na_act);
3475                         mnl_attr_put(nlh, TCA_VLAN_PARMS,
3476                                      sizeof(struct tc_vlan),
3477                                      &(struct tc_vlan){
3478                                         .action = TC_ACT_PIPE,
3479                                         .v_action = vlan_act,
3480                                      });
3481                         if (vlan_act == TCA_VLAN_ACT_POP) {
3482                                 mnl_attr_nest_end(nlh, na_act);
3483                                 mnl_attr_nest_end(nlh, na_act_index);
3484                                 break;
3485                         }
3486                         if (vlan_act == TCA_VLAN_ACT_PUSH)
3487                                 mnl_attr_put_u16(nlh,
3488                                                  TCA_VLAN_PUSH_VLAN_PROTOCOL,
3489                                                  conf.of_push_vlan->ethertype);
3490                         na_vlan_id = mnl_nlmsg_get_payload_tail(nlh);
3491                         mnl_attr_put_u16(nlh, TCA_VLAN_PAD, 0);
3492                         na_vlan_priority = mnl_nlmsg_get_payload_tail(nlh);
3493                         mnl_attr_put_u8(nlh, TCA_VLAN_PAD, 0);
3494                         mnl_attr_nest_end(nlh, na_act);
3495                         mnl_attr_nest_end(nlh, na_act_index);
3496                         if (actions->type ==
3497                             RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
3498 override_na_vlan_id:
3499                                 na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
3500                                 *(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
3501                                         rte_be_to_cpu_16
3502                                         (conf.of_set_vlan_vid->vlan_vid);
3503                         } else if (actions->type ==
3504                                    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
3505 override_na_vlan_priority:
3506                                 na_vlan_priority->nla_type =
3507                                         TCA_VLAN_PUSH_VLAN_PRIORITY;
3508                                 *(uint8_t *)mnl_attr_get_payload
3509                                         (na_vlan_priority) =
3510                                         conf.of_set_vlan_pcp->vlan_pcp;
3511                         }
3512                         break;
3513                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3514                         assert(decap.vxlan);
3515                         assert(dev_flow->tcf.tunnel);
3516                         dev_flow->tcf.tunnel->ifindex_ptr =
3517                                 (unsigned int *)&tcm->tcm_ifindex;
3518                         na_act_index =
3519                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3520                         assert(na_act_index);
3521                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3522                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3523                         assert(na_act);
3524                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3525                                 sizeof(struct tc_tunnel_key),
3526                                 &(struct tc_tunnel_key){
3527                                         .action = TC_ACT_PIPE,
3528                                         .t_action = TCA_TUNNEL_KEY_ACT_RELEASE,
3529                                         });
3530                         mnl_attr_nest_end(nlh, na_act);
3531                         mnl_attr_nest_end(nlh, na_act_index);
3532                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3533                         break;
3534                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3535                         assert(encap.vxlan);
3536                         flow_tcf_vxlan_encap_parse(actions, encap.vxlan);
3537                         na_act_index =
3538                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3539                         assert(na_act_index);
3540                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3541                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3542                         assert(na_act);
3543                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3544                                 sizeof(struct tc_tunnel_key),
3545                                 &(struct tc_tunnel_key){
3546                                         .action = TC_ACT_PIPE,
3547                                         .t_action = TCA_TUNNEL_KEY_ACT_SET,
3548                                         });
3549                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_UDP_DST)
3550                                 mnl_attr_put_u16(nlh,
3551                                          TCA_TUNNEL_KEY_ENC_DST_PORT,
3552                                          encap.vxlan->udp.dst);
3553                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_SRC)
3554                                 mnl_attr_put_u32(nlh,
3555                                          TCA_TUNNEL_KEY_ENC_IPV4_SRC,
3556                                          encap.vxlan->ipv4.src);
3557                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_DST)
3558                                 mnl_attr_put_u32(nlh,
3559                                          TCA_TUNNEL_KEY_ENC_IPV4_DST,
3560                                          encap.vxlan->ipv4.dst);
3561                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_SRC)
3562                                 mnl_attr_put(nlh,
3563                                          TCA_TUNNEL_KEY_ENC_IPV6_SRC,
3564                                          sizeof(encap.vxlan->ipv6.src),
3565                                          &encap.vxlan->ipv6.src);
3566                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_DST)
3567                                 mnl_attr_put(nlh,
3568                                          TCA_TUNNEL_KEY_ENC_IPV6_DST,
3569                                          sizeof(encap.vxlan->ipv6.dst),
3570                                          &encap.vxlan->ipv6.dst);
3571                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_VXLAN_VNI)
3572                                 mnl_attr_put_u32(nlh,
3573                                          TCA_TUNNEL_KEY_ENC_KEY_ID,
3574                                          vxlan_vni_as_be32
3575                                                 (encap.vxlan->vxlan.vni));
3576                         mnl_attr_put_u8(nlh, TCA_TUNNEL_KEY_NO_CSUM, 0);
3577                         mnl_attr_nest_end(nlh, na_act);
3578                         mnl_attr_nest_end(nlh, na_act_index);
3579                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3580                         break;
3581                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3582                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3583                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3584                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3585                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3586                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3587                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3588                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3589                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3590                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3591                         na_act_index =
3592                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3593                         flow_tcf_create_pedit_mnl_msg(nlh,
3594                                                       &actions, item_flags);
3595                         mnl_attr_nest_end(nlh, na_act_index);
3596                         break;
3597                 default:
3598                         return rte_flow_error_set(error, ENOTSUP,
3599                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3600                                                   actions,
3601                                                   "action not supported");
3602                 }
3603         }
3604         assert(na_flower);
3605         assert(na_flower_act);
3606         mnl_attr_nest_end(nlh, na_flower_act);
3607         dev_flow->tcf.ptc_flags = mnl_attr_get_payload
3608                                         (mnl_nlmsg_get_payload_tail(nlh));
3609         mnl_attr_put_u32(nlh, TCA_FLOWER_FLAGS, decap.vxlan ?
3610                                                 0 : TCA_CLS_FLAGS_SKIP_SW);
3611         mnl_attr_nest_end(nlh, na_flower);
3612         if (dev_flow->tcf.tunnel && dev_flow->tcf.tunnel->ifindex_ptr)
3613                 dev_flow->tcf.tunnel->ifindex_org =
3614                         *dev_flow->tcf.tunnel->ifindex_ptr;
3615         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3616         return 0;
3617 }
3618
3619 /**
3620  * Send Netlink message with acknowledgment.
3621  *
3622  * @param tcf
3623  *   Flow context to use.
3624  * @param nlh
3625  *   Message to send. This function always raises the NLM_F_ACK flag before
3626  *   sending.
3627  * @param[in] cb
3628  *   Callback handler for received message.
3629  * @param[in] arg
3630  *   Context pointer for callback handler.
3631  *
3632  * @return
3633  *   0 on success, a negative errno value otherwise and rte_errno is set.
3634  */
3635 static int
3636 flow_tcf_nl_ack(struct mlx5_flow_tcf_context *tcf,
3637                 struct nlmsghdr *nlh,
3638                 mnl_cb_t cb, void *arg)
3639 {
3640         unsigned int portid = mnl_socket_get_portid(tcf->nl);
3641         uint32_t seq = tcf->seq++;
3642         int ret, err = 0;
3643
3644         assert(tcf->nl);
3645         assert(tcf->buf);
3646         if (!seq) {
3647                 /* seq 0 is reserved for kernel event-driven notifications. */
3648                 seq = tcf->seq++;
3649         }
3650         nlh->nlmsg_seq = seq;
3651         nlh->nlmsg_flags |= NLM_F_ACK;
3652         ret = mnl_socket_sendto(tcf->nl, nlh, nlh->nlmsg_len);
3653         if (ret <= 0) {
3654                 /* Message send error occurres. */
3655                 rte_errno = errno;
3656                 return -rte_errno;
3657         }
3658         nlh = (struct nlmsghdr *)(tcf->buf);
3659         /*
3660          * The following loop postpones non-fatal errors until multipart
3661          * messages are complete.
3662          */
3663         while (true) {
3664                 ret = mnl_socket_recvfrom(tcf->nl, tcf->buf, tcf->buf_size);
3665                 if (ret < 0) {
3666                         err = errno;
3667                         /*
3668                          * In case of overflow Will receive till
3669                          * end of multipart message. We may lost part
3670                          * of reply messages but mark and return an error.
3671                          */
3672                         if (err != ENOSPC ||
3673                             !(nlh->nlmsg_flags & NLM_F_MULTI) ||
3674                             nlh->nlmsg_type == NLMSG_DONE)
3675                                 break;
3676                 } else {
3677                         ret = mnl_cb_run(nlh, ret, seq, portid, cb, arg);
3678                         if (!ret) {
3679                                 /*
3680                                  * libmnl returns 0 if DONE or
3681                                  * success ACK message found.
3682                                  */
3683                                 break;
3684                         }
3685                         if (ret < 0) {
3686                                 /*
3687                                  * ACK message with error found
3688                                  * or some error occurred.
3689                                  */
3690                                 err = errno;
3691                                 break;
3692                         }
3693                         /* We should continue receiving. */
3694                 }
3695         }
3696         if (!err)
3697                 return 0;
3698         rte_errno = err;
3699         return -err;
3700 }
3701
3702 #define MNL_BUF_EXTRA_SPACE 16
3703 #define MNL_REQUEST_SIZE_MIN 256
3704 #define MNL_REQUEST_SIZE_MAX 2048
3705 #define MNL_REQUEST_SIZE RTE_MIN(RTE_MAX(sysconf(_SC_PAGESIZE), \
3706                                  MNL_REQUEST_SIZE_MIN), MNL_REQUEST_SIZE_MAX)
3707
3708 /* Data structures used by flow_tcf_xxx_cb() routines. */
3709 struct tcf_nlcb_buf {
3710         LIST_ENTRY(tcf_nlcb_buf) next;
3711         uint32_t size;
3712         alignas(struct nlmsghdr)
3713         uint8_t msg[]; /**< Netlink message data. */
3714 };
3715
3716 struct tcf_nlcb_context {
3717         unsigned int ifindex; /**< Base interface index. */
3718         uint32_t bufsize;
3719         LIST_HEAD(, tcf_nlcb_buf) nlbuf;
3720 };
3721
3722 /**
3723  * Allocate space for netlink command in buffer list
3724  *
3725  * @param[in, out] ctx
3726  *   Pointer to callback context with command buffers list.
3727  * @param[in] size
3728  *   Required size of data buffer to be allocated.
3729  *
3730  * @return
3731  *   Pointer to allocated memory, aligned as message header.
3732  *   NULL if some error occurred.
3733  */
3734 static struct nlmsghdr *
3735 flow_tcf_alloc_nlcmd(struct tcf_nlcb_context *ctx, uint32_t size)
3736 {
3737         struct tcf_nlcb_buf *buf;
3738         struct nlmsghdr *nlh;
3739
3740         size = NLMSG_ALIGN(size);
3741         buf = LIST_FIRST(&ctx->nlbuf);
3742         if (buf && (buf->size + size) <= ctx->bufsize) {
3743                 nlh = (struct nlmsghdr *)&buf->msg[buf->size];
3744                 buf->size += size;
3745                 return nlh;
3746         }
3747         if (size > ctx->bufsize) {
3748                 DRV_LOG(WARNING, "netlink: too long command buffer requested");
3749                 return NULL;
3750         }
3751         buf = rte_malloc(__func__,
3752                         ctx->bufsize + sizeof(struct tcf_nlcb_buf),
3753                         alignof(struct tcf_nlcb_buf));
3754         if (!buf) {
3755                 DRV_LOG(WARNING, "netlink: no memory for command buffer");
3756                 return NULL;
3757         }
3758         LIST_INSERT_HEAD(&ctx->nlbuf, buf, next);
3759         buf->size = size;
3760         nlh = (struct nlmsghdr *)&buf->msg[0];
3761         return nlh;
3762 }
3763
3764 /**
3765  * Send the buffers with prepared netlink commands. Scans the list and
3766  * sends all found buffers. Buffers are sent and freed anyway in order
3767  * to prevent memory leakage if some every message in received packet.
3768  *
3769  * @param[in] tcf
3770  *   Context object initialized by mlx5_flow_tcf_context_create().
3771  * @param[in, out] ctx
3772  *   Pointer to callback context with command buffers list.
3773  *
3774  * @return
3775  *   Zero value on success, negative errno value otherwise
3776  *   and rte_errno is set.
3777  */
3778 static int
3779 flow_tcf_send_nlcmd(struct mlx5_flow_tcf_context *tcf,
3780                     struct tcf_nlcb_context *ctx)
3781 {
3782         struct tcf_nlcb_buf *bc = LIST_FIRST(&ctx->nlbuf);
3783         int ret = 0;
3784
3785         while (bc) {
3786                 struct tcf_nlcb_buf *bn = LIST_NEXT(bc, next);
3787                 struct nlmsghdr *nlh;
3788                 uint32_t msg = 0;
3789                 int rc;
3790
3791                 while (msg < bc->size) {
3792                         /*
3793                          * Send Netlink commands from buffer in one by one
3794                          * fashion. If we send multiple rule deletion commands
3795                          * in one Netlink message and some error occurs it may
3796                          * cause multiple ACK error messages and break sequence
3797                          * numbers of Netlink communication, because we expect
3798                          * the only one ACK reply.
3799                          */
3800                         assert((bc->size - msg) >= sizeof(struct nlmsghdr));
3801                         nlh = (struct nlmsghdr *)&bc->msg[msg];
3802                         assert((bc->size - msg) >= nlh->nlmsg_len);
3803                         msg += nlh->nlmsg_len;
3804                         rc = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
3805                         if (rc) {
3806                                 DRV_LOG(WARNING,
3807                                         "netlink: cleanup error %d", rc);
3808                                 if (!ret)
3809                                         ret = rc;
3810                         }
3811                 }
3812                 rte_free(bc);
3813                 bc = bn;
3814         }
3815         LIST_INIT(&ctx->nlbuf);
3816         return ret;
3817 }
3818
3819 /**
3820  * Collect local IP address rules with scope link attribute  on specified
3821  * network device. This is callback routine called by libmnl mnl_cb_run()
3822  * in loop for every message in received packet.
3823  *
3824  * @param[in] nlh
3825  *   Pointer to reply header.
3826  * @param[in, out] arg
3827  *   Opaque data pointer for this callback.
3828  *
3829  * @return
3830  *   A positive, nonzero value on success, negative errno value otherwise
3831  *   and rte_errno is set.
3832  */
3833 static int
3834 flow_tcf_collect_local_cb(const struct nlmsghdr *nlh, void *arg)
3835 {
3836         struct tcf_nlcb_context *ctx = arg;
3837         struct nlmsghdr *cmd;
3838         struct ifaddrmsg *ifa;
3839         struct nlattr *na;
3840         struct nlattr *na_local = NULL;
3841         struct nlattr *na_peer = NULL;
3842         unsigned char family;
3843         uint32_t size;
3844
3845         if (nlh->nlmsg_type != RTM_NEWADDR) {
3846                 rte_errno = EINVAL;
3847                 return -rte_errno;
3848         }
3849         ifa = mnl_nlmsg_get_payload(nlh);
3850         family = ifa->ifa_family;
3851         if (ifa->ifa_index != ctx->ifindex ||
3852             ifa->ifa_scope != RT_SCOPE_LINK ||
3853             !(ifa->ifa_flags & IFA_F_PERMANENT) ||
3854             (family != AF_INET && family != AF_INET6))
3855                 return 1;
3856         mnl_attr_for_each(na, nlh, sizeof(*ifa)) {
3857                 switch (mnl_attr_get_type(na)) {
3858                 case IFA_LOCAL:
3859                         na_local = na;
3860                         break;
3861                 case IFA_ADDRESS:
3862                         na_peer = na;
3863                         break;
3864                 }
3865                 if (na_local && na_peer)
3866                         break;
3867         }
3868         if (!na_local || !na_peer)
3869                 return 1;
3870         /* Local rule found with scope link, permanent and assigned peer. */
3871         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
3872                MNL_ALIGN(sizeof(struct ifaddrmsg)) +
3873                (family == AF_INET6 ? 2 * SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
3874                                    : 2 * SZ_NLATTR_TYPE_OF(uint32_t));
3875         cmd = flow_tcf_alloc_nlcmd(ctx, size);
3876         if (!cmd) {
3877                 rte_errno = ENOMEM;
3878                 return -rte_errno;
3879         }
3880         cmd = mnl_nlmsg_put_header(cmd);
3881         cmd->nlmsg_type = RTM_DELADDR;
3882         cmd->nlmsg_flags = NLM_F_REQUEST;
3883         ifa = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifa));
3884         ifa->ifa_flags = IFA_F_PERMANENT;
3885         ifa->ifa_scope = RT_SCOPE_LINK;
3886         ifa->ifa_index = ctx->ifindex;
3887         if (family == AF_INET) {
3888                 ifa->ifa_family = AF_INET;
3889                 ifa->ifa_prefixlen = 32;
3890                 mnl_attr_put_u32(cmd, IFA_LOCAL, mnl_attr_get_u32(na_local));
3891                 mnl_attr_put_u32(cmd, IFA_ADDRESS, mnl_attr_get_u32(na_peer));
3892         } else {
3893                 ifa->ifa_family = AF_INET6;
3894                 ifa->ifa_prefixlen = 128;
3895                 mnl_attr_put(cmd, IFA_LOCAL, IPV6_ADDR_LEN,
3896                         mnl_attr_get_payload(na_local));
3897                 mnl_attr_put(cmd, IFA_ADDRESS, IPV6_ADDR_LEN,
3898                         mnl_attr_get_payload(na_peer));
3899         }
3900         assert(size == cmd->nlmsg_len);
3901         return 1;
3902 }
3903
3904 /**
3905  * Cleanup the local IP addresses on outer interface.
3906  *
3907  * @param[in] tcf
3908  *   Context object initialized by mlx5_flow_tcf_context_create().
3909  * @param[in] ifindex
3910  *   Network inferface index to perform cleanup.
3911  */
3912 static void
3913 flow_tcf_encap_local_cleanup(struct mlx5_flow_tcf_context *tcf,
3914                             unsigned int ifindex)
3915 {
3916         struct nlmsghdr *nlh;
3917         struct ifaddrmsg *ifa;
3918         struct tcf_nlcb_context ctx = {
3919                 .ifindex = ifindex,
3920                 .bufsize = MNL_REQUEST_SIZE,
3921                 .nlbuf = LIST_HEAD_INITIALIZER(),
3922         };
3923         int ret;
3924
3925         assert(ifindex);
3926         /*
3927          * Seek and destroy leftovers of local IP addresses with
3928          * matching properties "scope link".
3929          */
3930         nlh = mnl_nlmsg_put_header(tcf->buf);
3931         nlh->nlmsg_type = RTM_GETADDR;
3932         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
3933         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
3934         ifa->ifa_family = AF_UNSPEC;
3935         ifa->ifa_index = ifindex;
3936         ifa->ifa_scope = RT_SCOPE_LINK;
3937         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_local_cb, &ctx);
3938         if (ret)
3939                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
3940         ret = flow_tcf_send_nlcmd(tcf, &ctx);
3941         if (ret)
3942                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
3943 }
3944
3945 /**
3946  * Collect neigh permament rules on specified network device.
3947  * This is callback routine called by libmnl mnl_cb_run() in loop for
3948  * every message in received packet.
3949  *
3950  * @param[in] nlh
3951  *   Pointer to reply header.
3952  * @param[in, out] arg
3953  *   Opaque data pointer for this callback.
3954  *
3955  * @return
3956  *   A positive, nonzero value on success, negative errno value otherwise
3957  *   and rte_errno is set.
3958  */
3959 static int
3960 flow_tcf_collect_neigh_cb(const struct nlmsghdr *nlh, void *arg)
3961 {
3962         struct tcf_nlcb_context *ctx = arg;
3963         struct nlmsghdr *cmd;
3964         struct ndmsg *ndm;
3965         struct nlattr *na;
3966         struct nlattr *na_ip = NULL;
3967         struct nlattr *na_mac = NULL;
3968         unsigned char family;
3969         uint32_t size;
3970
3971         if (nlh->nlmsg_type != RTM_NEWNEIGH) {
3972                 rte_errno = EINVAL;
3973                 return -rte_errno;
3974         }
3975         ndm = mnl_nlmsg_get_payload(nlh);
3976         family = ndm->ndm_family;
3977         if (ndm->ndm_ifindex != (int)ctx->ifindex ||
3978            !(ndm->ndm_state & NUD_PERMANENT) ||
3979            (family != AF_INET && family != AF_INET6))
3980                 return 1;
3981         mnl_attr_for_each(na, nlh, sizeof(*ndm)) {
3982                 switch (mnl_attr_get_type(na)) {
3983                 case NDA_DST:
3984                         na_ip = na;
3985                         break;
3986                 case NDA_LLADDR:
3987                         na_mac = na;
3988                         break;
3989                 }
3990                 if (na_mac && na_ip)
3991                         break;
3992         }
3993         if (!na_mac || !na_ip)
3994                 return 1;
3995         /* Neigh rule with permenent attribute found. */
3996         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
3997                MNL_ALIGN(sizeof(struct ndmsg)) +
3998                SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) +
3999                (family == AF_INET6 ? SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
4000                                    : SZ_NLATTR_TYPE_OF(uint32_t));
4001         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4002         if (!cmd) {
4003                 rte_errno = ENOMEM;
4004                 return -rte_errno;
4005         }
4006         cmd = mnl_nlmsg_put_header(cmd);
4007         cmd->nlmsg_type = RTM_DELNEIGH;
4008         cmd->nlmsg_flags = NLM_F_REQUEST;
4009         ndm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ndm));
4010         ndm->ndm_ifindex = ctx->ifindex;
4011         ndm->ndm_state = NUD_PERMANENT;
4012         ndm->ndm_flags = 0;
4013         ndm->ndm_type = 0;
4014         if (family == AF_INET) {
4015                 ndm->ndm_family = AF_INET;
4016                 mnl_attr_put_u32(cmd, NDA_DST, mnl_attr_get_u32(na_ip));
4017         } else {
4018                 ndm->ndm_family = AF_INET6;
4019                 mnl_attr_put(cmd, NDA_DST, IPV6_ADDR_LEN,
4020                              mnl_attr_get_payload(na_ip));
4021         }
4022         mnl_attr_put(cmd, NDA_LLADDR, ETHER_ADDR_LEN,
4023                      mnl_attr_get_payload(na_mac));
4024         assert(size == cmd->nlmsg_len);
4025         return 1;
4026 }
4027
4028 /**
4029  * Cleanup the neigh rules on outer interface.
4030  *
4031  * @param[in] tcf
4032  *   Context object initialized by mlx5_flow_tcf_context_create().
4033  * @param[in] ifindex
4034  *   Network inferface index to perform cleanup.
4035  */
4036 static void
4037 flow_tcf_encap_neigh_cleanup(struct mlx5_flow_tcf_context *tcf,
4038                             unsigned int ifindex)
4039 {
4040         struct nlmsghdr *nlh;
4041         struct ndmsg *ndm;
4042         struct tcf_nlcb_context ctx = {
4043                 .ifindex = ifindex,
4044                 .bufsize = MNL_REQUEST_SIZE,
4045                 .nlbuf = LIST_HEAD_INITIALIZER(),
4046         };
4047         int ret;
4048
4049         assert(ifindex);
4050         /* Seek and destroy leftovers of neigh rules. */
4051         nlh = mnl_nlmsg_put_header(tcf->buf);
4052         nlh->nlmsg_type = RTM_GETNEIGH;
4053         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4054         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4055         ndm->ndm_family = AF_UNSPEC;
4056         ndm->ndm_ifindex = ifindex;
4057         ndm->ndm_state = NUD_PERMANENT;
4058         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_neigh_cb, &ctx);
4059         if (ret)
4060                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4061         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4062         if (ret)
4063                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4064 }
4065
4066 /**
4067  * Collect indices of VXLAN encap/decap interfaces associated with device.
4068  * This is callback routine called by libmnl mnl_cb_run() in loop for
4069  * every message in received packet.
4070  *
4071  * @param[in] nlh
4072  *   Pointer to reply header.
4073  * @param[in, out] arg
4074  *   Opaque data pointer for this callback.
4075  *
4076  * @return
4077  *   A positive, nonzero value on success, negative errno value otherwise
4078  *   and rte_errno is set.
4079  */
4080 static int
4081 flow_tcf_collect_vxlan_cb(const struct nlmsghdr *nlh, void *arg)
4082 {
4083         struct tcf_nlcb_context *ctx = arg;
4084         struct nlmsghdr *cmd;
4085         struct ifinfomsg *ifm;
4086         struct nlattr *na;
4087         struct nlattr *na_info = NULL;
4088         struct nlattr *na_vxlan = NULL;
4089         bool found = false;
4090         unsigned int vxindex;
4091         uint32_t size;
4092
4093         if (nlh->nlmsg_type != RTM_NEWLINK) {
4094                 rte_errno = EINVAL;
4095                 return -rte_errno;
4096         }
4097         ifm = mnl_nlmsg_get_payload(nlh);
4098         if (!ifm->ifi_index) {
4099                 rte_errno = EINVAL;
4100                 return -rte_errno;
4101         }
4102         mnl_attr_for_each(na, nlh, sizeof(*ifm))
4103                 if (mnl_attr_get_type(na) == IFLA_LINKINFO) {
4104                         na_info = na;
4105                         break;
4106                 }
4107         if (!na_info)
4108                 return 1;
4109         mnl_attr_for_each_nested(na, na_info) {
4110                 switch (mnl_attr_get_type(na)) {
4111                 case IFLA_INFO_KIND:
4112                         if (!strncmp("vxlan", mnl_attr_get_str(na),
4113                                      mnl_attr_get_len(na)))
4114                                 found = true;
4115                         break;
4116                 case IFLA_INFO_DATA:
4117                         na_vxlan = na;
4118                         break;
4119                 }
4120                 if (found && na_vxlan)
4121                         break;
4122         }
4123         if (!found || !na_vxlan)
4124                 return 1;
4125         found = false;
4126         mnl_attr_for_each_nested(na, na_vxlan) {
4127                 if (mnl_attr_get_type(na) == IFLA_VXLAN_LINK &&
4128                     mnl_attr_get_u32(na) == ctx->ifindex) {
4129                         found = true;
4130                         break;
4131                 }
4132         }
4133         if (!found)
4134                 return 1;
4135         /* Attached VXLAN device found, store the command to delete. */
4136         vxindex = ifm->ifi_index;
4137         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4138                MNL_ALIGN(sizeof(struct ifinfomsg));
4139         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4140         if (!cmd) {
4141                 rte_errno = ENOMEM;
4142                 return -rte_errno;
4143         }
4144         cmd = mnl_nlmsg_put_header(cmd);
4145         cmd->nlmsg_type = RTM_DELLINK;
4146         cmd->nlmsg_flags = NLM_F_REQUEST;
4147         ifm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifm));
4148         ifm->ifi_family = AF_UNSPEC;
4149         ifm->ifi_index = vxindex;
4150         assert(size == cmd->nlmsg_len);
4151         return 1;
4152 }
4153
4154 /**
4155  * Cleanup the outer interface. Removes all found vxlan devices
4156  * attached to specified index, flushes the meigh and local IP
4157  * datavase.
4158  *
4159  * @param[in] tcf
4160  *   Context object initialized by mlx5_flow_tcf_context_create().
4161  * @param[in] ifindex
4162  *   Network inferface index to perform cleanup.
4163  */
4164 static void
4165 flow_tcf_encap_iface_cleanup(struct mlx5_flow_tcf_context *tcf,
4166                             unsigned int ifindex)
4167 {
4168         struct nlmsghdr *nlh;
4169         struct ifinfomsg *ifm;
4170         struct tcf_nlcb_context ctx = {
4171                 .ifindex = ifindex,
4172                 .bufsize = MNL_REQUEST_SIZE,
4173                 .nlbuf = LIST_HEAD_INITIALIZER(),
4174         };
4175         int ret;
4176
4177         assert(ifindex);
4178         /*
4179          * Seek and destroy leftover VXLAN encap/decap interfaces with
4180          * matching properties.
4181          */
4182         nlh = mnl_nlmsg_put_header(tcf->buf);
4183         nlh->nlmsg_type = RTM_GETLINK;
4184         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4185         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4186         ifm->ifi_family = AF_UNSPEC;
4187         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_vxlan_cb, &ctx);
4188         if (ret)
4189                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4190         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4191         if (ret)
4192                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4193 }
4194
4195 /**
4196  * Emit Netlink message to add/remove local address to the outer device.
4197  * The address being added is visible within the link only (scope link).
4198  *
4199  * Note that an implicit route is maintained by the kernel due to the
4200  * presence of a peer address (IFA_ADDRESS).
4201  *
4202  * These rules are used for encapsultion only and allow to assign
4203  * the outer tunnel source IP address.
4204  *
4205  * @param[in] tcf
4206  *   Libmnl socket context object.
4207  * @param[in] encap
4208  *   Encapsulation properties (source address and its peer).
4209  * @param[in] ifindex
4210  *   Network interface to apply rule.
4211  * @param[in] enable
4212  *   Toggle between add and remove.
4213  * @param[out] error
4214  *   Perform verbose error reporting if not NULL.
4215  *
4216  * @return
4217  *   0 on success, a negative errno value otherwise and rte_errno is set.
4218  */
4219 static int
4220 flow_tcf_rule_local(struct mlx5_flow_tcf_context *tcf,
4221                     const struct flow_tcf_vxlan_encap *encap,
4222                     unsigned int ifindex,
4223                     bool enable,
4224                     struct rte_flow_error *error)
4225 {
4226         struct nlmsghdr *nlh;
4227         struct ifaddrmsg *ifa;
4228         alignas(struct nlmsghdr)
4229         uint8_t buf[mnl_nlmsg_size(sizeof(*ifa) + 128)];
4230
4231         nlh = mnl_nlmsg_put_header(buf);
4232         nlh->nlmsg_type = enable ? RTM_NEWADDR : RTM_DELADDR;
4233         nlh->nlmsg_flags =
4234                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4235         nlh->nlmsg_seq = 0;
4236         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4237         ifa->ifa_flags = IFA_F_PERMANENT;
4238         ifa->ifa_scope = RT_SCOPE_LINK;
4239         ifa->ifa_index = ifindex;
4240         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4241                 ifa->ifa_family = AF_INET;
4242                 ifa->ifa_prefixlen = 32;
4243                 mnl_attr_put_u32(nlh, IFA_LOCAL, encap->ipv4.src);
4244                 if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST)
4245                         mnl_attr_put_u32(nlh, IFA_ADDRESS,
4246                                               encap->ipv4.dst);
4247         } else {
4248                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4249                 ifa->ifa_family = AF_INET6;
4250                 ifa->ifa_prefixlen = 128;
4251                 mnl_attr_put(nlh, IFA_LOCAL,
4252                                   sizeof(encap->ipv6.src),
4253                                   &encap->ipv6.src);
4254                 if (encap->mask & FLOW_TCF_ENCAP_IPV6_DST)
4255                         mnl_attr_put(nlh, IFA_ADDRESS,
4256                                           sizeof(encap->ipv6.dst),
4257                                           &encap->ipv6.dst);
4258         }
4259         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4260                 return 0;
4261         return rte_flow_error_set(error, rte_errno,
4262                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4263                                   "netlink: cannot complete IFA request"
4264                                   " (ip addr add)");
4265 }
4266
4267 /**
4268  * Emit Netlink message to add/remove neighbor.
4269  *
4270  * @param[in] tcf
4271  *   Libmnl socket context object.
4272  * @param[in] encap
4273  *   Encapsulation properties (destination address).
4274  * @param[in] ifindex
4275  *   Network interface.
4276  * @param[in] enable
4277  *   Toggle between add and remove.
4278  * @param[out] error
4279  *   Perform verbose error reporting if not NULL.
4280  *
4281  * @return
4282  *   0 on success, a negative errno value otherwise and rte_errno is set.
4283  */
4284 static int
4285 flow_tcf_rule_neigh(struct mlx5_flow_tcf_context *tcf,
4286                      const struct flow_tcf_vxlan_encap *encap,
4287                      unsigned int ifindex,
4288                      bool enable,
4289                      struct rte_flow_error *error)
4290 {
4291         struct nlmsghdr *nlh;
4292         struct ndmsg *ndm;
4293         alignas(struct nlmsghdr)
4294         uint8_t buf[mnl_nlmsg_size(sizeof(*ndm) + 128)];
4295
4296         nlh = mnl_nlmsg_put_header(buf);
4297         nlh->nlmsg_type = enable ? RTM_NEWNEIGH : RTM_DELNEIGH;
4298         nlh->nlmsg_flags =
4299                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4300         nlh->nlmsg_seq = 0;
4301         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4302         ndm->ndm_ifindex = ifindex;
4303         ndm->ndm_state = NUD_PERMANENT;
4304         ndm->ndm_flags = 0;
4305         ndm->ndm_type = 0;
4306         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4307                 ndm->ndm_family = AF_INET;
4308                 mnl_attr_put_u32(nlh, NDA_DST, encap->ipv4.dst);
4309         } else {
4310                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4311                 ndm->ndm_family = AF_INET6;
4312                 mnl_attr_put(nlh, NDA_DST, sizeof(encap->ipv6.dst),
4313                                                  &encap->ipv6.dst);
4314         }
4315         if (encap->mask & FLOW_TCF_ENCAP_ETH_SRC && enable)
4316                 DRV_LOG(WARNING,
4317                         "outer ethernet source address cannot be "
4318                         "forced for VXLAN encapsulation");
4319         if (encap->mask & FLOW_TCF_ENCAP_ETH_DST)
4320                 mnl_attr_put(nlh, NDA_LLADDR, sizeof(encap->eth.dst),
4321                                                     &encap->eth.dst);
4322         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4323                 return 0;
4324         return rte_flow_error_set(error, rte_errno,
4325                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4326                                   "netlink: cannot complete ND request"
4327                                   " (ip neigh)");
4328 }
4329
4330 /**
4331  * Manage the local IP addresses and their peers IP addresses on the
4332  * outer interface for encapsulation purposes. The kernel searches the
4333  * appropriate device for tunnel egress traffic using the outer source
4334  * IP, this IP should be assigned to the outer network device, otherwise
4335  * kernel rejects the rule.
4336  *
4337  * Adds or removes the addresses using the Netlink command like this:
4338  *   ip addr add <src_ip> peer <dst_ip> scope link dev <ifouter>
4339  *
4340  * The addresses are local to the netdev ("scope link"), this reduces
4341  * the risk of conflicts. Note that an implicit route is maintained by
4342  * the kernel due to the presence of a peer address (IFA_ADDRESS).
4343  *
4344  * @param[in] tcf
4345  *   Libmnl socket context object.
4346  * @param[in] vtep
4347  *   VTEP object, contains rule database and ifouter index.
4348  * @param[in] dev_flow
4349  *   Flow object, contains the tunnel parameters (for encap only).
4350  * @param[in] enable
4351  *   Toggle between add and remove.
4352  * @param[out] error
4353  *   Perform verbose error reporting if not NULL.
4354  *
4355  * @return
4356  *   0 on success, a negative errno value otherwise and rte_errno is set.
4357  */
4358 static int
4359 flow_tcf_encap_local(struct mlx5_flow_tcf_context *tcf,
4360                      struct tcf_vtep *vtep,
4361                      struct mlx5_flow *dev_flow,
4362                      bool enable,
4363                      struct rte_flow_error *error)
4364 {
4365         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4366         struct tcf_local_rule *rule;
4367         bool found = false;
4368         int ret;
4369
4370         assert(encap);
4371         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4372         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4373                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_DST);
4374                 LIST_FOREACH(rule, &vtep->local, next) {
4375                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_SRC &&
4376                             encap->ipv4.src == rule->ipv4.src &&
4377                             encap->ipv4.dst == rule->ipv4.dst) {
4378                                 found = true;
4379                                 break;
4380                         }
4381                 }
4382         } else {
4383                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4384                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4385                 LIST_FOREACH(rule, &vtep->local, next) {
4386                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_SRC &&
4387                             !memcmp(&encap->ipv6.src, &rule->ipv6.src,
4388                                             sizeof(encap->ipv6.src)) &&
4389                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4390                                             sizeof(encap->ipv6.dst))) {
4391                                 found = true;
4392                                 break;
4393                         }
4394                 }
4395         }
4396         if (found) {
4397                 if (enable) {
4398                         rule->refcnt++;
4399                         return 0;
4400                 }
4401                 if (!rule->refcnt || !--rule->refcnt) {
4402                         LIST_REMOVE(rule, next);
4403                         return flow_tcf_rule_local(tcf, encap,
4404                                         vtep->ifouter, false, error);
4405                 }
4406                 return 0;
4407         }
4408         if (!enable) {
4409                 DRV_LOG(WARNING, "disabling not existing local rule");
4410                 rte_flow_error_set(error, ENOENT,
4411                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4412                                    "disabling not existing local rule");
4413                 return -ENOENT;
4414         }
4415         rule = rte_zmalloc(__func__, sizeof(struct tcf_local_rule),
4416                                 alignof(struct tcf_local_rule));
4417         if (!rule) {
4418                 rte_flow_error_set(error, ENOMEM,
4419                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4420                                    "unable to allocate memory for local rule");
4421                 return -rte_errno;
4422         }
4423         *rule = (struct tcf_local_rule){.refcnt = 0,
4424                                         .mask = 0,
4425                                         };
4426         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4427                 rule->mask = FLOW_TCF_ENCAP_IPV4_SRC
4428                            | FLOW_TCF_ENCAP_IPV4_DST;
4429                 rule->ipv4.src = encap->ipv4.src;
4430                 rule->ipv4.dst = encap->ipv4.dst;
4431         } else {
4432                 rule->mask = FLOW_TCF_ENCAP_IPV6_SRC
4433                            | FLOW_TCF_ENCAP_IPV6_DST;
4434                 memcpy(&rule->ipv6.src, &encap->ipv6.src, IPV6_ADDR_LEN);
4435                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4436         }
4437         ret = flow_tcf_rule_local(tcf, encap, vtep->ifouter, true, error);
4438         if (ret) {
4439                 rte_free(rule);
4440                 return ret;
4441         }
4442         rule->refcnt++;
4443         LIST_INSERT_HEAD(&vtep->local, rule, next);
4444         return 0;
4445 }
4446
4447 /**
4448  * Manage the destination MAC/IP addresses neigh database, kernel uses
4449  * this one to determine the destination MAC address within encapsulation
4450  * header. Adds or removes the entries using the Netlink command like this:
4451  *   ip neigh add dev <ifouter> lladdr <dst_mac> to <dst_ip> nud permanent
4452  *
4453  * @param[in] tcf
4454  *   Libmnl socket context object.
4455  * @param[in] vtep
4456  *   VTEP object, contains rule database and ifouter index.
4457  * @param[in] dev_flow
4458  *   Flow object, contains the tunnel parameters (for encap only).
4459  * @param[in] enable
4460  *   Toggle between add and remove.
4461  * @param[out] error
4462  *   Perform verbose error reporting if not NULL.
4463  *
4464  * @return
4465  *   0 on success, a negative errno value otherwise and rte_errno is set.
4466  */
4467 static int
4468 flow_tcf_encap_neigh(struct mlx5_flow_tcf_context *tcf,
4469                      struct tcf_vtep *vtep,
4470                      struct mlx5_flow *dev_flow,
4471                      bool enable,
4472                      struct rte_flow_error *error)
4473 {
4474         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4475         struct tcf_neigh_rule *rule;
4476         bool found = false;
4477         int ret;
4478
4479         assert(encap);
4480         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4481         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4482                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_SRC);
4483                 LIST_FOREACH(rule, &vtep->neigh, next) {
4484                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_DST &&
4485                             encap->ipv4.dst == rule->ipv4.dst) {
4486                                 found = true;
4487                                 break;
4488                         }
4489                 }
4490         } else {
4491                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4492                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4493                 LIST_FOREACH(rule, &vtep->neigh, next) {
4494                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_DST &&
4495                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4496                                                 sizeof(encap->ipv6.dst))) {
4497                                 found = true;
4498                                 break;
4499                         }
4500                 }
4501         }
4502         if (found) {
4503                 if (memcmp(&encap->eth.dst, &rule->eth,
4504                            sizeof(encap->eth.dst))) {
4505                         DRV_LOG(WARNING, "Destination MAC differs"
4506                                          " in neigh rule");
4507                         rte_flow_error_set(error, EEXIST,
4508                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4509                                            NULL, "Different MAC address"
4510                                            " neigh rule for the same"
4511                                            " destination IP");
4512                                         return -EEXIST;
4513                 }
4514                 if (enable) {
4515                         rule->refcnt++;
4516                         return 0;
4517                 }
4518                 if (!rule->refcnt || !--rule->refcnt) {
4519                         LIST_REMOVE(rule, next);
4520                         return flow_tcf_rule_neigh(tcf, encap,
4521                                                    vtep->ifouter,
4522                                                    false, error);
4523                 }
4524                 return 0;
4525         }
4526         if (!enable) {
4527                 DRV_LOG(WARNING, "Disabling not existing neigh rule");
4528                 rte_flow_error_set(error, ENOENT,
4529                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4530                                    "unable to allocate memory for neigh rule");
4531                 return -ENOENT;
4532         }
4533         rule = rte_zmalloc(__func__, sizeof(struct tcf_neigh_rule),
4534                                 alignof(struct tcf_neigh_rule));
4535         if (!rule) {
4536                 rte_flow_error_set(error, ENOMEM,
4537                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4538                                    "unable to allocate memory for neigh rule");
4539                 return -rte_errno;
4540         }
4541         *rule = (struct tcf_neigh_rule){.refcnt = 0,
4542                                         .mask = 0,
4543                                         };
4544         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4545                 rule->mask = FLOW_TCF_ENCAP_IPV4_DST;
4546                 rule->ipv4.dst = encap->ipv4.dst;
4547         } else {
4548                 rule->mask = FLOW_TCF_ENCAP_IPV6_DST;
4549                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4550         }
4551         memcpy(&rule->eth, &encap->eth.dst, sizeof(rule->eth));
4552         ret = flow_tcf_rule_neigh(tcf, encap, vtep->ifouter, true, error);
4553         if (ret) {
4554                 rte_free(rule);
4555                 return ret;
4556         }
4557         rule->refcnt++;
4558         LIST_INSERT_HEAD(&vtep->neigh, rule, next);
4559         return 0;
4560 }
4561
4562 /* VTEP device list is shared between PMD port instances. */
4563 static LIST_HEAD(, tcf_vtep) vtep_list_vxlan = LIST_HEAD_INITIALIZER();
4564 static pthread_mutex_t vtep_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4565
4566 /**
4567  * Deletes VTEP network device.
4568  *
4569  * @param[in] tcf
4570  *   Context object initialized by mlx5_flow_tcf_context_create().
4571  * @param[in] vtep
4572  *   Object represinting the network device to delete. Memory
4573  *   allocated for this object is freed by routine.
4574  */
4575 static void
4576 flow_tcf_vtep_delete(struct mlx5_flow_tcf_context *tcf,
4577                      struct tcf_vtep *vtep)
4578 {
4579         struct nlmsghdr *nlh;
4580         struct ifinfomsg *ifm;
4581         alignas(struct nlmsghdr)
4582         uint8_t buf[mnl_nlmsg_size(MNL_ALIGN(sizeof(*ifm))) +
4583                     MNL_BUF_EXTRA_SPACE];
4584         int ret;
4585
4586         assert(!vtep->refcnt);
4587         /* Delete only ifaces those we actually created. */
4588         if (vtep->created && vtep->ifindex) {
4589                 DRV_LOG(INFO, "VTEP delete (%d)", vtep->ifindex);
4590                 nlh = mnl_nlmsg_put_header(buf);
4591                 nlh->nlmsg_type = RTM_DELLINK;
4592                 nlh->nlmsg_flags = NLM_F_REQUEST;
4593                 ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4594                 ifm->ifi_family = AF_UNSPEC;
4595                 ifm->ifi_index = vtep->ifindex;
4596                 assert(sizeof(buf) >= nlh->nlmsg_len);
4597                 ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4598                 if (ret)
4599                         DRV_LOG(WARNING, "netlink: error deleting vxlan"
4600                                          " encap/decap ifindex %u",
4601                                          ifm->ifi_index);
4602         }
4603         rte_free(vtep);
4604 }
4605
4606 /**
4607  * Creates VTEP network device.
4608  *
4609  * @param[in] tcf
4610  *   Context object initialized by mlx5_flow_tcf_context_create().
4611  * @param[in] ifouter
4612  *   Outer interface to attach new-created VXLAN device
4613  *   If zero the VXLAN device will not be attached to any device.
4614  *   These VTEPs are used for decapsulation and can be precreated
4615  *   and shared between processes.
4616  * @param[in] port
4617  *   UDP port of created VTEP device.
4618  * @param[out] error
4619  *   Perform verbose error reporting if not NULL.
4620  *
4621  * @return
4622  * Pointer to created device structure on success,
4623  * NULL otherwise and rte_errno is set.
4624  */
4625 #ifdef HAVE_IFLA_VXLAN_COLLECT_METADATA
4626 static struct tcf_vtep*
4627 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf,
4628                      unsigned int ifouter,
4629                      uint16_t port, struct rte_flow_error *error)
4630 {
4631         struct tcf_vtep *vtep;
4632         struct nlmsghdr *nlh;
4633         struct ifinfomsg *ifm;
4634         char name[sizeof(MLX5_VXLAN_DEVICE_PFX) + 24];
4635         alignas(struct nlmsghdr)
4636         uint8_t buf[mnl_nlmsg_size(sizeof(*ifm)) +
4637                     SZ_NLATTR_DATA_OF(sizeof(name)) +
4638                     SZ_NLATTR_NEST * 2 +
4639                     SZ_NLATTR_STRZ_OF("vxlan") +
4640                     SZ_NLATTR_DATA_OF(sizeof(uint32_t)) +
4641                     SZ_NLATTR_DATA_OF(sizeof(uint16_t)) +
4642                     SZ_NLATTR_DATA_OF(sizeof(uint8_t)) * 3 +
4643                     MNL_BUF_EXTRA_SPACE];
4644         struct nlattr *na_info;
4645         struct nlattr *na_vxlan;
4646         rte_be16_t vxlan_port = rte_cpu_to_be_16(port);
4647         int ret;
4648
4649         vtep = rte_zmalloc(__func__, sizeof(*vtep), alignof(struct tcf_vtep));
4650         if (!vtep) {
4651                 rte_flow_error_set(error, ENOMEM,
4652                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4653                                    "unable to allocate memory for VTEP");
4654                 return NULL;
4655         }
4656         *vtep = (struct tcf_vtep){
4657                         .port = port,
4658                         .local = LIST_HEAD_INITIALIZER(),
4659                         .neigh = LIST_HEAD_INITIALIZER(),
4660         };
4661         memset(buf, 0, sizeof(buf));
4662         nlh = mnl_nlmsg_put_header(buf);
4663         nlh->nlmsg_type = RTM_NEWLINK;
4664         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE  | NLM_F_EXCL;
4665         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4666         ifm->ifi_family = AF_UNSPEC;
4667         ifm->ifi_type = 0;
4668         ifm->ifi_index = 0;
4669         ifm->ifi_flags = IFF_UP;
4670         ifm->ifi_change = 0xffffffff;
4671         snprintf(name, sizeof(name), "%s%u", MLX5_VXLAN_DEVICE_PFX, port);
4672         mnl_attr_put_strz(nlh, IFLA_IFNAME, name);
4673         na_info = mnl_attr_nest_start(nlh, IFLA_LINKINFO);
4674         assert(na_info);
4675         mnl_attr_put_strz(nlh, IFLA_INFO_KIND, "vxlan");
4676         na_vxlan = mnl_attr_nest_start(nlh, IFLA_INFO_DATA);
4677         if (ifouter)
4678                 mnl_attr_put_u32(nlh, IFLA_VXLAN_LINK, ifouter);
4679         assert(na_vxlan);
4680         mnl_attr_put_u8(nlh, IFLA_VXLAN_COLLECT_METADATA, 1);
4681         mnl_attr_put_u8(nlh, IFLA_VXLAN_UDP_ZERO_CSUM6_RX, 1);
4682         mnl_attr_put_u8(nlh, IFLA_VXLAN_LEARNING, 0);
4683         mnl_attr_put_u16(nlh, IFLA_VXLAN_PORT, vxlan_port);
4684         mnl_attr_nest_end(nlh, na_vxlan);
4685         mnl_attr_nest_end(nlh, na_info);
4686         assert(sizeof(buf) >= nlh->nlmsg_len);
4687         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4688         if (ret) {
4689                 DRV_LOG(WARNING,
4690                         "netlink: VTEP %s create failure (%d)",
4691                         name, rte_errno);
4692                 if (rte_errno != EEXIST || ifouter)
4693                         /*
4694                          * Some unhandled error occurred or device is
4695                          * for encapsulation and cannot be shared.
4696                          */
4697                         goto error;
4698         } else {
4699                 /*
4700                  * Mark device we actually created.
4701                  * We should explicitly delete
4702                  * when we do not need it anymore.
4703                  */
4704                 vtep->created = 1;
4705         }
4706         /* Try to get ifindex of created of pre-existing device. */
4707         ret = if_nametoindex(name);
4708         if (!ret) {
4709                 DRV_LOG(WARNING,
4710                         "VTEP %s failed to get index (%d)", name, errno);
4711                 rte_flow_error_set
4712                         (error, -errno,
4713                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4714                          "netlink: failed to retrieve VTEP ifindex");
4715                 goto error;
4716         }
4717         vtep->ifindex = ret;
4718         vtep->ifouter = ifouter;
4719         memset(buf, 0, sizeof(buf));
4720         nlh = mnl_nlmsg_put_header(buf);
4721         nlh->nlmsg_type = RTM_NEWLINK;
4722         nlh->nlmsg_flags = NLM_F_REQUEST;
4723         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4724         ifm->ifi_family = AF_UNSPEC;
4725         ifm->ifi_type = 0;
4726         ifm->ifi_index = vtep->ifindex;
4727         ifm->ifi_flags = IFF_UP;
4728         ifm->ifi_change = IFF_UP;
4729         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4730         if (ret) {
4731                 rte_flow_error_set(error, -errno,
4732                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4733                                    "netlink: failed to set VTEP link up");
4734                 DRV_LOG(WARNING, "netlink: VTEP %s set link up failure (%d)",
4735                         name, rte_errno);
4736                 goto clean;
4737         }
4738         ret = mlx5_flow_tcf_init(tcf, vtep->ifindex, error);
4739         if (ret) {
4740                 DRV_LOG(WARNING, "VTEP %s init failure (%d)", name, rte_errno);
4741                 goto clean;
4742         }
4743         DRV_LOG(INFO, "VTEP create (%d, %d)", vtep->port, vtep->ifindex);
4744         vtep->refcnt = 1;
4745         return vtep;
4746 clean:
4747         flow_tcf_vtep_delete(tcf, vtep);
4748         return NULL;
4749 error:
4750         rte_free(vtep);
4751         return NULL;
4752 }
4753 #else
4754 static struct tcf_vtep*
4755 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf __rte_unused,
4756                      unsigned int ifouter __rte_unused,
4757                      uint16_t port __rte_unused,
4758                      struct rte_flow_error *error)
4759 {
4760         rte_flow_error_set(error, ENOTSUP,
4761                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4762                            "netlink: failed to create VTEP, "
4763                            "vxlan metadata are not supported by kernel");
4764         return NULL;
4765 }
4766 #endif /* HAVE_IFLA_VXLAN_COLLECT_METADATA */
4767
4768 /**
4769  * Acquire target interface index for VXLAN tunneling decapsulation.
4770  * In order to share the UDP port within the other interfaces the
4771  * VXLAN device created as not attached to any interface (if created).
4772  *
4773  * @param[in] tcf
4774  *   Context object initialized by mlx5_flow_tcf_context_create().
4775  * @param[in] dev_flow
4776  *   Flow tcf object with tunnel structure pointer set.
4777  * @param[out] error
4778  *   Perform verbose error reporting if not NULL.
4779  * @return
4780  *   Interface descriptor pointer on success,
4781  *   NULL otherwise and rte_errno is set.
4782  */
4783 static struct tcf_vtep*
4784 flow_tcf_decap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4785                             struct mlx5_flow *dev_flow,
4786                             struct rte_flow_error *error)
4787 {
4788         struct tcf_vtep *vtep;
4789         uint16_t port = dev_flow->tcf.vxlan_decap->udp_port;
4790
4791         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4792                 if (vtep->port == port)
4793                         break;
4794         }
4795         if (vtep && vtep->ifouter) {
4796                 rte_flow_error_set(error, -errno,
4797                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4798                                    "Failed to create decap VTEP with specified"
4799                                    " UDP port, atatched device exists");
4800                 return NULL;
4801         }
4802         if (vtep) {
4803                 /* Device exists, just increment the reference counter. */
4804                 vtep->refcnt++;
4805                 assert(vtep->ifindex);
4806                 return vtep;
4807         }
4808         /* No decapsulation device exists, try to create the new one. */
4809         vtep = flow_tcf_vtep_create(tcf, 0, port, error);
4810         if (vtep)
4811                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
4812         return vtep;
4813 }
4814
4815 /**
4816  * Aqcuire target interface index for VXLAN tunneling encapsulation.
4817  *
4818  * @param[in] tcf
4819  *   Context object initialized by mlx5_flow_tcf_context_create().
4820  * @param[in] ifouter
4821  *   Network interface index to attach VXLAN encap device to.
4822  * @param[in] dev_flow
4823  *   Flow tcf object with tunnel structure pointer set.
4824  * @param[out] error
4825  *   Perform verbose error reporting if not NULL.
4826  * @return
4827  *   Interface descriptor pointer on success,
4828  *   NULL otherwise and rte_errno is set.
4829  */
4830 static struct tcf_vtep*
4831 flow_tcf_encap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4832                             unsigned int ifouter,
4833                             struct mlx5_flow *dev_flow __rte_unused,
4834                             struct rte_flow_error *error)
4835 {
4836         static uint16_t encap_port = MLX5_VXLAN_PORT_MIN - 1;
4837         struct tcf_vtep *vtep;
4838         int ret;
4839
4840         assert(ifouter);
4841         /* Look whether the attached VTEP for encap is created. */
4842         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4843                 if (vtep->ifouter == ifouter)
4844                         break;
4845         }
4846         if (vtep) {
4847                 /* VTEP already exists, just increment the reference. */
4848                 vtep->refcnt++;
4849         } else {
4850                 uint16_t pcnt;
4851
4852                 /* Not found, we should create the new attached VTEP. */
4853                 flow_tcf_encap_iface_cleanup(tcf, ifouter);
4854                 flow_tcf_encap_local_cleanup(tcf, ifouter);
4855                 flow_tcf_encap_neigh_cleanup(tcf, ifouter);
4856                 for (pcnt = 0; pcnt <= (MLX5_VXLAN_PORT_MAX
4857                                      - MLX5_VXLAN_PORT_MIN); pcnt++) {
4858                         encap_port++;
4859                         /* Wraparound the UDP port index. */
4860                         if (encap_port < MLX5_VXLAN_PORT_MIN ||
4861                             encap_port > MLX5_VXLAN_PORT_MAX)
4862                                 encap_port = MLX5_VXLAN_PORT_MIN;
4863                         /* Check whether UDP port is in already in use. */
4864                         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4865                                 if (vtep->port == encap_port)
4866                                         break;
4867                         }
4868                         if (vtep) {
4869                                 /* Port is in use, try the next one. */
4870                                 vtep = NULL;
4871                                 continue;
4872                         }
4873                         vtep = flow_tcf_vtep_create(tcf, ifouter,
4874                                                     encap_port, error);
4875                         if (vtep) {
4876                                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
4877                                 break;
4878                         }
4879                         if (rte_errno != EEXIST)
4880                                 break;
4881                 }
4882                 if (!vtep)
4883                         return NULL;
4884         }
4885         assert(vtep->ifouter == ifouter);
4886         assert(vtep->ifindex);
4887         /* Create local ipaddr with peer to specify the outer IPs. */
4888         ret = flow_tcf_encap_local(tcf, vtep, dev_flow, true, error);
4889         if (!ret) {
4890                 /* Create neigh rule to specify outer destination MAC. */
4891                 ret = flow_tcf_encap_neigh(tcf, vtep, dev_flow, true, error);
4892                 if (ret)
4893                         flow_tcf_encap_local(tcf, vtep,
4894                                              dev_flow, false, error);
4895         }
4896         if (ret) {
4897                 if (--vtep->refcnt == 0)
4898                         flow_tcf_vtep_delete(tcf, vtep);
4899                 return NULL;
4900         }
4901         return vtep;
4902 }
4903
4904 /**
4905  * Acquires target interface index for tunneling of any type.
4906  * Creates the new VTEP if needed.
4907  *
4908  * @param[in] tcf
4909  *   Context object initialized by mlx5_flow_tcf_context_create().
4910  * @param[in] ifouter
4911  *   Network interface index to attach VXLAN encap device to.
4912  * @param[in] dev_flow
4913  *   Flow tcf object with tunnel structure pointer set.
4914  * @param[out] error
4915  *   Perform verbose error reporting if not NULL.
4916  * @return
4917  *   Interface descriptor pointer on success,
4918  *   NULL otherwise and rte_errno is set.
4919  */
4920 static struct tcf_vtep*
4921 flow_tcf_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4922                       unsigned int ifouter,
4923                       struct mlx5_flow *dev_flow,
4924                       struct rte_flow_error *error)
4925 {
4926         struct tcf_vtep *vtep = NULL;
4927
4928         assert(dev_flow->tcf.tunnel);
4929         pthread_mutex_lock(&vtep_list_mutex);
4930         switch (dev_flow->tcf.tunnel->type) {
4931         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
4932                 vtep = flow_tcf_encap_vtep_acquire(tcf, ifouter,
4933                                                   dev_flow, error);
4934                 break;
4935         case FLOW_TCF_TUNACT_VXLAN_DECAP:
4936                 vtep = flow_tcf_decap_vtep_acquire(tcf, dev_flow, error);
4937                 break;
4938         default:
4939                 rte_flow_error_set(error, ENOTSUP,
4940                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4941                                    "unsupported tunnel type");
4942                 break;
4943         }
4944         pthread_mutex_unlock(&vtep_list_mutex);
4945         return vtep;
4946 }
4947
4948 /**
4949  * Release tunneling interface by ifindex. Decrements reference
4950  * counter and actually removes the device if counter is zero.
4951  *
4952  * @param[in] tcf
4953  *   Context object initialized by mlx5_flow_tcf_context_create().
4954  * @param[in] vtep
4955  *   VTEP device descriptor structure.
4956  * @param[in] dev_flow
4957  *   Flow tcf object with tunnel structure pointer set.
4958  */
4959 static void
4960 flow_tcf_vtep_release(struct mlx5_flow_tcf_context *tcf,
4961                       struct tcf_vtep *vtep,
4962                       struct mlx5_flow *dev_flow)
4963 {
4964         assert(dev_flow->tcf.tunnel);
4965         pthread_mutex_lock(&vtep_list_mutex);
4966         switch (dev_flow->tcf.tunnel->type) {
4967         case FLOW_TCF_TUNACT_VXLAN_DECAP:
4968                 break;
4969         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
4970                 /* Remove the encap ancillary rules first. */
4971                 flow_tcf_encap_neigh(tcf, vtep, dev_flow, false, NULL);
4972                 flow_tcf_encap_local(tcf, vtep, dev_flow, false, NULL);
4973                 break;
4974         default:
4975                 assert(false);
4976                 DRV_LOG(WARNING, "Unsupported tunnel type");
4977                 break;
4978         }
4979         assert(vtep->refcnt);
4980         if (--vtep->refcnt == 0) {
4981                 LIST_REMOVE(vtep, next);
4982                 flow_tcf_vtep_delete(tcf, vtep);
4983         }
4984         pthread_mutex_unlock(&vtep_list_mutex);
4985 }
4986
4987 struct tcf_nlcb_query {
4988         uint32_t handle;
4989         uint32_t tc_flags;
4990         uint32_t flags_valid:1;
4991 };
4992
4993 /**
4994  * Collect queried rule attributes. This is callback routine called by
4995  * libmnl mnl_cb_run() in loop for every message in received packet.
4996  * Current implementation collects the flower flags only.
4997  *
4998  * @param[in] nlh
4999  *   Pointer to reply header.
5000  * @param[in, out] arg
5001  *   Context pointer for this callback.
5002  *
5003  * @return
5004  *   A positive, nonzero value on success (required by libmnl
5005  *   to continue messages processing).
5006  */
5007 static int
5008 flow_tcf_collect_query_cb(const struct nlmsghdr *nlh, void *arg)
5009 {
5010         struct tcf_nlcb_query *query = arg;
5011         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
5012         struct nlattr *na, *na_opt;
5013         bool flower = false;
5014
5015         if (nlh->nlmsg_type != RTM_NEWTFILTER ||
5016             tcm->tcm_handle != query->handle)
5017                 return 1;
5018         mnl_attr_for_each(na, nlh, sizeof(*tcm)) {
5019                 switch (mnl_attr_get_type(na)) {
5020                 case TCA_KIND:
5021                         if (strcmp(mnl_attr_get_payload(na), "flower")) {
5022                                 /* Not flower filter, drop entire message. */
5023                                 return 1;
5024                         }
5025                         flower = true;
5026                         break;
5027                 case TCA_OPTIONS:
5028                         if (!flower) {
5029                                 /* Not flower options, drop entire message. */
5030                                 return 1;
5031                         }
5032                         /* Check nested flower options. */
5033                         mnl_attr_for_each_nested(na_opt, na) {
5034                                 switch (mnl_attr_get_type(na_opt)) {
5035                                 case TCA_FLOWER_FLAGS:
5036                                         query->flags_valid = 1;
5037                                         query->tc_flags =
5038                                                 mnl_attr_get_u32(na_opt);
5039                                         break;
5040                                 }
5041                         }
5042                         break;
5043                 }
5044         }
5045         return 1;
5046 }
5047
5048 /**
5049  * Query a TC flower rule flags via netlink.
5050  *
5051  * @param[in] tcf
5052  *   Context object initialized by mlx5_flow_tcf_context_create().
5053  * @param[in] dev_flow
5054  *   Pointer to the flow.
5055  * @param[out] pflags
5056  *   pointer to the data retrieved by the query.
5057  *
5058  * @return
5059  *   0 on success, a negative errno value otherwise.
5060  */
5061 static int
5062 flow_tcf_query_flags(struct mlx5_flow_tcf_context *tcf,
5063                      struct mlx5_flow *dev_flow,
5064                      uint32_t *pflags)
5065 {
5066         struct nlmsghdr *nlh;
5067         struct tcmsg *tcm;
5068         struct tcf_nlcb_query query = {
5069                 .handle = dev_flow->tcf.tcm->tcm_handle,
5070         };
5071
5072         nlh = mnl_nlmsg_put_header(tcf->buf);
5073         nlh->nlmsg_type = RTM_GETTFILTER;
5074         nlh->nlmsg_flags = NLM_F_REQUEST;
5075         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5076         memcpy(tcm, dev_flow->tcf.tcm, sizeof(*tcm));
5077         /*
5078          * Ignore Netlink error for filter query operations.
5079          * The reply length is sent by kernel as errno.
5080          * Just check we got the flags option.
5081          */
5082         flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_query_cb, &query);
5083         if (!query.flags_valid) {
5084                 *pflags = 0;
5085                 return -ENOENT;
5086         }
5087         *pflags = query.tc_flags;
5088         return 0;
5089 }
5090
5091 /**
5092  * Query and check the in_hw set for specified rule.
5093  *
5094  * @param[in] tcf
5095  *   Context object initialized by mlx5_flow_tcf_context_create().
5096  * @param[in] dev_flow
5097  *   Pointer to the flow to check.
5098  *
5099  * @return
5100  *   0 on success, a negative errno value otherwise.
5101  */
5102 static int
5103 flow_tcf_check_inhw(struct mlx5_flow_tcf_context *tcf,
5104                     struct mlx5_flow *dev_flow)
5105 {
5106         uint32_t flags;
5107         int ret;
5108
5109         ret = flow_tcf_query_flags(tcf, dev_flow, &flags);
5110         if (ret)
5111                 return ret;
5112         return  (flags & TCA_CLS_FLAGS_IN_HW) ? 0 : -ENOENT;
5113 }
5114
5115 /**
5116  * Remove flow from E-Switch by sending Netlink message.
5117  *
5118  * @param[in] dev
5119  *   Pointer to Ethernet device.
5120  * @param[in, out] flow
5121  *   Pointer to the sub flow.
5122  */
5123 static void
5124 flow_tcf_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5125 {
5126         struct priv *priv = dev->data->dev_private;
5127         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5128         struct mlx5_flow *dev_flow;
5129         struct nlmsghdr *nlh;
5130
5131         if (!flow)
5132                 return;
5133         dev_flow = LIST_FIRST(&flow->dev_flows);
5134         if (!dev_flow)
5135                 return;
5136         /* E-Switch flow can't be expanded. */
5137         assert(!LIST_NEXT(dev_flow, next));
5138         if (dev_flow->tcf.applied) {
5139                 nlh = dev_flow->tcf.nlh;
5140                 nlh->nlmsg_type = RTM_DELTFILTER;
5141                 nlh->nlmsg_flags = NLM_F_REQUEST;
5142                 flow_tcf_nl_ack(ctx, nlh, NULL, NULL);
5143                 if (dev_flow->tcf.tunnel) {
5144                         assert(dev_flow->tcf.tunnel->vtep);
5145                         flow_tcf_vtep_release(ctx,
5146                                 dev_flow->tcf.tunnel->vtep,
5147                                 dev_flow);
5148                         dev_flow->tcf.tunnel->vtep = NULL;
5149                 }
5150                 dev_flow->tcf.applied = 0;
5151         }
5152 }
5153
5154 /**
5155  * Apply flow to E-Switch by sending Netlink message.
5156  *
5157  * @param[in] dev
5158  *   Pointer to Ethernet device.
5159  * @param[in, out] flow
5160  *   Pointer to the sub flow.
5161  * @param[out] error
5162  *   Pointer to the error structure.
5163  *
5164  * @return
5165  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5166  */
5167 static int
5168 flow_tcf_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5169                struct rte_flow_error *error)
5170 {
5171         struct priv *priv = dev->data->dev_private;
5172         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5173         struct mlx5_flow *dev_flow;
5174         struct nlmsghdr *nlh;
5175
5176         dev_flow = LIST_FIRST(&flow->dev_flows);
5177         /* E-Switch flow can't be expanded. */
5178         assert(!LIST_NEXT(dev_flow, next));
5179         if (dev_flow->tcf.applied)
5180                 return 0;
5181         nlh = dev_flow->tcf.nlh;
5182         nlh->nlmsg_type = RTM_NEWTFILTER;
5183         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5184         if (dev_flow->tcf.tunnel) {
5185                 /*
5186                  * Replace the interface index, target for
5187                  * encapsulation, source for decapsulation.
5188                  */
5189                 assert(!dev_flow->tcf.tunnel->vtep);
5190                 assert(dev_flow->tcf.tunnel->ifindex_ptr);
5191                 /* Acquire actual VTEP device when rule is being applied. */
5192                 dev_flow->tcf.tunnel->vtep =
5193                         flow_tcf_vtep_acquire(ctx,
5194                                         dev_flow->tcf.tunnel->ifindex_org,
5195                                         dev_flow, error);
5196                 if (!dev_flow->tcf.tunnel->vtep)
5197                         return -rte_errno;
5198                 DRV_LOG(INFO, "Replace ifindex: %d->%d",
5199                                 dev_flow->tcf.tunnel->vtep->ifindex,
5200                                 dev_flow->tcf.tunnel->ifindex_org);
5201                 *dev_flow->tcf.tunnel->ifindex_ptr =
5202                         dev_flow->tcf.tunnel->vtep->ifindex;
5203         }
5204         if (!flow_tcf_nl_ack(ctx, nlh, NULL, NULL)) {
5205                 dev_flow->tcf.applied = 1;
5206                 if (*dev_flow->tcf.ptc_flags & TCA_CLS_FLAGS_SKIP_SW)
5207                         return 0;
5208                 /*
5209                  * Rule was applied without skip_sw flag set.
5210                  * We should check whether the rule was acctually
5211                  * accepted by hardware (have look at in_hw flag).
5212                  */
5213                 if (flow_tcf_check_inhw(ctx, dev_flow)) {
5214                         flow_tcf_remove(dev, flow);
5215                         return rte_flow_error_set
5216                                 (error, ENOENT,
5217                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5218                                  "netlink: rule has no in_hw flag set");
5219                 }
5220                 return 0;
5221         }
5222         if (dev_flow->tcf.tunnel) {
5223                 /* Rollback the VTEP configuration if rule apply failed. */
5224                 assert(dev_flow->tcf.tunnel->vtep);
5225                 flow_tcf_vtep_release(ctx, dev_flow->tcf.tunnel->vtep,
5226                                       dev_flow);
5227                 dev_flow->tcf.tunnel->vtep = NULL;
5228         }
5229         return rte_flow_error_set(error, rte_errno,
5230                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5231                                   "netlink: failed to create TC flow rule");
5232 }
5233
5234 /**
5235  * Remove flow from E-Switch and release resources of the device flow.
5236  *
5237  * @param[in] dev
5238  *   Pointer to Ethernet device.
5239  * @param[in, out] flow
5240  *   Pointer to the sub flow.
5241  */
5242 static void
5243 flow_tcf_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5244 {
5245         struct mlx5_flow *dev_flow;
5246
5247         if (!flow)
5248                 return;
5249         flow_tcf_remove(dev, flow);
5250         if (flow->counter) {
5251                 if (--flow->counter->ref_cnt == 0) {
5252                         rte_free(flow->counter);
5253                         flow->counter = NULL;
5254                 }
5255         }
5256         dev_flow = LIST_FIRST(&flow->dev_flows);
5257         if (!dev_flow)
5258                 return;
5259         /* E-Switch flow can't be expanded. */
5260         assert(!LIST_NEXT(dev_flow, next));
5261         LIST_REMOVE(dev_flow, next);
5262         rte_free(dev_flow);
5263 }
5264
5265 /**
5266  * Helper routine for figuring the space size required for a parse buffer.
5267  *
5268  * @param array
5269  *   array of values to use.
5270  * @param idx
5271  *   Current location in array.
5272  * @param value
5273  *   Value to compare with.
5274  *
5275  * @return
5276  *   The maximum between the given value and the array value on index.
5277  */
5278 static uint16_t
5279 flow_tcf_arr_val_max(uint16_t array[], int idx, uint16_t value)
5280 {
5281         return idx < 0 ? (value) : RTE_MAX((array)[idx], value);
5282 }
5283
5284 /**
5285  * Parse rtnetlink message attributes filling the attribute table with the info
5286  * retrieved.
5287  *
5288  * @param tb
5289  *   Attribute table to be filled.
5290  * @param[out] max
5291  *   Maxinum entry in the attribute table.
5292  * @param rte
5293  *   The attributes section in the message to be parsed.
5294  * @param len
5295  *   The length of the attributes section in the message.
5296  */
5297 static void
5298 flow_tcf_nl_parse_rtattr(struct rtattr *tb[], int max,
5299                          struct rtattr *rta, int len)
5300 {
5301         unsigned short type;
5302         memset(tb, 0, sizeof(struct rtattr *) * (max + 1));
5303         while (RTA_OK(rta, len)) {
5304                 type = rta->rta_type;
5305                 if (type <= max && !tb[type])
5306                         tb[type] = rta;
5307                 rta = RTA_NEXT(rta, len);
5308         }
5309 }
5310
5311 /**
5312  * Extract flow counters from flower action.
5313  *
5314  * @param rta
5315  *   flower action stats properties in the Netlink message received.
5316  * @param rta_type
5317  *   The backward sequence of rta_types, as written in the attribute table,
5318  *   we need to traverse in order to get to the requested object.
5319  * @param idx
5320  *   Current location in rta_type table.
5321  * @param[out] data
5322  *   data holding the count statistics of the rte_flow retrieved from
5323  *   the message.
5324  *
5325  * @return
5326  *   0 if data was found and retrieved, -1 otherwise.
5327  */
5328 static int
5329 flow_tcf_nl_action_stats_parse_and_get(struct rtattr *rta,
5330                                        uint16_t rta_type[], int idx,
5331                                        struct gnet_stats_basic *data)
5332 {
5333         int tca_stats_max = flow_tcf_arr_val_max(rta_type, idx,
5334                                                  TCA_STATS_BASIC);
5335         struct rtattr *tbs[tca_stats_max + 1];
5336
5337         if (rta == NULL || idx < 0)
5338                 return -1;
5339         flow_tcf_nl_parse_rtattr(tbs, tca_stats_max,
5340                                  RTA_DATA(rta), RTA_PAYLOAD(rta));
5341         switch (rta_type[idx]) {
5342         case TCA_STATS_BASIC:
5343                 if (tbs[TCA_STATS_BASIC]) {
5344                         memcpy(data, RTA_DATA(tbs[TCA_STATS_BASIC]),
5345                                RTE_MIN(RTA_PAYLOAD(tbs[TCA_STATS_BASIC]),
5346                                sizeof(*data)));
5347                         return 0;
5348                 }
5349                 break;
5350         default:
5351                 break;
5352         }
5353         return -1;
5354 }
5355
5356 /**
5357  * Parse flower single action retrieving the requested action attribute,
5358  * if found.
5359  *
5360  * @param arg
5361  *   flower action properties in the Netlink message received.
5362  * @param rta_type
5363  *   The backward sequence of rta_types, as written in the attribute table,
5364  *   we need to traverse in order to get to the requested object.
5365  * @param idx
5366  *   Current location in rta_type table.
5367  * @param[out] data
5368  *   Count statistics retrieved from the message query.
5369  *
5370  * @return
5371  *   0 if data was found and retrieved, -1 otherwise.
5372  */
5373 static int
5374 flow_tcf_nl_parse_one_action_and_get(struct rtattr *arg,
5375                                      uint16_t rta_type[], int idx, void *data)
5376 {
5377         int tca_act_max = flow_tcf_arr_val_max(rta_type, idx, TCA_ACT_STATS);
5378         struct rtattr *tb[tca_act_max + 1];
5379
5380         if (arg == NULL || idx < 0)
5381                 return -1;
5382         flow_tcf_nl_parse_rtattr(tb, tca_act_max,
5383                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5384         if (tb[TCA_ACT_KIND] == NULL)
5385                 return -1;
5386         switch (rta_type[idx]) {
5387         case TCA_ACT_STATS:
5388                 if (tb[TCA_ACT_STATS])
5389                         return flow_tcf_nl_action_stats_parse_and_get
5390                                         (tb[TCA_ACT_STATS],
5391                                          rta_type, --idx,
5392                                          (struct gnet_stats_basic *)data);
5393                 break;
5394         default:
5395                 break;
5396         }
5397         return -1;
5398 }
5399
5400 /**
5401  * Parse flower action section in the message retrieving the requested
5402  * attribute from the first action that provides it.
5403  *
5404  * @param opt
5405  *   flower section in the Netlink message received.
5406  * @param rta_type
5407  *   The backward sequence of rta_types, as written in the attribute table,
5408  *   we need to traverse in order to get to the requested object.
5409  * @param idx
5410  *   Current location in rta_type table.
5411  * @param[out] data
5412  *   data retrieved from the message query.
5413  *
5414  * @return
5415  *   0 if data was found and retrieved, -1 otherwise.
5416  */
5417 static int
5418 flow_tcf_nl_action_parse_and_get(struct rtattr *arg,
5419                                  uint16_t rta_type[], int idx, void *data)
5420 {
5421         struct rtattr *tb[TCA_ACT_MAX_PRIO + 1];
5422         int i;
5423
5424         if (arg == NULL || idx < 0)
5425                 return -1;
5426         flow_tcf_nl_parse_rtattr(tb, TCA_ACT_MAX_PRIO,
5427                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5428         switch (rta_type[idx]) {
5429         /*
5430          * flow counters are stored in the actions defined by the flow
5431          * and not in the flow itself, therefore we need to traverse the
5432          * flower chain of actions in search for them.
5433          *
5434          * Note that the index is not decremented here.
5435          */
5436         case TCA_ACT_STATS:
5437                 for (i = 0; i <= TCA_ACT_MAX_PRIO; i++) {
5438                         if (tb[i] &&
5439                         !flow_tcf_nl_parse_one_action_and_get(tb[i],
5440                                                               rta_type,
5441                                                               idx, data))
5442                                 return 0;
5443                 }
5444                 break;
5445         default:
5446                 break;
5447         }
5448         return -1;
5449 }
5450
5451 /**
5452  * Parse flower classifier options in the message, retrieving the requested
5453  * attribute if found.
5454  *
5455  * @param opt
5456  *   flower section in the Netlink message received.
5457  * @param rta_type
5458  *   The backward sequence of rta_types, as written in the attribute table,
5459  *   we need to traverse in order to get to the requested object.
5460  * @param idx
5461  *   Current location in rta_type table.
5462  * @param[out] data
5463  *   data retrieved from the message query.
5464  *
5465  * @return
5466  *   0 if data was found and retrieved, -1 otherwise.
5467  */
5468 static int
5469 flow_tcf_nl_opts_parse_and_get(struct rtattr *opt,
5470                                uint16_t rta_type[], int idx, void *data)
5471 {
5472         int tca_flower_max = flow_tcf_arr_val_max(rta_type, idx,
5473                                                   TCA_FLOWER_ACT);
5474         struct rtattr *tb[tca_flower_max + 1];
5475
5476         if (!opt || idx < 0)
5477                 return -1;
5478         flow_tcf_nl_parse_rtattr(tb, tca_flower_max,
5479                                  RTA_DATA(opt), RTA_PAYLOAD(opt));
5480         switch (rta_type[idx]) {
5481         case TCA_FLOWER_ACT:
5482                 if (tb[TCA_FLOWER_ACT])
5483                         return flow_tcf_nl_action_parse_and_get
5484                                                         (tb[TCA_FLOWER_ACT],
5485                                                          rta_type, --idx, data);
5486                 break;
5487         default:
5488                 break;
5489         }
5490         return -1;
5491 }
5492
5493 /**
5494  * Parse Netlink reply on filter query, retrieving the flow counters.
5495  *
5496  * @param nlh
5497  *   Message received from Netlink.
5498  * @param rta_type
5499  *   The backward sequence of rta_types, as written in the attribute table,
5500  *   we need to traverse in order to get to the requested object.
5501  * @param idx
5502  *   Current location in rta_type table.
5503  * @param[out] data
5504  *   data retrieved from the message query.
5505  *
5506  * @return
5507  *   0 if data was found and retrieved, -1 otherwise.
5508  */
5509 static int
5510 flow_tcf_nl_filter_parse_and_get(struct nlmsghdr *cnlh,
5511                                  uint16_t rta_type[], int idx, void *data)
5512 {
5513         struct nlmsghdr *nlh = cnlh;
5514         struct tcmsg *t = NLMSG_DATA(nlh);
5515         int len = nlh->nlmsg_len;
5516         int tca_max = flow_tcf_arr_val_max(rta_type, idx, TCA_OPTIONS);
5517         struct rtattr *tb[tca_max + 1];
5518
5519         if (idx < 0)
5520                 return -1;
5521         if (nlh->nlmsg_type != RTM_NEWTFILTER &&
5522             nlh->nlmsg_type != RTM_GETTFILTER &&
5523             nlh->nlmsg_type != RTM_DELTFILTER)
5524                 return -1;
5525         len -= NLMSG_LENGTH(sizeof(*t));
5526         if (len < 0)
5527                 return -1;
5528         flow_tcf_nl_parse_rtattr(tb, tca_max, TCA_RTA(t), len);
5529         /* Not a TC flower flow - bail out */
5530         if (!tb[TCA_KIND] ||
5531             strcmp(RTA_DATA(tb[TCA_KIND]), "flower"))
5532                 return -1;
5533         switch (rta_type[idx]) {
5534         case TCA_OPTIONS:
5535                 if (tb[TCA_OPTIONS])
5536                         return flow_tcf_nl_opts_parse_and_get(tb[TCA_OPTIONS],
5537                                                               rta_type,
5538                                                               --idx, data);
5539                 break;
5540         default:
5541                 break;
5542         }
5543         return -1;
5544 }
5545
5546 /**
5547  * A callback to parse Netlink reply on TC flower query.
5548  *
5549  * @param nlh
5550  *   Message received from Netlink.
5551  * @param[out] data
5552  *   Pointer to data area to be filled by the parsing routine.
5553  *   assumed to be a pointer to struct flow_tcf_stats_basic.
5554  *
5555  * @return
5556  *   MNL_CB_OK value.
5557  */
5558 static int
5559 flow_tcf_nl_message_get_stats_basic(const struct nlmsghdr *nlh, void *data)
5560 {
5561         /*
5562          * The backward sequence of rta_types to pass in order to get
5563          *  to the counters.
5564          */
5565         uint16_t rta_type[] = { TCA_STATS_BASIC, TCA_ACT_STATS,
5566                                 TCA_FLOWER_ACT, TCA_OPTIONS };
5567         struct flow_tcf_stats_basic *sb_data = data;
5568         union {
5569                 const struct nlmsghdr *c;
5570                 struct nlmsghdr *nc;
5571         } tnlh = { .c = nlh };
5572
5573         if (!flow_tcf_nl_filter_parse_and_get(tnlh.nc, rta_type,
5574                                               RTE_DIM(rta_type) - 1,
5575                                               (void *)&sb_data->counters))
5576                 sb_data->valid = true;
5577         return MNL_CB_OK;
5578 }
5579
5580 /**
5581  * Query a TC flower rule for its statistics via netlink.
5582  *
5583  * @param[in] dev
5584  *   Pointer to Ethernet device.
5585  * @param[in] flow
5586  *   Pointer to the sub flow.
5587  * @param[out] data
5588  *   data retrieved by the query.
5589  * @param[out] error
5590  *   Perform verbose error reporting if not NULL.
5591  *
5592  * @return
5593  *   0 on success, a negative errno value otherwise and rte_errno is set.
5594  */
5595 static int
5596 flow_tcf_query_count(struct rte_eth_dev *dev,
5597                           struct rte_flow *flow,
5598                           void *data,
5599                           struct rte_flow_error *error)
5600 {
5601         struct flow_tcf_stats_basic sb_data;
5602         struct rte_flow_query_count *qc = data;
5603         struct priv *priv = dev->data->dev_private;
5604         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5605         struct mnl_socket *nl = ctx->nl;
5606         struct mlx5_flow *dev_flow;
5607         struct nlmsghdr *nlh;
5608         uint32_t seq = priv->tcf_context->seq++;
5609         ssize_t ret;
5610         assert(qc);
5611
5612         memset(&sb_data, 0, sizeof(sb_data));
5613         dev_flow = LIST_FIRST(&flow->dev_flows);
5614         /* E-Switch flow can't be expanded. */
5615         assert(!LIST_NEXT(dev_flow, next));
5616         if (!dev_flow->flow->counter)
5617                 goto notsup_exit;
5618         nlh = dev_flow->tcf.nlh;
5619         nlh->nlmsg_type = RTM_GETTFILTER;
5620         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_ECHO;
5621         nlh->nlmsg_seq = seq;
5622         if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) == -1)
5623                 goto error_exit;
5624         do {
5625                 ret = mnl_socket_recvfrom(nl, ctx->buf, ctx->buf_size);
5626                 if (ret <= 0)
5627                         break;
5628                 ret = mnl_cb_run(ctx->buf, ret, seq,
5629                                  mnl_socket_get_portid(nl),
5630                                  flow_tcf_nl_message_get_stats_basic,
5631                                  (void *)&sb_data);
5632         } while (ret > 0);
5633         /* Return the delta from last reset. */
5634         if (sb_data.valid) {
5635                 /* Return the delta from last reset. */
5636                 qc->hits_set = 1;
5637                 qc->bytes_set = 1;
5638                 qc->hits = sb_data.counters.packets - flow->counter->hits;
5639                 qc->bytes = sb_data.counters.bytes - flow->counter->bytes;
5640                 if (qc->reset) {
5641                         flow->counter->hits = sb_data.counters.packets;
5642                         flow->counter->bytes = sb_data.counters.bytes;
5643                 }
5644                 return 0;
5645         }
5646         return rte_flow_error_set(error, EINVAL,
5647                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5648                                   NULL,
5649                                   "flow does not have counter");
5650 error_exit:
5651         return rte_flow_error_set
5652                         (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5653                          NULL, "netlink: failed to read flow rule counters");
5654 notsup_exit:
5655         return rte_flow_error_set
5656                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5657                          NULL, "counters are not available.");
5658 }
5659
5660 /**
5661  * Query a flow.
5662  *
5663  * @see rte_flow_query()
5664  * @see rte_flow_ops
5665  */
5666 static int
5667 flow_tcf_query(struct rte_eth_dev *dev,
5668                struct rte_flow *flow,
5669                const struct rte_flow_action *actions,
5670                void *data,
5671                struct rte_flow_error *error)
5672 {
5673         int ret = -EINVAL;
5674
5675         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5676                 switch (actions->type) {
5677                 case RTE_FLOW_ACTION_TYPE_VOID:
5678                         break;
5679                 case RTE_FLOW_ACTION_TYPE_COUNT:
5680                         ret = flow_tcf_query_count(dev, flow, data, error);
5681                         break;
5682                 default:
5683                         return rte_flow_error_set(error, ENOTSUP,
5684                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5685                                                   actions,
5686                                                   "action not supported");
5687                 }
5688         }
5689         return ret;
5690 }
5691
5692 const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops = {
5693         .validate = flow_tcf_validate,
5694         .prepare = flow_tcf_prepare,
5695         .translate = flow_tcf_translate,
5696         .apply = flow_tcf_apply,
5697         .remove = flow_tcf_remove,
5698         .destroy = flow_tcf_destroy,
5699         .query = flow_tcf_query,
5700 };
5701
5702 /**
5703  * Create and configure a libmnl socket for Netlink flow rules.
5704  *
5705  * @return
5706  *   A valid libmnl socket object pointer on success, NULL otherwise and
5707  *   rte_errno is set.
5708  */
5709 static struct mnl_socket *
5710 flow_tcf_mnl_socket_create(void)
5711 {
5712         struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
5713
5714         if (nl) {
5715                 mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
5716                                       sizeof(int));
5717                 if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
5718                         return nl;
5719         }
5720         rte_errno = errno;
5721         if (nl)
5722                 mnl_socket_close(nl);
5723         return NULL;
5724 }
5725
5726 /**
5727  * Destroy a libmnl socket.
5728  *
5729  * @param nl
5730  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5731  */
5732 static void
5733 flow_tcf_mnl_socket_destroy(struct mnl_socket *nl)
5734 {
5735         if (nl)
5736                 mnl_socket_close(nl);
5737 }
5738
5739 /**
5740  * Initialize ingress qdisc of a given network interface.
5741  *
5742  * @param ctx
5743  *   Pointer to tc-flower context to use.
5744  * @param ifindex
5745  *   Index of network interface to initialize.
5746  * @param[out] error
5747  *   Perform verbose error reporting if not NULL.
5748  *
5749  * @return
5750  *   0 on success, a negative errno value otherwise and rte_errno is set.
5751  */
5752 int
5753 mlx5_flow_tcf_init(struct mlx5_flow_tcf_context *ctx,
5754                    unsigned int ifindex, struct rte_flow_error *error)
5755 {
5756         struct nlmsghdr *nlh;
5757         struct tcmsg *tcm;
5758         alignas(struct nlmsghdr)
5759         uint8_t buf[mnl_nlmsg_size(sizeof(*tcm)) +
5760                     SZ_NLATTR_STRZ_OF("ingress") +
5761                     MNL_BUF_EXTRA_SPACE];
5762
5763         /* Destroy existing ingress qdisc and everything attached to it. */
5764         nlh = mnl_nlmsg_put_header(buf);
5765         nlh->nlmsg_type = RTM_DELQDISC;
5766         nlh->nlmsg_flags = NLM_F_REQUEST;
5767         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5768         tcm->tcm_family = AF_UNSPEC;
5769         tcm->tcm_ifindex = ifindex;
5770         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5771         tcm->tcm_parent = TC_H_INGRESS;
5772         assert(sizeof(buf) >= nlh->nlmsg_len);
5773         /* Ignore errors when qdisc is already absent. */
5774         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL) &&
5775             rte_errno != EINVAL && rte_errno != ENOENT)
5776                 return rte_flow_error_set(error, rte_errno,
5777                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5778                                           "netlink: failed to remove ingress"
5779                                           " qdisc");
5780         /* Create fresh ingress qdisc. */
5781         nlh = mnl_nlmsg_put_header(buf);
5782         nlh->nlmsg_type = RTM_NEWQDISC;
5783         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5784         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5785         tcm->tcm_family = AF_UNSPEC;
5786         tcm->tcm_ifindex = ifindex;
5787         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5788         tcm->tcm_parent = TC_H_INGRESS;
5789         mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
5790         assert(sizeof(buf) >= nlh->nlmsg_len);
5791         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL))
5792                 return rte_flow_error_set(error, rte_errno,
5793                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5794                                           "netlink: failed to create ingress"
5795                                           " qdisc");
5796         return 0;
5797 }
5798
5799 /**
5800  * Create libmnl context for Netlink flow rules.
5801  *
5802  * @return
5803  *   A valid libmnl socket object pointer on success, NULL otherwise and
5804  *   rte_errno is set.
5805  */
5806 struct mlx5_flow_tcf_context *
5807 mlx5_flow_tcf_context_create(void)
5808 {
5809         struct mlx5_flow_tcf_context *ctx = rte_zmalloc(__func__,
5810                                                         sizeof(*ctx),
5811                                                         sizeof(uint32_t));
5812         if (!ctx)
5813                 goto error;
5814         ctx->nl = flow_tcf_mnl_socket_create();
5815         if (!ctx->nl)
5816                 goto error;
5817         ctx->buf_size = MNL_SOCKET_BUFFER_SIZE;
5818         ctx->buf = rte_zmalloc(__func__,
5819                                ctx->buf_size, sizeof(uint32_t));
5820         if (!ctx->buf)
5821                 goto error;
5822         ctx->seq = random();
5823         return ctx;
5824 error:
5825         mlx5_flow_tcf_context_destroy(ctx);
5826         return NULL;
5827 }
5828
5829 /**
5830  * Destroy a libmnl context.
5831  *
5832  * @param ctx
5833  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5834  */
5835 void
5836 mlx5_flow_tcf_context_destroy(struct mlx5_flow_tcf_context *ctx)
5837 {
5838         if (!ctx)
5839                 return;
5840         flow_tcf_mnl_socket_destroy(ctx->nl);
5841         rte_free(ctx->buf);
5842         rte_free(ctx);
5843 }