drivers/net/mlx5/mlx5_flow_tcf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <assert.h>
   7 #include <errno.h>
   8 #include <libmnl/libmnl.h>
   9 #include <linux/gen_stats.h>
  10 #include <linux/if_ether.h>
  11 #include <linux/netlink.h>
  12 #include <linux/pkt_cls.h>
  13 #include <linux/pkt_sched.h>
  14 #include <linux/rtnetlink.h>
  15 #include <linux/tc_act/tc_gact.h>
  16 #include <linux/tc_act/tc_mirred.h>
  17 #include <netinet/in.h>
  18 #include <stdalign.h>
  19 #include <stdbool.h>
  20 #include <stddef.h>
  21 #include <stdint.h>
  22 #include <stdlib.h>
  23 #include <sys/socket.h>
  24
  25 #include <rte_byteorder.h>
  26 #include <rte_errno.h>
  27 #include <rte_ether.h>
  28 #include <rte_flow.h>
  29 #include <rte_malloc.h>
  30 #include <rte_common.h>
  31
  32 #include "mlx5.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_autoconf.h"
  35
  36 #ifdef HAVE_TC_ACT_VLAN
  37
  38 #include <linux/tc_act/tc_vlan.h>
  39
  40 #else /* HAVE_TC_ACT_VLAN */
  41
  42 #define TCA_VLAN_ACT_POP 1
  43 #define TCA_VLAN_ACT_PUSH 2
  44 #define TCA_VLAN_ACT_MODIFY 3
  45 #define TCA_VLAN_PARMS 2
  46 #define TCA_VLAN_PUSH_VLAN_ID 3
  47 #define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
  48 #define TCA_VLAN_PAD 5
  49 #define TCA_VLAN_PUSH_VLAN_PRIORITY 6
  50
  51 struct tc_vlan {
  52         tc_gen;
  53         int v_action;
  54 };
  55
  56 #endif /* HAVE_TC_ACT_VLAN */
  57
  58 #ifdef HAVE_TC_ACT_PEDIT
  59
  60 #include <linux/tc_act/tc_pedit.h>
  61
  62 #else /* HAVE_TC_ACT_VLAN */
  63
  64 enum {
  65         TCA_PEDIT_UNSPEC,
  66         TCA_PEDIT_TM,
  67         TCA_PEDIT_PARMS,
  68         TCA_PEDIT_PAD,
  69         TCA_PEDIT_PARMS_EX,
  70         TCA_PEDIT_KEYS_EX,
  71         TCA_PEDIT_KEY_EX,
  72         __TCA_PEDIT_MAX
  73 };
  74
  75 enum {
  76         TCA_PEDIT_KEY_EX_HTYPE = 1,
  77         TCA_PEDIT_KEY_EX_CMD = 2,
  78         __TCA_PEDIT_KEY_EX_MAX
  79 };
  80
  81 enum pedit_header_type {
  82         TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK = 0,
  83         TCA_PEDIT_KEY_EX_HDR_TYPE_ETH = 1,
  84         TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 = 2,
  85         TCA_PEDIT_KEY_EX_HDR_TYPE_IP6 = 3,
  86         TCA_PEDIT_KEY_EX_HDR_TYPE_TCP = 4,
  87         TCA_PEDIT_KEY_EX_HDR_TYPE_UDP = 5,
  88         __PEDIT_HDR_TYPE_MAX,
  89 };
  90
  91 enum pedit_cmd {
  92         TCA_PEDIT_KEY_EX_CMD_SET = 0,
  93         TCA_PEDIT_KEY_EX_CMD_ADD = 1,
  94         __PEDIT_CMD_MAX,
  95 };
  96
  97 struct tc_pedit_key {
  98         __u32 mask; /* AND */
  99         __u32 val; /*XOR */
 100         __u32 off; /*offset */
 101         __u32 at;
 102         __u32 offmask;
 103         __u32 shift;
 104 };
 105
 106 __extension__
 107 struct tc_pedit_sel {
 108         tc_gen;
 109         unsigned char nkeys;
 110         unsigned char flags;
 111         struct tc_pedit_key keys[0];
 112 };
 113
 114 #endif /* HAVE_TC_ACT_VLAN */
 115
 116 #ifdef HAVE_TC_ACT_TUNNEL_KEY
 117
 118 #include <linux/tc_act/tc_tunnel_key.h>
 119
 120 #ifndef HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT
 121 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 122 #endif
 123
 124 #ifndef HAVE_TCA_TUNNEL_KEY_NO_CSUM
 125 #define TCA_TUNNEL_KEY_NO_CSUM 10
 126 #endif
 127
 128 #else /* HAVE_TC_ACT_TUNNEL_KEY */
 129
 130 #define TCA_ACT_TUNNEL_KEY 17
 131 #define TCA_TUNNEL_KEY_ACT_SET 1
 132 #define TCA_TUNNEL_KEY_ACT_RELEASE 2
 133 #define TCA_TUNNEL_KEY_PARMS 2
 134 #define TCA_TUNNEL_KEY_ENC_IPV4_SRC 3
 135 #define TCA_TUNNEL_KEY_ENC_IPV4_DST 4
 136 #define TCA_TUNNEL_KEY_ENC_IPV6_SRC 5
 137 #define TCA_TUNNEL_KEY_ENC_IPV6_DST 6
 138 #define TCA_TUNNEL_KEY_ENC_KEY_ID 7
 139 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 140 #define TCA_TUNNEL_KEY_NO_CSUM 10
 141
 142 struct tc_tunnel_key {
 143         tc_gen;
 144         int t_action;
 145 };
 146
 147 #endif /* HAVE_TC_ACT_TUNNEL_KEY */
 148
 149 /* Normally found in linux/netlink.h. */
 150 #ifndef NETLINK_CAP_ACK
 151 #define NETLINK_CAP_ACK 10
 152 #endif
 153
 154 /* Normally found in linux/pkt_sched.h. */
 155 #ifndef TC_H_MIN_INGRESS
 156 #define TC_H_MIN_INGRESS 0xfff2u
 157 #endif
 158
 159 /* Normally found in linux/pkt_cls.h. */
 160 #ifndef TCA_CLS_FLAGS_SKIP_SW
 161 #define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
 162 #endif
 163 #ifndef TCA_CLS_FLAGS_IN_HW
 164 #define TCA_CLS_FLAGS_IN_HW (1 << 2)
 165 #endif
 166 #ifndef HAVE_TCA_CHAIN
 167 #define TCA_CHAIN 11
 168 #endif
 169 #ifndef HAVE_TCA_FLOWER_ACT
 170 #define TCA_FLOWER_ACT 3
 171 #endif
 172 #ifndef HAVE_TCA_FLOWER_FLAGS
 173 #define TCA_FLOWER_FLAGS 22
 174 #endif
 175 #ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
 176 #define TCA_FLOWER_KEY_ETH_TYPE 8
 177 #endif
 178 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
 179 #define TCA_FLOWER_KEY_ETH_DST 4
 180 #endif
 181 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
 182 #define TCA_FLOWER_KEY_ETH_DST_MASK 5
 183 #endif
 184 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
 185 #define TCA_FLOWER_KEY_ETH_SRC 6
 186 #endif
 187 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
 188 #define TCA_FLOWER_KEY_ETH_SRC_MASK 7
 189 #endif
 190 #ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
 191 #define TCA_FLOWER_KEY_IP_PROTO 9
 192 #endif
 193 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
 194 #define TCA_FLOWER_KEY_IPV4_SRC 10
 195 #endif
 196 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
 197 #define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
 198 #endif
 199 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
 200 #define TCA_FLOWER_KEY_IPV4_DST 12
 201 #endif
 202 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
 203 #define TCA_FLOWER_KEY_IPV4_DST_MASK 13
 204 #endif
 205 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
 206 #define TCA_FLOWER_KEY_IPV6_SRC 14
 207 #endif
 208 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
 209 #define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
 210 #endif
 211 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
 212 #define TCA_FLOWER_KEY_IPV6_DST 16
 213 #endif
 214 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
 215 #define TCA_FLOWER_KEY_IPV6_DST_MASK 17
 216 #endif
 217 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
 218 #define TCA_FLOWER_KEY_TCP_SRC 18
 219 #endif
 220 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
 221 #define TCA_FLOWER_KEY_TCP_SRC_MASK 35
 222 #endif
 223 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
 224 #define TCA_FLOWER_KEY_TCP_DST 19
 225 #endif
 226 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
 227 #define TCA_FLOWER_KEY_TCP_DST_MASK 36
 228 #endif
 229 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
 230 #define TCA_FLOWER_KEY_UDP_SRC 20
 231 #endif
 232 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
 233 #define TCA_FLOWER_KEY_UDP_SRC_MASK 37
 234 #endif
 235 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
 236 #define TCA_FLOWER_KEY_UDP_DST 21
 237 #endif
 238 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
 239 #define TCA_FLOWER_KEY_UDP_DST_MASK 38
 240 #endif
 241 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
 242 #define TCA_FLOWER_KEY_VLAN_ID 23
 243 #endif
 244 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
 245 #define TCA_FLOWER_KEY_VLAN_PRIO 24
 246 #endif
 247 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
 248 #define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
 249 #endif
 250 #ifndef HAVE_TCA_FLOWER_KEY_ENC_KEY_ID
 251 #define TCA_FLOWER_KEY_ENC_KEY_ID 26
 252 #endif
 253 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC
 254 #define TCA_FLOWER_KEY_ENC_IPV4_SRC 27
 255 #endif
 256 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK
 257 #define TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK 28
 258 #endif
 259 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST
 260 #define TCA_FLOWER_KEY_ENC_IPV4_DST 29
 261 #endif
 262 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK
 263 #define TCA_FLOWER_KEY_ENC_IPV4_DST_MASK 30
 264 #endif
 265 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC
 266 #define TCA_FLOWER_KEY_ENC_IPV6_SRC 31
 267 #endif
 268 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK
 269 #define TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK 32
 270 #endif
 271 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST
 272 #define TCA_FLOWER_KEY_ENC_IPV6_DST 33
 273 #endif
 274 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK
 275 #define TCA_FLOWER_KEY_ENC_IPV6_DST_MASK 34
 276 #endif
 277 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT
 278 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT 43
 279 #endif
 280 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK
 281 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK 44
 282 #endif
 283 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT
 284 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT 45
 285 #endif
 286 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK
 287 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK 46
 288 #endif
 289 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS
 290 #define TCA_FLOWER_KEY_TCP_FLAGS 71
 291 #endif
 292 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK
 293 #define TCA_FLOWER_KEY_TCP_FLAGS_MASK 72
 294 #endif
 295 #ifndef HAVE_TC_ACT_GOTO_CHAIN
 296 #define TC_ACT_GOTO_CHAIN 0x20000000
 297 #endif
 298
 299 #ifndef IPV6_ADDR_LEN
 300 #define IPV6_ADDR_LEN 16
 301 #endif
 302
 303 #ifndef IPV4_ADDR_LEN
 304 #define IPV4_ADDR_LEN 4
 305 #endif
 306
 307 #ifndef TP_PORT_LEN
 308 #define TP_PORT_LEN 2 /* Transport Port (UDP/TCP) Length */
 309 #endif
 310
 311 #ifndef TTL_LEN
 312 #define TTL_LEN 1
 313 #endif
 314
 315 #ifndef TCA_ACT_MAX_PRIO
 316 #define TCA_ACT_MAX_PRIO 32
 317 #endif
 318
 319 /** UDP port range of VXLAN devices created by driver. */
 320 #define MLX5_VXLAN_PORT_MIN 30000
 321 #define MLX5_VXLAN_PORT_MAX 60000
 322 #define MLX5_VXLAN_DEVICE_PFX "vmlx_"
 323
 324 /** Tunnel action type, used for @p type in header structure. */
 325 enum flow_tcf_tunact_type {
 326         FLOW_TCF_TUNACT_VXLAN_DECAP,
 327         FLOW_TCF_TUNACT_VXLAN_ENCAP,
 328 };
 329
 330 /** Flags used for @p mask in tunnel action encap descriptors. */
 331 #define FLOW_TCF_ENCAP_ETH_SRC (1u << 0)
 332 #define FLOW_TCF_ENCAP_ETH_DST (1u << 1)
 333 #define FLOW_TCF_ENCAP_IPV4_SRC (1u << 2)
 334 #define FLOW_TCF_ENCAP_IPV4_DST (1u << 3)
 335 #define FLOW_TCF_ENCAP_IPV6_SRC (1u << 4)
 336 #define FLOW_TCF_ENCAP_IPV6_DST (1u << 5)
 337 #define FLOW_TCF_ENCAP_UDP_SRC (1u << 6)
 338 #define FLOW_TCF_ENCAP_UDP_DST (1u << 7)
 339 #define FLOW_TCF_ENCAP_VXLAN_VNI (1u << 8)
 340
 341 /**
 342  * Structure for holding netlink context.
 343  * Note the size of the message buffer which is MNL_SOCKET_BUFFER_SIZE.
 344  * Using this (8KB) buffer size ensures that netlink messages will never be
 345  * truncated.
 346  */
 347 struct mlx5_flow_tcf_context {
 348         struct mnl_socket *nl; /* NETLINK_ROUTE libmnl socket. */
 349         uint32_t seq; /* Message sequence number. */
 350         uint32_t buf_size; /* Message buffer size. */
 351         uint8_t *buf; /* Message buffer. */
 352 };
 353
 354 /**
 355  * Neigh rule structure. The neigh rule is applied via Netlink to
 356  * outer tunnel iface in order to provide destination MAC address
 357  * for the VXLAN encapsultion. The neigh rule is implicitly related
 358  * to the Flow itself and can be shared by multiple Flows.
 359  */
 360 struct tcf_neigh_rule {
 361         LIST_ENTRY(tcf_neigh_rule) next;
 362         uint32_t refcnt;
 363         struct ether_addr eth;
 364         uint16_t mask;
 365         union {
 366                 struct {
 367                         rte_be32_t dst;
 368                 } ipv4;
 369                 struct {
 370                         uint8_t dst[IPV6_ADDR_LEN];
 371                 } ipv6;
 372         };
 373 };
 374
 375 /**
 376  * Local rule structure. The local rule is applied via Netlink to
 377  * outer tunnel iface in order to provide local and peer IP addresses
 378  * of the VXLAN tunnel for encapsulation. The local rule is implicitly
 379  * related to the Flow itself and can be shared by multiple Flows.
 380  */
 381 struct tcf_local_rule {
 382         LIST_ENTRY(tcf_local_rule) next;
 383         uint32_t refcnt;
 384         uint16_t mask;
 385         union {
 386                 struct {
 387                         rte_be32_t dst;
 388                         rte_be32_t src;
 389                 } ipv4;
 390                 struct {
 391                         uint8_t dst[IPV6_ADDR_LEN];
 392                         uint8_t src[IPV6_ADDR_LEN];
 393                 } ipv6;
 394         };
 395 };
 396
 397 /** Outer interface VXLAN encapsulation rules container. */
 398 struct tcf_irule {
 399         LIST_ENTRY(tcf_irule) next;
 400         LIST_HEAD(, tcf_neigh_rule) neigh;
 401         LIST_HEAD(, tcf_local_rule) local;
 402         uint32_t refcnt;
 403         unsigned int ifouter; /**< Own interface index. */
 404 };
 405
 406 /** VXLAN virtual netdev. */
 407 struct tcf_vtep {
 408         LIST_ENTRY(tcf_vtep) next;
 409         uint32_t refcnt;
 410         unsigned int ifindex; /**< Own interface index. */
 411         uint16_t port;
 412         uint8_t created;
 413 };
 414
 415 /** Tunnel descriptor header, common for all tunnel types. */
 416 struct flow_tcf_tunnel_hdr {
 417         uint32_t type; /**< Tunnel action type. */
 418         struct tcf_vtep *vtep; /**< Virtual tunnel endpoint device. */
 419         unsigned int ifindex_org; /**< Original dst/src interface */
 420         unsigned int *ifindex_ptr; /**< Interface ptr in message. */
 421 };
 422
 423 struct flow_tcf_vxlan_decap {
 424         struct flow_tcf_tunnel_hdr hdr;
 425         uint16_t udp_port;
 426 };
 427
 428 struct flow_tcf_vxlan_encap {
 429         struct flow_tcf_tunnel_hdr hdr;
 430         struct tcf_irule *iface;
 431         uint32_t mask;
 432         struct {
 433                 struct ether_addr dst;
 434                 struct ether_addr src;
 435         } eth;
 436         union {
 437                 struct {
 438                         rte_be32_t dst;
 439                         rte_be32_t src;
 440                 } ipv4;
 441                 struct {
 442                         uint8_t dst[IPV6_ADDR_LEN];
 443                         uint8_t src[IPV6_ADDR_LEN];
 444                 } ipv6;
 445         };
 446         struct {
 447                 rte_be16_t src;
 448                 rte_be16_t dst;
 449         } udp;
 450         struct {
 451                 uint8_t vni[3];
 452         } vxlan;
 453 };
 454
 455 /** Structure used when extracting the values of a flow counters
 456  * from a netlink message.
 457  */
 458 struct flow_tcf_stats_basic {
 459         bool valid;
 460         struct gnet_stats_basic counters;
 461 };
 462
 463 /** Empty masks for known item types. */
 464 static const union {
 465         struct rte_flow_item_port_id port_id;
 466         struct rte_flow_item_eth eth;
 467         struct rte_flow_item_vlan vlan;
 468         struct rte_flow_item_ipv4 ipv4;
 469         struct rte_flow_item_ipv6 ipv6;
 470         struct rte_flow_item_tcp tcp;
 471         struct rte_flow_item_udp udp;
 472         struct rte_flow_item_vxlan vxlan;
 473 } flow_tcf_mask_empty = {
 474         {0},
 475 };
 476
 477 /** Supported masks for known item types. */
 478 static const struct {
 479         struct rte_flow_item_port_id port_id;
 480         struct rte_flow_item_eth eth;
 481         struct rte_flow_item_vlan vlan;
 482         struct rte_flow_item_ipv4 ipv4;
 483         struct rte_flow_item_ipv6 ipv6;
 484         struct rte_flow_item_tcp tcp;
 485         struct rte_flow_item_udp udp;
 486         struct rte_flow_item_vxlan vxlan;
 487 } flow_tcf_mask_supported = {
 488         .port_id = {
 489                 .id = 0xffffffff,
 490         },
 491         .eth = {
 492                 .type = RTE_BE16(0xffff),
 493                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 494                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 495         },
 496         .vlan = {
 497                 /* PCP and VID only, no DEI. */
 498                 .tci = RTE_BE16(0xefff),
 499                 .inner_type = RTE_BE16(0xffff),
 500         },
 501         .ipv4.hdr = {
 502                 .next_proto_id = 0xff,
 503                 .src_addr = RTE_BE32(0xffffffff),
 504                 .dst_addr = RTE_BE32(0xffffffff),
 505         },
 506         .ipv6.hdr = {
 507                 .proto = 0xff,
 508                 .src_addr =
 509                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 510                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 511                 .dst_addr =
 512                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 513                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 514         },
 515         .tcp.hdr = {
 516                 .src_port = RTE_BE16(0xffff),
 517                 .dst_port = RTE_BE16(0xffff),
 518                 .tcp_flags = 0xff,
 519         },
 520         .udp.hdr = {
 521                 .src_port = RTE_BE16(0xffff),
 522                 .dst_port = RTE_BE16(0xffff),
 523         },
 524         .vxlan = {
 525                .vni = "\xff\xff\xff",
 526         },
 527 };
 528
 529 #define SZ_NLATTR_HDR MNL_ALIGN(sizeof(struct nlattr))
 530 #define SZ_NLATTR_NEST SZ_NLATTR_HDR
 531 #define SZ_NLATTR_DATA_OF(len) MNL_ALIGN(SZ_NLATTR_HDR + (len))
 532 #define SZ_NLATTR_TYPE_OF(typ) SZ_NLATTR_DATA_OF(sizeof(typ))
 533 #define SZ_NLATTR_STRZ_OF(str) SZ_NLATTR_DATA_OF(strlen(str) + 1)
 534
 535 #define PTOI_TABLE_SZ_MAX(dev) (mlx5_dev_to_port_id((dev)->device, NULL, 0) + 2)
 536
 537 /** DPDK port to network interface index (ifindex) conversion. */
 538 struct flow_tcf_ptoi {
 539         uint16_t port_id; /**< DPDK port ID. */
 540         unsigned int ifindex; /**< Network interface index. */
 541 };
 542
 543 /* Due to a limitation on driver/FW. */
 544 #define MLX5_TCF_GROUP_ID_MAX 3
 545
 546 /*
 547  * Due to a limitation on driver/FW, priority ranges from 1 to 16 in kernel.
 548  * Priority in rte_flow attribute starts from 0 and is added by 1 in
 549  * translation. This is subject to be changed to determine the max priority
 550  * based on trial-and-error like Verbs driver once the restriction is lifted or
 551  * the range is extended.
 552  */
 553 #define MLX5_TCF_GROUP_PRIORITY_MAX 15
 554
 555 #define MLX5_TCF_FATE_ACTIONS \
 556         (MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
 557          MLX5_FLOW_ACTION_JUMP)
 558
 559 #define MLX5_TCF_VLAN_ACTIONS \
 560         (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN | \
 561          MLX5_FLOW_ACTION_OF_SET_VLAN_VID | MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
 562
 563 #define MLX5_TCF_VXLAN_ACTIONS \
 564         (MLX5_FLOW_ACTION_VXLAN_ENCAP | MLX5_FLOW_ACTION_VXLAN_DECAP)
 565
 566 #define MLX5_TCF_PEDIT_ACTIONS \
 567         (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST | \
 568          MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST | \
 569          MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST | \
 570          MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL | \
 571          MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)
 572
 573 #define MLX5_TCF_CONFIG_ACTIONS \
 574         (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_JUMP | \
 575          MLX5_FLOW_ACTION_OF_PUSH_VLAN | MLX5_FLOW_ACTION_OF_SET_VLAN_VID | \
 576          MLX5_FLOW_ACTION_OF_SET_VLAN_PCP | \
 577          (MLX5_TCF_PEDIT_ACTIONS & ~MLX5_FLOW_ACTION_DEC_TTL))
 578
 579 #define MAX_PEDIT_KEYS 128
 580 #define SZ_PEDIT_KEY_VAL 4
 581
 582 #define NUM_OF_PEDIT_KEYS(sz) \
 583         (((sz) / SZ_PEDIT_KEY_VAL) + (((sz) % SZ_PEDIT_KEY_VAL) ? 1 : 0))
 584
 585 struct pedit_key_ex {
 586         enum pedit_header_type htype;
 587         enum pedit_cmd cmd;
 588 };
 589
 590 struct pedit_parser {
 591         struct tc_pedit_sel sel;
 592         struct tc_pedit_key keys[MAX_PEDIT_KEYS];
 593         struct pedit_key_ex keys_ex[MAX_PEDIT_KEYS];
 594 };
 595
 596 /**
 597  * Create space for using the implicitly created TC flow counter.
 598  *
 599  * @param[in] dev
 600  *   Pointer to the Ethernet device structure.
 601  *
 602  * @return
 603  *   A pointer to the counter data structure, NULL otherwise and
 604  *   rte_errno is set.
 605  */
 606 static struct mlx5_flow_counter *
 607 flow_tcf_counter_new(void)
 608 {
 609         struct mlx5_flow_counter *cnt;
 610
 611         /*
 612          * eswitch counter cannot be shared and its id is unknown.
 613          * currently returning all with id 0.
 614          * in the future maybe better to switch to unique numbers.
 615          */
 616         struct mlx5_flow_counter tmpl = {
 617                 .ref_cnt = 1,
 618         };
 619         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 620         if (!cnt) {
 621                 rte_errno = ENOMEM;
 622                 return NULL;
 623         }
 624         *cnt = tmpl;
 625         /* Implicit counter, do not add to list. */
 626         return cnt;
 627 }
 628
 629 /**
 630  * Set pedit key of MAC address
 631  *
 632  * @param[in] actions
 633  *   pointer to action specification
 634  * @param[in,out] p_parser
 635  *   pointer to pedit_parser
 636  */
 637 static void
 638 flow_tcf_pedit_key_set_mac(const struct rte_flow_action *actions,
 639                            struct pedit_parser *p_parser)
 640 {
 641         int idx = p_parser->sel.nkeys;
 642         uint32_t off = actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
 643                                         offsetof(struct ether_hdr, s_addr) :
 644                                         offsetof(struct ether_hdr, d_addr);
 645         const struct rte_flow_action_set_mac *conf =
 646                 (const struct rte_flow_action_set_mac *)actions->conf;
 647
 648         p_parser->keys[idx].off = off;
 649         p_parser->keys[idx].mask = ~UINT32_MAX;
 650         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 651         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 652         memcpy(&p_parser->keys[idx].val,
 653                 conf->mac_addr, SZ_PEDIT_KEY_VAL);
 654         idx++;
 655         p_parser->keys[idx].off = off + SZ_PEDIT_KEY_VAL;
 656         p_parser->keys[idx].mask = 0xFFFF0000;
 657         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 658         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 659         memcpy(&p_parser->keys[idx].val,
 660                 conf->mac_addr + SZ_PEDIT_KEY_VAL,
 661                 ETHER_ADDR_LEN - SZ_PEDIT_KEY_VAL);
 662         p_parser->sel.nkeys = (++idx);
 663 }
 664
 665 /**
 666  * Set pedit key of decrease/set ttl
 667  *
 668  * @param[in] actions
 669  *   pointer to action specification
 670  * @param[in,out] p_parser
 671  *   pointer to pedit_parser
 672  * @param[in] item_flags
 673  *   flags of all items presented
 674  */
 675 static void
 676 flow_tcf_pedit_key_set_dec_ttl(const struct rte_flow_action *actions,
 677                                 struct pedit_parser *p_parser,
 678                                 uint64_t item_flags)
 679 {
 680         int idx = p_parser->sel.nkeys;
 681
 682         p_parser->keys[idx].mask = 0xFFFFFF00;
 683         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) {
 684                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 685                 p_parser->keys[idx].off =
 686                         offsetof(struct ipv4_hdr, time_to_live);
 687         }
 688         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6) {
 689                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 690                 p_parser->keys[idx].off =
 691                         offsetof(struct ipv6_hdr, hop_limits);
 692         }
 693         if (actions->type == RTE_FLOW_ACTION_TYPE_DEC_TTL) {
 694                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_ADD;
 695                 p_parser->keys[idx].val = 0x000000FF;
 696         } else {
 697                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 698                 p_parser->keys[idx].val =
 699                         (__u32)((const struct rte_flow_action_set_ttl *)
 700                          actions->conf)->ttl_value;
 701         }
 702         p_parser->sel.nkeys = (++idx);
 703 }
 704
 705 /**
 706  * Set pedit key of transport (TCP/UDP) port value
 707  *
 708  * @param[in] actions
 709  *   pointer to action specification
 710  * @param[in,out] p_parser
 711  *   pointer to pedit_parser
 712  * @param[in] item_flags
 713  *   flags of all items presented
 714  */
 715 static void
 716 flow_tcf_pedit_key_set_tp_port(const struct rte_flow_action *actions,
 717                                 struct pedit_parser *p_parser,
 718                                 uint64_t item_flags)
 719 {
 720         int idx = p_parser->sel.nkeys;
 721
 722         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 723                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_UDP;
 724         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 725                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_TCP;
 726         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 727         /* offset of src/dst port is same for TCP and UDP */
 728         p_parser->keys[idx].off =
 729                 actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
 730                 offsetof(struct tcp_hdr, src_port) :
 731                 offsetof(struct tcp_hdr, dst_port);
 732         p_parser->keys[idx].mask = 0xFFFF0000;
 733         p_parser->keys[idx].val =
 734                 (__u32)((const struct rte_flow_action_set_tp *)
 735                                 actions->conf)->port;
 736         p_parser->sel.nkeys = (++idx);
 737 }
 738
 739 /**
 740  * Set pedit key of ipv6 address
 741  *
 742  * @param[in] actions
 743  *   pointer to action specification
 744  * @param[in,out] p_parser
 745  *   pointer to pedit_parser
 746  */
 747 static void
 748 flow_tcf_pedit_key_set_ipv6_addr(const struct rte_flow_action *actions,
 749                                  struct pedit_parser *p_parser)
 750 {
 751         int idx = p_parser->sel.nkeys;
 752         int keys = NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 753         int off_base =
 754                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
 755                 offsetof(struct ipv6_hdr, src_addr) :
 756                 offsetof(struct ipv6_hdr, dst_addr);
 757         const struct rte_flow_action_set_ipv6 *conf =
 758                 (const struct rte_flow_action_set_ipv6 *)actions->conf;
 759
 760         for (int i = 0; i < keys; i++, idx++) {
 761                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 762                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 763                 p_parser->keys[idx].off = off_base + i * SZ_PEDIT_KEY_VAL;
 764                 p_parser->keys[idx].mask = ~UINT32_MAX;
 765                 memcpy(&p_parser->keys[idx].val,
 766                         conf->ipv6_addr + i *  SZ_PEDIT_KEY_VAL,
 767                         SZ_PEDIT_KEY_VAL);
 768         }
 769         p_parser->sel.nkeys += keys;
 770 }
 771
 772 /**
 773  * Set pedit key of ipv4 address
 774  *
 775  * @param[in] actions
 776  *   pointer to action specification
 777  * @param[in,out] p_parser
 778  *   pointer to pedit_parser
 779  */
 780 static void
 781 flow_tcf_pedit_key_set_ipv4_addr(const struct rte_flow_action *actions,
 782                                  struct pedit_parser *p_parser)
 783 {
 784         int idx = p_parser->sel.nkeys;
 785
 786         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 787         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 788         p_parser->keys[idx].off =
 789                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
 790                 offsetof(struct ipv4_hdr, src_addr) :
 791                 offsetof(struct ipv4_hdr, dst_addr);
 792         p_parser->keys[idx].mask = ~UINT32_MAX;
 793         p_parser->keys[idx].val =
 794                 ((const struct rte_flow_action_set_ipv4 *)
 795                  actions->conf)->ipv4_addr;
 796         p_parser->sel.nkeys = (++idx);
 797 }
 798
 799 /**
 800  * Create the pedit's na attribute in netlink message
 801  * on pre-allocate message buffer
 802  *
 803  * @param[in,out] nl
 804  *   pointer to pre-allocated netlink message buffer
 805  * @param[in,out] actions
 806  *   pointer to pointer of actions specification.
 807  * @param[in,out] action_flags
 808  *   pointer to actions flags
 809  * @param[in] item_flags
 810  *   flags of all item presented
 811  */
 812 static void
 813 flow_tcf_create_pedit_mnl_msg(struct nlmsghdr *nl,
 814                               const struct rte_flow_action **actions,
 815                               uint64_t item_flags)
 816 {
 817         struct pedit_parser p_parser;
 818         struct nlattr *na_act_options;
 819         struct nlattr *na_pedit_keys;
 820
 821         memset(&p_parser, 0, sizeof(p_parser));
 822         mnl_attr_put_strz(nl, TCA_ACT_KIND, "pedit");
 823         na_act_options = mnl_attr_nest_start(nl, TCA_ACT_OPTIONS);
 824         /* all modify header actions should be in one tc-pedit action */
 825         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 826                 switch ((*actions)->type) {
 827                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 828                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 829                         flow_tcf_pedit_key_set_ipv4_addr(*actions, &p_parser);
 830                         break;
 831                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 832                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 833                         flow_tcf_pedit_key_set_ipv6_addr(*actions, &p_parser);
 834                         break;
 835                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 836                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 837                         flow_tcf_pedit_key_set_tp_port(*actions,
 838                                                         &p_parser, item_flags);
 839                         break;
 840                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 841                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 842                         flow_tcf_pedit_key_set_dec_ttl(*actions,
 843                                                         &p_parser, item_flags);
 844                         break;
 845                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 846                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 847                         flow_tcf_pedit_key_set_mac(*actions, &p_parser);
 848                         break;
 849                 default:
 850                         goto pedit_mnl_msg_done;
 851                 }
 852         }
 853 pedit_mnl_msg_done:
 854         p_parser.sel.action = TC_ACT_PIPE;
 855         mnl_attr_put(nl, TCA_PEDIT_PARMS_EX,
 856                      sizeof(p_parser.sel) +
 857                      p_parser.sel.nkeys * sizeof(struct tc_pedit_key),
 858                      &p_parser);
 859         na_pedit_keys =
 860                 mnl_attr_nest_start(nl, TCA_PEDIT_KEYS_EX | NLA_F_NESTED);
 861         for (int i = 0; i < p_parser.sel.nkeys; i++) {
 862                 struct nlattr *na_pedit_key =
 863                         mnl_attr_nest_start(nl,
 864                                             TCA_PEDIT_KEY_EX | NLA_F_NESTED);
 865                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_HTYPE,
 866                                  p_parser.keys_ex[i].htype);
 867                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_CMD,
 868                                  p_parser.keys_ex[i].cmd);
 869                 mnl_attr_nest_end(nl, na_pedit_key);
 870         }
 871         mnl_attr_nest_end(nl, na_pedit_keys);
 872         mnl_attr_nest_end(nl, na_act_options);
 873         (*actions)--;
 874 }
 875
 876 /**
 877  * Calculate max memory size of one TC-pedit actions.
 878  * One TC-pedit action can contain set of keys each defining
 879  * a rewrite element (rte_flow action)
 880  *
 881  * @param[in,out] actions
 882  *   actions specification.
 883  * @param[in,out] action_flags
 884  *   actions flags
 885  * @param[in,out] size
 886  *   accumulated size
 887  * @return
 888  *   Max memory size of one TC-pedit action
 889  */
 890 static int
 891 flow_tcf_get_pedit_actions_size(const struct rte_flow_action **actions,
 892                                 uint64_t *action_flags)
 893 {
 894         int pedit_size = 0;
 895         int keys = 0;
 896         uint64_t flags = 0;
 897
 898         pedit_size += SZ_NLATTR_NEST + /* na_act_index. */
 899                       SZ_NLATTR_STRZ_OF("pedit") +
 900                       SZ_NLATTR_NEST; /* TCA_ACT_OPTIONS. */
 901         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 902                 switch ((*actions)->type) {
 903                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 904                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 905                         flags |= MLX5_FLOW_ACTION_SET_IPV4_SRC;
 906                         break;
 907                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 908                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 909                         flags |= MLX5_FLOW_ACTION_SET_IPV4_DST;
 910                         break;
 911                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 912                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 913                         flags |= MLX5_FLOW_ACTION_SET_IPV6_SRC;
 914                         break;
 915                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 916                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 917                         flags |= MLX5_FLOW_ACTION_SET_IPV6_DST;
 918                         break;
 919                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 920                         /* TCP is as same as UDP */
 921                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 922                         flags |= MLX5_FLOW_ACTION_SET_TP_SRC;
 923                         break;
 924                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 925                         /* TCP is as same as UDP */
 926                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 927                         flags |= MLX5_FLOW_ACTION_SET_TP_DST;
 928                         break;
 929                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 930                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 931                         flags |= MLX5_FLOW_ACTION_SET_TTL;
 932                         break;
 933                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 934                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 935                         flags |= MLX5_FLOW_ACTION_DEC_TTL;
 936                         break;
 937                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 938                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 939                         flags |= MLX5_FLOW_ACTION_SET_MAC_SRC;
 940                         break;
 941                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 942                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 943                         flags |= MLX5_FLOW_ACTION_SET_MAC_DST;
 944                         break;
 945                 default:
 946                         goto get_pedit_action_size_done;
 947                 }
 948         }
 949 get_pedit_action_size_done:
 950         /* TCA_PEDIT_PARAMS_EX */
 951         pedit_size +=
 952                 SZ_NLATTR_DATA_OF(sizeof(struct tc_pedit_sel) +
 953                                   keys * sizeof(struct tc_pedit_key));
 954         pedit_size += SZ_NLATTR_NEST; /* TCA_PEDIT_KEYS */
 955         pedit_size += keys *
 956                       /* TCA_PEDIT_KEY_EX + HTYPE + CMD */
 957                       (SZ_NLATTR_NEST + SZ_NLATTR_DATA_OF(2) +
 958                        SZ_NLATTR_DATA_OF(2));
 959         (*action_flags) |= flags;
 960         (*actions)--;
 961         return pedit_size;
 962 }
 963
 964 /**
 965  * Retrieve mask for pattern item.
 966  *
 967  * This function does basic sanity checks on a pattern item in order to
 968  * return the most appropriate mask for it.
 969  *
 970  * @param[in] item
 971  *   Item specification.
 972  * @param[in] mask_default
 973  *   Default mask for pattern item as specified by the flow API.
 974  * @param[in] mask_supported
 975  *   Mask fields supported by the implementation.
 976  * @param[in] mask_empty
 977  *   Empty mask to return when there is no specification.
 978  * @param[out] error
 979  *   Perform verbose error reporting if not NULL.
 980  *
 981  * @return
 982  *   Either @p item->mask or one of the mask parameters on success, NULL
 983  *   otherwise and rte_errno is set.
 984  */
 985 static const void *
 986 flow_tcf_item_mask(const struct rte_flow_item *item, const void *mask_default,
 987                    const void *mask_supported, const void *mask_empty,
 988                    size_t mask_size, struct rte_flow_error *error)
 989 {
 990         const uint8_t *mask;
 991         size_t i;
 992
 993         /* item->last and item->mask cannot exist without item->spec. */
 994         if (!item->spec && (item->mask || item->last)) {
 995                 rte_flow_error_set(error, EINVAL,
 996                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 997                                    "\"mask\" or \"last\" field provided without"
 998                                    " a corresponding \"spec\"");
 999                 return NULL;
1000         }
1001         /* No spec, no mask, no problem. */
1002         if (!item->spec)
1003                 return mask_empty;
1004         mask = item->mask ? item->mask : mask_default;
1005         assert(mask);
1006         /*
1007          * Single-pass check to make sure that:
1008          * - Mask is supported, no bits are set outside mask_supported.
1009          * - Both item->spec and item->last are included in mask.
1010          */
1011         for (i = 0; i != mask_size; ++i) {
1012                 if (!mask[i])
1013                         continue;
1014                 if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
1015                     ((const uint8_t *)mask_supported)[i]) {
1016                         rte_flow_error_set(error, ENOTSUP,
1017                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1018                                            "unsupported field found"
1019                                            " in \"mask\"");
1020                         return NULL;
1021                 }
1022                 if (item->last &&
1023                     (((const uint8_t *)item->spec)[i] & mask[i]) !=
1024                     (((const uint8_t *)item->last)[i] & mask[i])) {
1025                         rte_flow_error_set(error, EINVAL,
1026                                            RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1027                                            item->last,
1028                                            "range between \"spec\" and \"last\""
1029                                            " not comprised in \"mask\"");
1030                         return NULL;
1031                 }
1032         }
1033         return mask;
1034 }
1035
1036 /**
1037  * Build a conversion table between port ID and ifindex.
1038  *
1039  * @param[in] dev
1040  *   Pointer to Ethernet device.
1041  * @param[out] ptoi
1042  *   Pointer to ptoi table.
1043  * @param[in] len
1044  *   Size of ptoi table provided.
1045  *
1046  * @return
1047  *   Size of ptoi table filled.
1048  */
1049 static unsigned int
1050 flow_tcf_build_ptoi_table(struct rte_eth_dev *dev, struct flow_tcf_ptoi *ptoi,
1051                           unsigned int len)
1052 {
1053         unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
1054         uint16_t port_id[n + 1];
1055         unsigned int i;
1056         unsigned int own = 0;
1057
1058         /* At least one port is needed when no switch domain is present. */
1059         if (!n) {
1060                 n = 1;
1061                 port_id[0] = dev->data->port_id;
1062         } else {
1063                 n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
1064         }
1065         if (n > len)
1066                 return 0;
1067         for (i = 0; i != n; ++i) {
1068                 struct rte_eth_dev_info dev_info;
1069
1070                 rte_eth_dev_info_get(port_id[i], &dev_info);
1071                 if (port_id[i] == dev->data->port_id)
1072                         own = i;
1073                 ptoi[i].port_id = port_id[i];
1074                 ptoi[i].ifindex = dev_info.if_index;
1075         }
1076         /* Ensure first entry of ptoi[] is the current device. */
1077         if (own) {
1078                 ptoi[n] = ptoi[0];
1079                 ptoi[0] = ptoi[own];
1080                 ptoi[own] = ptoi[n];
1081         }
1082         /* An entry with zero ifindex terminates ptoi[]. */
1083         ptoi[n].port_id = 0;
1084         ptoi[n].ifindex = 0;
1085         return n;
1086 }
1087
1088 /**
1089  * Verify the @p attr will be correctly understood by the E-switch.
1090  *
1091  * @param[in] attr
1092  *   Pointer to flow attributes
1093  * @param[out] error
1094  *   Pointer to error structure.
1095  *
1096  * @return
1097  *   0 on success, a negative errno value otherwise and rte_errno is set.
1098  */
1099 static int
1100 flow_tcf_validate_attributes(const struct rte_flow_attr *attr,
1101                              struct rte_flow_error *error)
1102 {
1103         /*
1104          * Supported attributes: groups, some priorities and ingress only.
1105          * group is supported only if kernel supports chain. Don't care about
1106          * transfer as it is the caller's problem.
1107          */
1108         if (attr->group > MLX5_TCF_GROUP_ID_MAX)
1109                 return rte_flow_error_set(error, ENOTSUP,
1110                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1111                                           "group ID larger than "
1112                                           RTE_STR(MLX5_TCF_GROUP_ID_MAX)
1113                                           " isn't supported");
1114         else if (attr->priority > MLX5_TCF_GROUP_PRIORITY_MAX)
1115                 return rte_flow_error_set(error, ENOTSUP,
1116                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1117                                           attr,
1118                                           "priority more than "
1119                                           RTE_STR(MLX5_TCF_GROUP_PRIORITY_MAX)
1120                                           " is not supported");
1121         if (!attr->ingress)
1122                 return rte_flow_error_set(error, EINVAL,
1123                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1124                                           attr, "only ingress is supported");
1125         if (attr->egress)
1126                 return rte_flow_error_set(error, ENOTSUP,
1127                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1128                                           attr, "egress is not supported");
1129         return 0;
1130 }
1131
1132 /**
1133  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_ETH item for E-Switch.
1134  * The routine checks the L2 fields to be used in encapsulation header.
1135  *
1136  * @param[in] item
1137  *   Pointer to the item structure.
1138  * @param[out] error
1139  *   Pointer to the error structure.
1140  *
1141  * @return
1142  *   0 on success, a negative errno value otherwise and rte_errno is set.
1143  **/
1144 static int
1145 flow_tcf_validate_vxlan_encap_eth(const struct rte_flow_item *item,
1146                                   struct rte_flow_error *error)
1147 {
1148         const struct rte_flow_item_eth *spec = item->spec;
1149         const struct rte_flow_item_eth *mask = item->mask;
1150
1151         if (!spec) {
1152                 /*
1153                  * Specification for L2 addresses can be empty
1154                  * because these ones are optional and not
1155                  * required directly by tc rule. Kernel tries
1156                  * to resolve these ones on its own
1157                  */
1158                 return 0;
1159         }
1160         if (!mask) {
1161                 /* If mask is not specified use the default one. */
1162                 mask = &rte_flow_item_eth_mask;
1163         }
1164         if (memcmp(&mask->dst,
1165                    &flow_tcf_mask_empty.eth.dst,
1166                    sizeof(flow_tcf_mask_empty.eth.dst))) {
1167                 if (memcmp(&mask->dst,
1168                            &rte_flow_item_eth_mask.dst,
1169                            sizeof(rte_flow_item_eth_mask.dst)))
1170                         return rte_flow_error_set
1171                                 (error, ENOTSUP,
1172                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1173                                  "no support for partial mask on"
1174                                  " \"eth.dst\" field");
1175         }
1176         if (memcmp(&mask->src,
1177                    &flow_tcf_mask_empty.eth.src,
1178                    sizeof(flow_tcf_mask_empty.eth.src))) {
1179                 if (memcmp(&mask->src,
1180                            &rte_flow_item_eth_mask.src,
1181                            sizeof(rte_flow_item_eth_mask.src)))
1182                         return rte_flow_error_set
1183                                 (error, ENOTSUP,
1184                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1185                                  "no support for partial mask on"
1186                                  " \"eth.src\" field");
1187         }
1188         if (mask->type != RTE_BE16(0x0000)) {
1189                 if (mask->type != RTE_BE16(0xffff))
1190                         return rte_flow_error_set
1191                                 (error, ENOTSUP,
1192                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1193                                  "no support for partial mask on"
1194                                  " \"eth.type\" field");
1195                 DRV_LOG(WARNING,
1196                         "outer ethernet type field"
1197                         " cannot be forced for vxlan"
1198                         " encapsulation, parameter ignored");
1199         }
1200         return 0;
1201 }
1202
1203 /**
1204  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV4 item for E-Switch.
1205  * The routine checks the IPv4 fields to be used in encapsulation header.
1206  *
1207  * @param[in] item
1208  *   Pointer to the item structure.
1209  * @param[out] error
1210  *   Pointer to the error structure.
1211  *
1212  * @return
1213  *   0 on success, a negative errno value otherwise and rte_errno is set.
1214  **/
1215 static int
1216 flow_tcf_validate_vxlan_encap_ipv4(const struct rte_flow_item *item,
1217                                    struct rte_flow_error *error)
1218 {
1219         const struct rte_flow_item_ipv4 *spec = item->spec;
1220         const struct rte_flow_item_ipv4 *mask = item->mask;
1221
1222         if (!spec) {
1223                 /*
1224                  * Specification for IP addresses cannot be empty
1225                  * because it is required by tunnel_key parameter.
1226                  */
1227                 return rte_flow_error_set(error, EINVAL,
1228                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1229                                           "NULL outer ipv4 address"
1230                                           " specification for vxlan"
1231                                           " encapsulation");
1232         }
1233         if (!mask)
1234                 mask = &rte_flow_item_ipv4_mask;
1235         if (mask->hdr.dst_addr != RTE_BE32(0x00000000)) {
1236                 if (mask->hdr.dst_addr != RTE_BE32(0xffffffff))
1237                         return rte_flow_error_set
1238                                 (error, ENOTSUP,
1239                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1240                                  "no support for partial mask on"
1241                                  " \"ipv4.hdr.dst_addr\" field"
1242                                  " for vxlan encapsulation");
1243                 /* More IPv4 address validations can be put here. */
1244         } else {
1245                 /*
1246                  * Kernel uses the destination IP address to determine
1247                  * the routing path and obtain the MAC destination
1248                  * address, so IP destination address must be
1249                  * specified in the tc rule.
1250                  */
1251                 return rte_flow_error_set(error, EINVAL,
1252                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1253                                           "outer ipv4 destination address"
1254                                           " must be specified for"
1255                                           " vxlan encapsulation");
1256         }
1257         if (mask->hdr.src_addr != RTE_BE32(0x00000000)) {
1258                 if (mask->hdr.src_addr != RTE_BE32(0xffffffff))
1259                         return rte_flow_error_set
1260                                 (error, ENOTSUP,
1261                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1262                                  "no support for partial mask on"
1263                                  " \"ipv4.hdr.src_addr\" field"
1264                                  " for vxlan encapsulation");
1265                 /* More IPv4 address validations can be put here. */
1266         } else {
1267                 /*
1268                  * Kernel uses the source IP address to select the
1269                  * interface for egress encapsulated traffic, so
1270                  * it must be specified in the tc rule.
1271                  */
1272                 return rte_flow_error_set(error, EINVAL,
1273                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1274                                           "outer ipv4 source address"
1275                                           " must be specified for"
1276                                           " vxlan encapsulation");
1277         }
1278         return 0;
1279 }
1280
1281 /**
1282  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV6 item for E-Switch.
1283  * The routine checks the IPv6 fields to be used in encapsulation header.
1284  *
1285  * @param[in] item
1286  *   Pointer to the item structure.
1287  * @param[out] error
1288  *   Pointer to the error structure.
1289  *
1290  * @return
1291  *   0 on success, a negative errno value otherwise and rte_errno is set.
1292  **/
1293 static int
1294 flow_tcf_validate_vxlan_encap_ipv6(const struct rte_flow_item *item,
1295                                    struct rte_flow_error *error)
1296 {
1297         const struct rte_flow_item_ipv6 *spec = item->spec;
1298         const struct rte_flow_item_ipv6 *mask = item->mask;
1299
1300         if (!spec) {
1301                 /*
1302                  * Specification for IP addresses cannot be empty
1303                  * because it is required by tunnel_key parameter.
1304                  */
1305                 return rte_flow_error_set(error, EINVAL,
1306                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1307                                           "NULL outer ipv6 address"
1308                                           " specification for"
1309                                           " vxlan encapsulation");
1310         }
1311         if (!mask)
1312                 mask = &rte_flow_item_ipv6_mask;
1313         if (memcmp(&mask->hdr.dst_addr,
1314                    &flow_tcf_mask_empty.ipv6.hdr.dst_addr,
1315                    IPV6_ADDR_LEN)) {
1316                 if (memcmp(&mask->hdr.dst_addr,
1317                            &rte_flow_item_ipv6_mask.hdr.dst_addr,
1318                            IPV6_ADDR_LEN))
1319                         return rte_flow_error_set
1320                                         (error, ENOTSUP,
1321                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1322                                          "no support for partial mask on"
1323                                          " \"ipv6.hdr.dst_addr\" field"
1324                                          " for vxlan encapsulation");
1325                 /* More IPv6 address validations can be put here. */
1326         } else {
1327                 /*
1328                  * Kernel uses the destination IP address to determine
1329                  * the routing path and obtain the MAC destination
1330                  * address (heigh or gate), so IP destination address
1331                  * must be specified within the tc rule.
1332                  */
1333                 return rte_flow_error_set(error, EINVAL,
1334                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1335                                           "outer ipv6 destination address"
1336                                           " must be specified for"
1337                                           " vxlan encapsulation");
1338         }
1339         if (memcmp(&mask->hdr.src_addr,
1340                    &flow_tcf_mask_empty.ipv6.hdr.src_addr,
1341                    IPV6_ADDR_LEN)) {
1342                 if (memcmp(&mask->hdr.src_addr,
1343                            &rte_flow_item_ipv6_mask.hdr.src_addr,
1344                            IPV6_ADDR_LEN))
1345                         return rte_flow_error_set
1346                                         (error, ENOTSUP,
1347                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1348                                          "no support for partial mask on"
1349                                          " \"ipv6.hdr.src_addr\" field"
1350                                          " for vxlan encapsulation");
1351                 /* More L3 address validation can be put here. */
1352         } else {
1353                 /*
1354                  * Kernel uses the source IP address to select the
1355                  * interface for egress encapsulated traffic, so
1356                  * it must be specified in the tc rule.
1357                  */
1358                 return rte_flow_error_set(error, EINVAL,
1359                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1360                                           "outer L3 source address"
1361                                           " must be specified for"
1362                                           " vxlan encapsulation");
1363         }
1364         return 0;
1365 }
1366
1367 /**
1368  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_UDP item for E-Switch.
1369  * The routine checks the UDP fields to be used in encapsulation header.
1370  *
1371  * @param[in] item
1372  *   Pointer to the item structure.
1373  * @param[out] error
1374  *   Pointer to the error structure.
1375  *
1376  * @return
1377  *   0 on success, a negative errno value otherwise and rte_errno is set.
1378  **/
1379 static int
1380 flow_tcf_validate_vxlan_encap_udp(const struct rte_flow_item *item,
1381                                   struct rte_flow_error *error)
1382 {
1383         const struct rte_flow_item_udp *spec = item->spec;
1384         const struct rte_flow_item_udp *mask = item->mask;
1385
1386         if (!spec) {
1387                 /*
1388                  * Specification for UDP ports cannot be empty
1389                  * because it is required by tunnel_key parameter.
1390                  */
1391                 return rte_flow_error_set(error, EINVAL,
1392                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1393                                           "NULL UDP port specification "
1394                                           " for vxlan encapsulation");
1395         }
1396         if (!mask)
1397                 mask = &rte_flow_item_udp_mask;
1398         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1399                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1400                         return rte_flow_error_set
1401                                         (error, ENOTSUP,
1402                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1403                                          "no support for partial mask on"
1404                                          " \"udp.hdr.dst_port\" field"
1405                                          " for vxlan encapsulation");
1406                 if (!spec->hdr.dst_port)
1407                         return rte_flow_error_set
1408                                         (error, EINVAL,
1409                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
1410                                          "outer UDP remote port cannot be"
1411                                          " 0 for vxlan encapsulation");
1412         } else {
1413                 return rte_flow_error_set(error, EINVAL,
1414                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1415                                           "outer UDP remote port"
1416                                           " must be specified for"
1417                                           " vxlan encapsulation");
1418         }
1419         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1420                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1421                         return rte_flow_error_set
1422                                         (error, ENOTSUP,
1423                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1424                                          "no support for partial mask on"
1425                                          " \"udp.hdr.src_port\" field"
1426                                          " for vxlan encapsulation");
1427                 DRV_LOG(WARNING,
1428                         "outer UDP source port cannot be"
1429                         " forced for vxlan encapsulation,"
1430                         " parameter ignored");
1431         }
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_VXLAN item for E-Switch.
1437  * The routine checks the VNIP fields to be used in encapsulation header.
1438  *
1439  * @param[in] item
1440  *   Pointer to the item structure.
1441  * @param[out] error
1442  *   Pointer to the error structure.
1443  *
1444  * @return
1445  *   0 on success, a negative errno value otherwise and rte_errno is set.
1446  **/
1447 static int
1448 flow_tcf_validate_vxlan_encap_vni(const struct rte_flow_item *item,
1449                                   struct rte_flow_error *error)
1450 {
1451         const struct rte_flow_item_vxlan *spec = item->spec;
1452         const struct rte_flow_item_vxlan *mask = item->mask;
1453
1454         if (!spec) {
1455                 /* Outer VNI is required by tunnel_key parameter. */
1456                 return rte_flow_error_set(error, EINVAL,
1457                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1458                                           "NULL VNI specification"
1459                                           " for vxlan encapsulation");
1460         }
1461         if (!mask)
1462                 mask = &rte_flow_item_vxlan_mask;
1463         if (!mask->vni[0] && !mask->vni[1] && !mask->vni[2])
1464                 return rte_flow_error_set(error, EINVAL,
1465                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1466                                           "outer VNI must be specified "
1467                                           "for vxlan encapsulation");
1468         if (mask->vni[0] != 0xff ||
1469             mask->vni[1] != 0xff ||
1470             mask->vni[2] != 0xff)
1471                 return rte_flow_error_set(error, ENOTSUP,
1472                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1473                                           "no support for partial mask on"
1474                                           " \"vxlan.vni\" field");
1475
1476         if (!spec->vni[0] && !spec->vni[1] && !spec->vni[2])
1477                 return rte_flow_error_set(error, EINVAL,
1478                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1479                                           "vxlan vni cannot be 0");
1480         return 0;
1481 }
1482
1483 /**
1484  * Validate VXLAN_ENCAP action item list for E-Switch.
1485  * The routine checks items to be used in encapsulation header.
1486  *
1487  * @param[in] action
1488  *   Pointer to the VXLAN_ENCAP action structure.
1489  * @param[out] error
1490  *   Pointer to the error structure.
1491  *
1492  * @return
1493  *   0 on success, a negative errno value otherwise and rte_errno is set.
1494  **/
1495 static int
1496 flow_tcf_validate_vxlan_encap(const struct rte_flow_action *action,
1497                               struct rte_flow_error *error)
1498 {
1499         const struct rte_flow_item *items;
1500         int ret;
1501         uint32_t item_flags = 0;
1502
1503         if (!action->conf)
1504                 return rte_flow_error_set(error, EINVAL,
1505                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1506                                           "Missing vxlan tunnel"
1507                                           " action configuration");
1508         items = ((const struct rte_flow_action_vxlan_encap *)
1509                                         action->conf)->definition;
1510         if (!items)
1511                 return rte_flow_error_set(error, EINVAL,
1512                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1513                                           "Missing vxlan tunnel"
1514                                           " encapsulation parameters");
1515         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1516                 switch (items->type) {
1517                 case RTE_FLOW_ITEM_TYPE_VOID:
1518                         break;
1519                 case RTE_FLOW_ITEM_TYPE_ETH:
1520                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1521                                                           error);
1522                         if (ret < 0)
1523                                 return ret;
1524                         ret = flow_tcf_validate_vxlan_encap_eth(items, error);
1525                         if (ret < 0)
1526                                 return ret;
1527                         item_flags |= MLX5_FLOW_LAYER_OUTER_L2;
1528                         break;
1529                 break;
1530                 case RTE_FLOW_ITEM_TYPE_IPV4:
1531                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1532                                                            error);
1533                         if (ret < 0)
1534                                 return ret;
1535                         ret = flow_tcf_validate_vxlan_encap_ipv4(items, error);
1536                         if (ret < 0)
1537                                 return ret;
1538                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1539                         break;
1540                 case RTE_FLOW_ITEM_TYPE_IPV6:
1541                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1542                                                            error);
1543                         if (ret < 0)
1544                                 return ret;
1545                         ret = flow_tcf_validate_vxlan_encap_ipv6(items, error);
1546                         if (ret < 0)
1547                                 return ret;
1548                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1549                         break;
1550                 case RTE_FLOW_ITEM_TYPE_UDP:
1551                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1552                                                            0xFF, error);
1553                         if (ret < 0)
1554                                 return ret;
1555                         ret = flow_tcf_validate_vxlan_encap_udp(items, error);
1556                         if (ret < 0)
1557                                 return ret;
1558                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
1559                         break;
1560                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1561                         ret = mlx5_flow_validate_item_vxlan(items,
1562                                                             item_flags, error);
1563                         if (ret < 0)
1564                                 return ret;
1565                         ret = flow_tcf_validate_vxlan_encap_vni(items, error);
1566                         if (ret < 0)
1567                                 return ret;
1568                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1569                         break;
1570                 default:
1571                         return rte_flow_error_set
1572                                         (error, ENOTSUP,
1573                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1574                                          "vxlan encap item not supported");
1575                 }
1576         }
1577         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1578                 return rte_flow_error_set(error, EINVAL,
1579                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1580                                           "no outer IP layer found"
1581                                           " for vxlan encapsulation");
1582         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1583                 return rte_flow_error_set(error, EINVAL,
1584                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1585                                           "no outer UDP layer found"
1586                                           " for vxlan encapsulation");
1587         if (!(item_flags & MLX5_FLOW_LAYER_VXLAN))
1588                 return rte_flow_error_set(error, EINVAL,
1589                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1590                                           "no VXLAN VNI found"
1591                                           " for vxlan encapsulation");
1592         return 0;
1593 }
1594
1595 /**
1596  * Validate outer RTE_FLOW_ITEM_TYPE_UDP item if tunnel item
1597  * RTE_FLOW_ITEM_TYPE_VXLAN is present in item list.
1598  *
1599  * @param[in] udp
1600  *   Outer UDP layer item (if any, NULL otherwise).
1601  * @param[out] error
1602  *   Pointer to the error structure.
1603  *
1604  * @return
1605  *   0 on success, a negative errno value otherwise and rte_errno is set.
1606  **/
1607 static int
1608 flow_tcf_validate_vxlan_decap_udp(const struct rte_flow_item *udp,
1609                                   struct rte_flow_error *error)
1610 {
1611         const struct rte_flow_item_udp *spec = udp->spec;
1612         const struct rte_flow_item_udp *mask = udp->mask;
1613
1614         if (!spec)
1615                 /*
1616                  * Specification for UDP ports cannot be empty
1617                  * because it is required as decap parameter.
1618                  */
1619                 return rte_flow_error_set(error, EINVAL,
1620                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1621                                           "NULL UDP port specification"
1622                                           " for VXLAN decapsulation");
1623         if (!mask)
1624                 mask = &rte_flow_item_udp_mask;
1625         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1626                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1627                         return rte_flow_error_set
1628                                         (error, ENOTSUP,
1629                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1630                                          "no support for partial mask on"
1631                                          " \"udp.hdr.dst_port\" field");
1632                 if (!spec->hdr.dst_port)
1633                         return rte_flow_error_set
1634                                         (error, EINVAL,
1635                                          RTE_FLOW_ERROR_TYPE_ITEM, udp,
1636                                          "zero decap local UDP port");
1637         } else {
1638                 return rte_flow_error_set(error, EINVAL,
1639                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1640                                           "outer UDP destination port must be "
1641                                           "specified for vxlan decapsulation");
1642         }
1643         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1644                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1645                         return rte_flow_error_set
1646                                         (error, ENOTSUP,
1647                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1648                                          "no support for partial mask on"
1649                                          " \"udp.hdr.src_port\" field");
1650                 DRV_LOG(WARNING,
1651                         "outer UDP local port cannot be "
1652                         "forced for VXLAN encapsulation, "
1653                         "parameter ignored");
1654         }
1655         return 0;
1656 }
1657
1658 /**
1659  * Validate flow for E-Switch.
1660  *
1661  * @param[in] priv
1662  *   Pointer to the priv structure.
1663  * @param[in] attr
1664  *   Pointer to the flow attributes.
1665  * @param[in] items
1666  *   Pointer to the list of items.
1667  * @param[in] actions
1668  *   Pointer to the list of actions.
1669  * @param[out] error
1670  *   Pointer to the error structure.
1671  *
1672  * @return
1673  *   0 on success, a negative errno value otherwise and rte_errno is set.
1674  */
1675 static int
1676 flow_tcf_validate(struct rte_eth_dev *dev,
1677                   const struct rte_flow_attr *attr,
1678                   const struct rte_flow_item items[],
1679                   const struct rte_flow_action actions[],
1680                   struct rte_flow_error *error)
1681 {
1682         union {
1683                 const struct rte_flow_item_port_id *port_id;
1684                 const struct rte_flow_item_eth *eth;
1685                 const struct rte_flow_item_vlan *vlan;
1686                 const struct rte_flow_item_ipv4 *ipv4;
1687                 const struct rte_flow_item_ipv6 *ipv6;
1688                 const struct rte_flow_item_tcp *tcp;
1689                 const struct rte_flow_item_udp *udp;
1690                 const struct rte_flow_item_vxlan *vxlan;
1691         } spec, mask;
1692         union {
1693                 const struct rte_flow_action_port_id *port_id;
1694                 const struct rte_flow_action_jump *jump;
1695                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
1696                 const struct rte_flow_action_of_set_vlan_vid *
1697                         of_set_vlan_vid;
1698                 const struct rte_flow_action_of_set_vlan_pcp *
1699                         of_set_vlan_pcp;
1700                 const struct rte_flow_action_vxlan_encap *vxlan_encap;
1701                 const struct rte_flow_action_set_ipv4 *set_ipv4;
1702                 const struct rte_flow_action_set_ipv6 *set_ipv6;
1703         } conf;
1704         const struct rte_flow_item *outer_udp = NULL;
1705         rte_be16_t inner_etype = RTE_BE16(ETH_P_ALL);
1706         rte_be16_t outer_etype = RTE_BE16(ETH_P_ALL);
1707         rte_be16_t vlan_etype = RTE_BE16(ETH_P_ALL);
1708         uint64_t item_flags = 0;
1709         uint64_t action_flags = 0;
1710         uint8_t next_protocol = 0xff;
1711         unsigned int tcm_ifindex = 0;
1712         uint8_t pedit_validated = 0;
1713         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
1714         struct rte_eth_dev *port_id_dev = NULL;
1715         bool in_port_id_set;
1716         int ret;
1717
1718         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
1719                                                 PTOI_TABLE_SZ_MAX(dev)));
1720         ret = flow_tcf_validate_attributes(attr, error);
1721         if (ret < 0)
1722                 return ret;
1723         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1724                 unsigned int i;
1725                 uint64_t current_action_flag = 0;
1726
1727                 switch (actions->type) {
1728                 case RTE_FLOW_ACTION_TYPE_VOID:
1729                         break;
1730                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
1731                         current_action_flag = MLX5_FLOW_ACTION_PORT_ID;
1732                         if (!actions->conf)
1733                                 break;
1734                         conf.port_id = actions->conf;
1735                         if (conf.port_id->original)
1736                                 i = 0;
1737                         else
1738                                 for (i = 0; ptoi[i].ifindex; ++i)
1739                                         if (ptoi[i].port_id == conf.port_id->id)
1740                                                 break;
1741                         if (!ptoi[i].ifindex)
1742                                 return rte_flow_error_set
1743                                         (error, ENODEV,
1744                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1745                                          conf.port_id,
1746                                          "missing data to convert port ID to"
1747                                          " ifindex");
1748                         port_id_dev = &rte_eth_devices[conf.port_id->id];
1749                         break;
1750                 case RTE_FLOW_ACTION_TYPE_JUMP:
1751                         current_action_flag = MLX5_FLOW_ACTION_JUMP;
1752                         if (!actions->conf)
1753                                 break;
1754                         conf.jump = actions->conf;
1755                         if (attr->group >= conf.jump->group)
1756                                 return rte_flow_error_set
1757                                         (error, ENOTSUP,
1758                                          RTE_FLOW_ERROR_TYPE_ACTION,
1759                                          actions,
1760                                          "can jump only to a group forward");
1761                         break;
1762                 case RTE_FLOW_ACTION_TYPE_DROP:
1763                         current_action_flag = MLX5_FLOW_ACTION_DROP;
1764                         break;
1765                 case RTE_FLOW_ACTION_TYPE_COUNT:
1766                         break;
1767                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
1768                         current_action_flag = MLX5_FLOW_ACTION_OF_POP_VLAN;
1769                         break;
1770                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1771                         rte_be16_t ethertype;
1772
1773                         current_action_flag = MLX5_FLOW_ACTION_OF_PUSH_VLAN;
1774                         if (!actions->conf)
1775                                 break;
1776                         conf.of_push_vlan = actions->conf;
1777                         ethertype = conf.of_push_vlan->ethertype;
1778                         if (ethertype != RTE_BE16(ETH_P_8021Q) &&
1779                             ethertype != RTE_BE16(ETH_P_8021AD))
1780                                 return rte_flow_error_set
1781                                         (error, EINVAL,
1782                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1783                                          "vlan push TPID must be "
1784                                          "802.1Q or 802.1AD");
1785                         break;
1786                 }
1787                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
1788                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1789                                 return rte_flow_error_set
1790                                         (error, ENOTSUP,
1791                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1792                                          "vlan modify is not supported,"
1793                                          " set action must follow push action");
1794                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
1795                         break;
1796                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
1797                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1798                                 return rte_flow_error_set
1799                                         (error, ENOTSUP,
1800                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1801                                          "vlan modify is not supported,"
1802                                          " set action must follow push action");
1803                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
1804                         break;
1805                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1806                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_DECAP;
1807                         break;
1808                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1809                         ret = flow_tcf_validate_vxlan_encap(actions, error);
1810                         if (ret < 0)
1811                                 return ret;
1812                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_ENCAP;
1813                         break;
1814                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
1815                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_SRC;
1816                         break;
1817                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
1818                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_DST;
1819                         break;
1820                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
1821                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_SRC;
1822                         break;
1823                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
1824                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_DST;
1825                         break;
1826                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
1827                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_SRC;
1828                         break;
1829                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
1830                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_DST;
1831                         break;
1832                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
1833                         current_action_flag = MLX5_FLOW_ACTION_SET_TTL;
1834                         break;
1835                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
1836                         current_action_flag = MLX5_FLOW_ACTION_DEC_TTL;
1837                         break;
1838                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
1839                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_SRC;
1840                         break;
1841                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
1842                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_DST;
1843                         break;
1844                 default:
1845                         return rte_flow_error_set(error, ENOTSUP,
1846                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1847                                                   actions,
1848                                                   "action not supported");
1849                 }
1850                 if (current_action_flag & MLX5_TCF_CONFIG_ACTIONS) {
1851                         if (!actions->conf)
1852                                 return rte_flow_error_set
1853                                         (error, EINVAL,
1854                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1855                                          actions,
1856                                          "action configuration not set");
1857                 }
1858                 if ((current_action_flag & MLX5_TCF_PEDIT_ACTIONS) &&
1859                     pedit_validated)
1860                         return rte_flow_error_set(error, ENOTSUP,
1861                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1862                                                   actions,
1863                                                   "set actions should be "
1864                                                   "listed successively");
1865                 if ((current_action_flag & ~MLX5_TCF_PEDIT_ACTIONS) &&
1866                     (action_flags & MLX5_TCF_PEDIT_ACTIONS))
1867                         pedit_validated = 1;
1868                 if ((current_action_flag & MLX5_TCF_FATE_ACTIONS) &&
1869                     (action_flags & MLX5_TCF_FATE_ACTIONS))
1870                         return rte_flow_error_set(error, EINVAL,
1871                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1872                                                   actions,
1873                                                   "can't have multiple fate"
1874                                                   " actions");
1875                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1876                     (action_flags & MLX5_TCF_VXLAN_ACTIONS))
1877                         return rte_flow_error_set(error, EINVAL,
1878                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1879                                                   actions,
1880                                                   "can't have multiple vxlan"
1881                                                   " actions");
1882                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1883                     (action_flags & MLX5_TCF_VLAN_ACTIONS))
1884                         return rte_flow_error_set(error, ENOTSUP,
1885                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1886                                                   actions,
1887                                                   "can't have vxlan and vlan"
1888                                                   " actions in the same rule");
1889                 action_flags |= current_action_flag;
1890         }
1891         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1892                 unsigned int i;
1893
1894                 switch (items->type) {
1895                 case RTE_FLOW_ITEM_TYPE_VOID:
1896                         break;
1897                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
1898                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1899                                 return rte_flow_error_set
1900                                         (error, ENOTSUP,
1901                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1902                                          "inner tunnel port id"
1903                                          " item is not supported");
1904                         mask.port_id = flow_tcf_item_mask
1905                                 (items, &rte_flow_item_port_id_mask,
1906                                  &flow_tcf_mask_supported.port_id,
1907                                  &flow_tcf_mask_empty.port_id,
1908                                  sizeof(flow_tcf_mask_supported.port_id),
1909                                  error);
1910                         if (!mask.port_id)
1911                                 return -rte_errno;
1912                         if (mask.port_id == &flow_tcf_mask_empty.port_id) {
1913                                 in_port_id_set = 1;
1914                                 break;
1915                         }
1916                         spec.port_id = items->spec;
1917                         if (mask.port_id->id && mask.port_id->id != 0xffffffff)
1918                                 return rte_flow_error_set
1919                                         (error, ENOTSUP,
1920                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1921                                          mask.port_id,
1922                                          "no support for partial mask on"
1923                                          " \"id\" field");
1924                         if (!mask.port_id->id)
1925                                 i = 0;
1926                         else
1927                                 for (i = 0; ptoi[i].ifindex; ++i)
1928                                         if (ptoi[i].port_id == spec.port_id->id)
1929                                                 break;
1930                         if (!ptoi[i].ifindex)
1931                                 return rte_flow_error_set
1932                                         (error, ENODEV,
1933                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1934                                          spec.port_id,
1935                                          "missing data to convert port ID to"
1936                                          " ifindex");
1937                         if (in_port_id_set && ptoi[i].ifindex != tcm_ifindex)
1938                                 return rte_flow_error_set
1939                                         (error, ENOTSUP,
1940                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1941                                          spec.port_id,
1942                                          "cannot match traffic for"
1943                                          " several port IDs through"
1944                                          " a single flow rule");
1945                         tcm_ifindex = ptoi[i].ifindex;
1946                         in_port_id_set = 1;
1947                         break;
1948                 case RTE_FLOW_ITEM_TYPE_ETH:
1949                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1950                                                           error);
1951                         if (ret < 0)
1952                                 return ret;
1953                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
1954                                       MLX5_FLOW_LAYER_INNER_L2 :
1955                                       MLX5_FLOW_LAYER_OUTER_L2;
1956                         /* TODO:
1957                          * Redundant check due to different supported mask.
1958                          * Same for the rest of items.
1959                          */
1960                         mask.eth = flow_tcf_item_mask
1961                                 (items, &rte_flow_item_eth_mask,
1962                                  &flow_tcf_mask_supported.eth,
1963                                  &flow_tcf_mask_empty.eth,
1964                                  sizeof(flow_tcf_mask_supported.eth),
1965                                  error);
1966                         if (!mask.eth)
1967                                 return -rte_errno;
1968                         if (mask.eth->type && mask.eth->type !=
1969                             RTE_BE16(0xffff))
1970                                 return rte_flow_error_set
1971                                         (error, ENOTSUP,
1972                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1973                                          mask.eth,
1974                                          "no support for partial mask on"
1975                                          " \"type\" field");
1976                         assert(items->spec);
1977                         spec.eth = items->spec;
1978                         if (mask.eth->type &&
1979                             (item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
1980                             inner_etype != RTE_BE16(ETH_P_ALL) &&
1981                             inner_etype != spec.eth->type)
1982                                 return rte_flow_error_set
1983                                         (error, EINVAL,
1984                                          RTE_FLOW_ERROR_TYPE_ITEM,
1985                                          items,
1986                                          "inner eth_type conflict");
1987                         if (mask.eth->type &&
1988                             !(item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
1989                             outer_etype != RTE_BE16(ETH_P_ALL) &&
1990                             outer_etype != spec.eth->type)
1991                                 return rte_flow_error_set
1992                                         (error, EINVAL,
1993                                          RTE_FLOW_ERROR_TYPE_ITEM,
1994                                          items,
1995                                          "outer eth_type conflict");
1996                         if (mask.eth->type) {
1997                                 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1998                                         inner_etype = spec.eth->type;
1999                                 else
2000                                         outer_etype = spec.eth->type;
2001                         }
2002                         break;
2003                 case RTE_FLOW_ITEM_TYPE_VLAN:
2004                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2005                                 return rte_flow_error_set
2006                                         (error, ENOTSUP,
2007                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
2008                                          "inner tunnel VLAN"
2009                                          " is not supported");
2010                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
2011                                                            error);
2012                         if (ret < 0)
2013                                 return ret;
2014                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
2015                         mask.vlan = flow_tcf_item_mask
2016                                 (items, &rte_flow_item_vlan_mask,
2017                                  &flow_tcf_mask_supported.vlan,
2018                                  &flow_tcf_mask_empty.vlan,
2019                                  sizeof(flow_tcf_mask_supported.vlan),
2020                                  error);
2021                         if (!mask.vlan)
2022                                 return -rte_errno;
2023                         if ((mask.vlan->tci & RTE_BE16(0xe000) &&
2024                              (mask.vlan->tci & RTE_BE16(0xe000)) !=
2025                               RTE_BE16(0xe000)) ||
2026                             (mask.vlan->tci & RTE_BE16(0x0fff) &&
2027                              (mask.vlan->tci & RTE_BE16(0x0fff)) !=
2028                               RTE_BE16(0x0fff)) ||
2029                             (mask.vlan->inner_type &&
2030                              mask.vlan->inner_type != RTE_BE16(0xffff)))
2031                                 return rte_flow_error_set
2032                                         (error, ENOTSUP,
2033                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2034                                          mask.vlan,
2035                                          "no support for partial masks on"
2036                                          " \"tci\" (PCP and VID parts) and"
2037                                          " \"inner_type\" fields");
2038                         if (outer_etype != RTE_BE16(ETH_P_ALL) &&
2039                             outer_etype != RTE_BE16(ETH_P_8021Q))
2040                                 return rte_flow_error_set
2041                                         (error, EINVAL,
2042                                          RTE_FLOW_ERROR_TYPE_ITEM,
2043                                          items,
2044                                          "outer eth_type conflict,"
2045                                          " must be 802.1Q");
2046                         outer_etype = RTE_BE16(ETH_P_8021Q);
2047                         assert(items->spec);
2048                         spec.vlan = items->spec;
2049                         if (mask.vlan->inner_type &&
2050                             vlan_etype != RTE_BE16(ETH_P_ALL) &&
2051                             vlan_etype != spec.vlan->inner_type)
2052                                 return rte_flow_error_set
2053                                         (error, EINVAL,
2054                                          RTE_FLOW_ERROR_TYPE_ITEM,
2055                                          items,
2056                                          "vlan eth_type conflict");
2057                         if (mask.vlan->inner_type)
2058                                 vlan_etype = spec.vlan->inner_type;
2059                         break;
2060                 case RTE_FLOW_ITEM_TYPE_IPV4:
2061                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2062                                                            error);
2063                         if (ret < 0)
2064                                 return ret;
2065                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2066                                       MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2067                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2068                         mask.ipv4 = flow_tcf_item_mask
2069                                 (items, &rte_flow_item_ipv4_mask,
2070                                  &flow_tcf_mask_supported.ipv4,
2071                                  &flow_tcf_mask_empty.ipv4,
2072                                  sizeof(flow_tcf_mask_supported.ipv4),
2073                                  error);
2074                         if (!mask.ipv4)
2075                                 return -rte_errno;
2076                         if (mask.ipv4->hdr.next_proto_id &&
2077                             mask.ipv4->hdr.next_proto_id != 0xff)
2078                                 return rte_flow_error_set
2079                                         (error, ENOTSUP,
2080                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2081                                          mask.ipv4,
2082                                          "no support for partial mask on"
2083                                          " \"hdr.next_proto_id\" field");
2084                         else if (mask.ipv4->hdr.next_proto_id)
2085                                 next_protocol =
2086                                         ((const struct rte_flow_item_ipv4 *)
2087                                          (items->spec))->hdr.next_proto_id;
2088                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2089                                 if (inner_etype != RTE_BE16(ETH_P_ALL) &&
2090                                     inner_etype != RTE_BE16(ETH_P_IP))
2091                                         return rte_flow_error_set
2092                                                 (error, EINVAL,
2093                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2094                                                  items,
2095                                                  "inner eth_type conflict,"
2096                                                  " IPv4 is required");
2097                                 inner_etype = RTE_BE16(ETH_P_IP);
2098                         } else if (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN) {
2099                                 if (vlan_etype != RTE_BE16(ETH_P_ALL) &&
2100                                     vlan_etype != RTE_BE16(ETH_P_IP))
2101                                         return rte_flow_error_set
2102                                                 (error, EINVAL,
2103                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2104                                                  items,
2105                                                  "vlan eth_type conflict,"
2106                                                  " IPv4 is required");
2107                                 vlan_etype = RTE_BE16(ETH_P_IP);
2108                         } else {
2109                                 if (outer_etype != RTE_BE16(ETH_P_ALL) &&
2110                                     outer_etype != RTE_BE16(ETH_P_IP))
2111                                         return rte_flow_error_set
2112                                                 (error, EINVAL,
2113                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2114                                                  items,
2115                                                  "eth_type conflict,"
2116                                                  " IPv4 is required");
2117                                 outer_etype = RTE_BE16(ETH_P_IP);
2118                         }
2119                         break;
2120                 case RTE_FLOW_ITEM_TYPE_IPV6:
2121                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2122                                                            error);
2123                         if (ret < 0)
2124                                 return ret;
2125                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2126                                       MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2127                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2128                         mask.ipv6 = flow_tcf_item_mask
2129                                 (items, &rte_flow_item_ipv6_mask,
2130                                  &flow_tcf_mask_supported.ipv6,
2131                                  &flow_tcf_mask_empty.ipv6,
2132                                  sizeof(flow_tcf_mask_supported.ipv6),
2133                                  error);
2134                         if (!mask.ipv6)
2135                                 return -rte_errno;
2136                         if (mask.ipv6->hdr.proto &&
2137                             mask.ipv6->hdr.proto != 0xff)
2138                                 return rte_flow_error_set
2139                                         (error, ENOTSUP,
2140                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2141                                          mask.ipv6,
2142                                          "no support for partial mask on"
2143                                          " \"hdr.proto\" field");
2144                         else if (mask.ipv6->hdr.proto)
2145                                 next_protocol =
2146                                         ((const struct rte_flow_item_ipv6 *)
2147                                          (items->spec))->hdr.proto;
2148                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2149                                 if (inner_etype != RTE_BE16(ETH_P_ALL) &&
2150                                     inner_etype != RTE_BE16(ETH_P_IPV6))
2151                                         return rte_flow_error_set
2152                                                 (error, EINVAL,
2153                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2154                                                  items,
2155                                                  "inner eth_type conflict,"
2156                                                  " IPv6 is required");
2157                                 inner_etype = RTE_BE16(ETH_P_IPV6);
2158                         } else if (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN) {
2159                                 if (vlan_etype != RTE_BE16(ETH_P_ALL) &&
2160                                     vlan_etype != RTE_BE16(ETH_P_IPV6))
2161                                         return rte_flow_error_set
2162                                                 (error, EINVAL,
2163                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2164                                                  items,
2165                                                  "vlan eth_type conflict,"
2166                                                  " IPv6 is required");
2167                                 vlan_etype = RTE_BE16(ETH_P_IPV6);
2168                         } else {
2169                                 if (outer_etype != RTE_BE16(ETH_P_ALL) &&
2170                                     outer_etype != RTE_BE16(ETH_P_IPV6))
2171                                         return rte_flow_error_set
2172                                                 (error, EINVAL,
2173                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2174                                                  items,
2175                                                  "eth_type conflict,"
2176                                                  " IPv6 is required");
2177                                 outer_etype = RTE_BE16(ETH_P_IPV6);
2178                         }
2179                         break;
2180                 case RTE_FLOW_ITEM_TYPE_UDP:
2181                         ret = mlx5_flow_validate_item_udp(items, item_flags,
2182                                                           next_protocol, error);
2183                         if (ret < 0)
2184                                 return ret;
2185                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2186                                       MLX5_FLOW_LAYER_INNER_L4_UDP :
2187                                       MLX5_FLOW_LAYER_OUTER_L4_UDP;
2188                         mask.udp = flow_tcf_item_mask
2189                                 (items, &rte_flow_item_udp_mask,
2190                                  &flow_tcf_mask_supported.udp,
2191                                  &flow_tcf_mask_empty.udp,
2192                                  sizeof(flow_tcf_mask_supported.udp),
2193                                  error);
2194                         if (!mask.udp)
2195                                 return -rte_errno;
2196                         /*
2197                          * Save the presumed outer UDP item for extra check
2198                          * if the tunnel item will be found later in the list.
2199                          */
2200                         if (!(item_flags & MLX5_FLOW_LAYER_TUNNEL))
2201                                 outer_udp = items;
2202                         break;
2203                 case RTE_FLOW_ITEM_TYPE_TCP:
2204                         ret = mlx5_flow_validate_item_tcp
2205                                              (items, item_flags,
2206                                               next_protocol,
2207                                               &flow_tcf_mask_supported.tcp,
2208                                               error);
2209                         if (ret < 0)
2210                                 return ret;
2211                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2212                                       MLX5_FLOW_LAYER_INNER_L4_TCP :
2213                                       MLX5_FLOW_LAYER_OUTER_L4_TCP;
2214                         mask.tcp = flow_tcf_item_mask
2215                                 (items, &rte_flow_item_tcp_mask,
2216                                  &flow_tcf_mask_supported.tcp,
2217                                  &flow_tcf_mask_empty.tcp,
2218                                  sizeof(flow_tcf_mask_supported.tcp),
2219                                  error);
2220                         if (!mask.tcp)
2221                                 return -rte_errno;
2222                         break;
2223                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2224                         if (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)
2225                                 return rte_flow_error_set
2226                                         (error, ENOTSUP,
2227                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
2228                                          "vxlan tunnel over vlan"
2229                                          " is not supported");
2230                         ret = mlx5_flow_validate_item_vxlan(items,
2231                                                             item_flags, error);
2232                         if (ret < 0)
2233                                 return ret;
2234                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
2235                         mask.vxlan = flow_tcf_item_mask
2236                                 (items, &rte_flow_item_vxlan_mask,
2237                                  &flow_tcf_mask_supported.vxlan,
2238                                  &flow_tcf_mask_empty.vxlan,
2239                                  sizeof(flow_tcf_mask_supported.vxlan), error);
2240                         if (!mask.vxlan)
2241                                 return -rte_errno;
2242                         if (mask.vxlan->vni[0] != 0xff ||
2243                             mask.vxlan->vni[1] != 0xff ||
2244                             mask.vxlan->vni[2] != 0xff)
2245                                 return rte_flow_error_set
2246                                         (error, ENOTSUP,
2247                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2248                                          mask.vxlan,
2249                                          "no support for partial or "
2250                                          "empty mask on \"vxlan.vni\" field");
2251                         /*
2252                          * The VNI item assumes the VXLAN tunnel, it requires
2253                          * at least the outer destination UDP port must be
2254                          * specified without wildcards to allow kernel select
2255                          * the virtual VXLAN device by port. Also outer IPv4
2256                          * or IPv6 item must be specified (wilcards or even
2257                          * zero mask are allowed) to let driver know the tunnel
2258                          * IP version and process UDP traffic correctly.
2259                          */
2260                         if (!(item_flags &
2261                              (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2262                               MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2263                                 return rte_flow_error_set
2264                                                  (error, EINVAL,
2265                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2266                                                   NULL,
2267                                                   "no outer IP pattern found"
2268                                                   " for vxlan tunnel");
2269                         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2270                                 return rte_flow_error_set
2271                                                  (error, EINVAL,
2272                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2273                                                   NULL,
2274                                                   "no outer UDP pattern found"
2275                                                   " for vxlan tunnel");
2276                         /*
2277                          * All items preceding the tunnel item become outer
2278                          * ones and we should do extra validation for them
2279                          * due to tc limitations for tunnel outer parameters.
2280                          * Currently only outer UDP item requres extra check,
2281                          * use the saved pointer instead of item list rescan.
2282                          */
2283                         assert(outer_udp);
2284                         ret = flow_tcf_validate_vxlan_decap_udp
2285                                                 (outer_udp, error);
2286                         if (ret < 0)
2287                                 return ret;
2288                         /* Reset L4 protocol for inner parameters. */
2289                         next_protocol = 0xff;
2290                         break;
2291                 default:
2292                         return rte_flow_error_set(error, ENOTSUP,
2293                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2294                                                   items, "item not supported");
2295                 }
2296         }
2297         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2298             (action_flags & MLX5_FLOW_ACTION_DROP))
2299                 return rte_flow_error_set(error, ENOTSUP,
2300                                           RTE_FLOW_ERROR_TYPE_ACTION,
2301                                           actions,
2302                                           "set action is not compatible with "
2303                                           "drop action");
2304         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2305             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2306                 return rte_flow_error_set(error, ENOTSUP,
2307                                           RTE_FLOW_ERROR_TYPE_ACTION,
2308                                           actions,
2309                                           "set action must be followed by "
2310                                           "port_id action");
2311         if (action_flags &
2312            (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST)) {
2313                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4))
2314                         return rte_flow_error_set(error, EINVAL,
2315                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2316                                                   actions,
2317                                                   "no ipv4 item found in"
2318                                                   " pattern");
2319         }
2320         if (action_flags &
2321            (MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST)) {
2322                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6))
2323                         return rte_flow_error_set(error, EINVAL,
2324                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2325                                                   actions,
2326                                                   "no ipv6 item found in"
2327                                                   " pattern");
2328         }
2329         if (action_flags &
2330            (MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST)) {
2331                 if (!(item_flags &
2332                      (MLX5_FLOW_LAYER_OUTER_L4_UDP |
2333                       MLX5_FLOW_LAYER_OUTER_L4_TCP)))
2334                         return rte_flow_error_set(error, EINVAL,
2335                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2336                                                   actions,
2337                                                   "no TCP/UDP item found in"
2338                                                   " pattern");
2339         }
2340         /*
2341          * FW syndrome (0xA9C090):
2342          *     set_flow_table_entry: push vlan action fte in fdb can ONLY be
2343          *     forward to the uplink.
2344          */
2345         if ((action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2346             (action_flags & MLX5_FLOW_ACTION_PORT_ID) &&
2347             ((struct priv *)port_id_dev->data->dev_private)->representor)
2348                 return rte_flow_error_set(error, ENOTSUP,
2349                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2350                                           "vlan push can only be applied"
2351                                           " when forwarding to uplink port");
2352         /*
2353          * FW syndrome (0x294609):
2354          *     set_flow_table_entry: modify/pop/push actions in fdb flow table
2355          *     are supported only while forwarding to vport.
2356          */
2357         if ((action_flags & MLX5_TCF_VLAN_ACTIONS) &&
2358             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2359                 return rte_flow_error_set(error, ENOTSUP,
2360                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2361                                           "vlan actions are supported"
2362                                           " only with port_id action");
2363         if ((action_flags & MLX5_TCF_VXLAN_ACTIONS) &&
2364             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2365                 return rte_flow_error_set(error, ENOTSUP,
2366                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2367                                           "vxlan actions are supported"
2368                                           " only with port_id action");
2369         if (!(action_flags & MLX5_TCF_FATE_ACTIONS))
2370                 return rte_flow_error_set(error, EINVAL,
2371                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2372                                           "no fate action is found");
2373         if (action_flags &
2374            (MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL)) {
2375                 if (!(item_flags &
2376                      (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2377                       MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2378                         return rte_flow_error_set(error, EINVAL,
2379                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2380                                                   actions,
2381                                                   "no IP found in pattern");
2382         }
2383         if (action_flags &
2384             (MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)) {
2385                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L2))
2386                         return rte_flow_error_set(error, ENOTSUP,
2387                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2388                                                   actions,
2389                                                   "no ethernet found in"
2390                                                   " pattern");
2391         }
2392         if ((action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) &&
2393             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
2394                 return rte_flow_error_set(error, EINVAL,
2395                                           RTE_FLOW_ERROR_TYPE_ACTION,
2396                                           NULL,
2397                                           "no VNI pattern found"
2398                                           " for vxlan decap action");
2399         if ((action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) &&
2400             (item_flags & MLX5_FLOW_LAYER_TUNNEL))
2401                 return rte_flow_error_set(error, EINVAL,
2402                                           RTE_FLOW_ERROR_TYPE_ACTION,
2403                                           NULL,
2404                                           "vxlan encap not supported"
2405                                           " for tunneled traffic");
2406         return 0;
2407 }
2408
2409 /**
2410  * Calculate maximum size of memory for flow items of Linux TC flower.
2411  *
2412  * @param[in] attr
2413  *   Pointer to the flow attributes.
2414  * @param[in] items
2415  *   Pointer to the list of items.
2416  * @param[out] action_flags
2417  *   Pointer to the detected actions.
2418  *
2419  * @return
2420  *   Maximum size of memory for items.
2421  */
2422 static int
2423 flow_tcf_get_items_size(const struct rte_flow_attr *attr,
2424                         const struct rte_flow_item items[],
2425                         uint64_t *action_flags)
2426 {
2427         int size = 0;
2428
2429         size += SZ_NLATTR_STRZ_OF("flower") +
2430                 SZ_NLATTR_TYPE_OF(uint16_t) + /* Outer ether type. */
2431                 SZ_NLATTR_NEST + /* TCA_OPTIONS. */
2432                 SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CLS_FLAGS_SKIP_SW. */
2433         if (attr->group > 0)
2434                 size += SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CHAIN. */
2435         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2436                 switch (items->type) {
2437                 case RTE_FLOW_ITEM_TYPE_VOID:
2438                         break;
2439                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2440                         break;
2441                 case RTE_FLOW_ITEM_TYPE_ETH:
2442                         size += SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) * 4;
2443                                 /* dst/src MAC addr and mask. */
2444                         break;
2445                 case RTE_FLOW_ITEM_TYPE_VLAN:
2446                         size += SZ_NLATTR_TYPE_OF(uint16_t) +
2447                                 /* VLAN Ether type. */
2448                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* VLAN prio. */
2449                                 SZ_NLATTR_TYPE_OF(uint16_t); /* VLAN ID. */
2450                         break;
2451                 case RTE_FLOW_ITEM_TYPE_IPV4:
2452                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2453                                 SZ_NLATTR_TYPE_OF(uint32_t) * 4;
2454                                 /* dst/src IP addr and mask. */
2455                         break;
2456                 case RTE_FLOW_ITEM_TYPE_IPV6:
2457                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2458                                 SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 4;
2459                                 /* dst/src IP addr and mask. */
2460                         break;
2461                 case RTE_FLOW_ITEM_TYPE_UDP:
2462                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2463                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2464                                 /* dst/src port and mask. */
2465                         break;
2466                 case RTE_FLOW_ITEM_TYPE_TCP:
2467                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2468                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2469                                 /* dst/src port and mask. */
2470                         break;
2471                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2472                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2473                         /*
2474                          * There might be no VXLAN decap action in the action
2475                          * list, nonetheless the VXLAN tunnel flow requires
2476                          * the decap structure to be correctly applied to
2477                          * VXLAN device, set the flag to create the structure.
2478                          * Translation routine will not put the decap action
2479                          * in tne Netlink message if there is no actual action
2480                          * in the list.
2481                          */
2482                         *action_flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2483                         break;
2484                 default:
2485                         DRV_LOG(WARNING,
2486                                 "unsupported item %p type %d,"
2487                                 " items must be validated before flow creation",
2488                                 (const void *)items, items->type);
2489                         break;
2490                 }
2491         }
2492         return size;
2493 }
2494
2495 /**
2496  * Calculate size of memory to store the VXLAN encapsultion
2497  * related items in the Netlink message buffer. Items list
2498  * is specified by RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action.
2499  * The item list should be validated.
2500  *
2501  * @param[in] action
2502  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2503  *   List of pattern items to scan data from.
2504  *
2505  * @return
2506  *   The size the part of Netlink message buffer to store the
2507  *   VXLAN encapsulation item attributes.
2508  */
2509 static int
2510 flow_tcf_vxlan_encap_size(const struct rte_flow_action *action)
2511 {
2512         const struct rte_flow_item *items;
2513         int size = 0;
2514
2515         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2516         assert(action->conf);
2517
2518         items = ((const struct rte_flow_action_vxlan_encap *)
2519                                         action->conf)->definition;
2520         assert(items);
2521         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2522                 switch (items->type) {
2523                 case RTE_FLOW_ITEM_TYPE_VOID:
2524                         break;
2525                 case RTE_FLOW_ITEM_TYPE_ETH:
2526                         /* This item does not require message buffer. */
2527                         break;
2528                 case RTE_FLOW_ITEM_TYPE_IPV4:
2529                         size += SZ_NLATTR_DATA_OF(IPV4_ADDR_LEN) * 2;
2530                         break;
2531                 case RTE_FLOW_ITEM_TYPE_IPV6:
2532                         size += SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 2;
2533                         break;
2534                 case RTE_FLOW_ITEM_TYPE_UDP: {
2535                         const struct rte_flow_item_udp *udp = items->mask;
2536
2537                         size += SZ_NLATTR_TYPE_OF(uint16_t);
2538                         if (!udp || udp->hdr.src_port != RTE_BE16(0x0000))
2539                                 size += SZ_NLATTR_TYPE_OF(uint16_t);
2540                         break;
2541                 }
2542                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2543                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2544                         break;
2545                 default:
2546                         assert(false);
2547                         DRV_LOG(WARNING,
2548                                 "unsupported item %p type %d,"
2549                                 " items must be validated"
2550                                 " before flow creation",
2551                                 (const void *)items, items->type);
2552                         return 0;
2553                 }
2554         }
2555         return size;
2556 }
2557
2558 /**
2559  * Calculate maximum size of memory for flow actions of Linux TC flower and
2560  * extract specified actions.
2561  *
2562  * @param[in] actions
2563  *   Pointer to the list of actions.
2564  * @param[out] action_flags
2565  *   Pointer to the detected actions.
2566  *
2567  * @return
2568  *   Maximum size of memory for actions.
2569  */
2570 static int
2571 flow_tcf_get_actions_and_size(const struct rte_flow_action actions[],
2572                               uint64_t *action_flags)
2573 {
2574         int size = 0;
2575         uint64_t flags = 0;
2576
2577         size += SZ_NLATTR_NEST; /* TCA_FLOWER_ACT. */
2578         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2579                 switch (actions->type) {
2580                 case RTE_FLOW_ACTION_TYPE_VOID:
2581                         break;
2582                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
2583                         size += SZ_NLATTR_NEST + /* na_act_index. */
2584                                 SZ_NLATTR_STRZ_OF("mirred") +
2585                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2586                                 SZ_NLATTR_TYPE_OF(struct tc_mirred);
2587                         flags |= MLX5_FLOW_ACTION_PORT_ID;
2588                         break;
2589                 case RTE_FLOW_ACTION_TYPE_JUMP:
2590                         size += SZ_NLATTR_NEST + /* na_act_index. */
2591                                 SZ_NLATTR_STRZ_OF("gact") +
2592                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2593                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2594                         flags |= MLX5_FLOW_ACTION_JUMP;
2595                         break;
2596                 case RTE_FLOW_ACTION_TYPE_DROP:
2597                         size += SZ_NLATTR_NEST + /* na_act_index. */
2598                                 SZ_NLATTR_STRZ_OF("gact") +
2599                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2600                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2601                         flags |= MLX5_FLOW_ACTION_DROP;
2602                         break;
2603                 case RTE_FLOW_ACTION_TYPE_COUNT:
2604                         break;
2605                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
2606                         flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
2607                         goto action_of_vlan;
2608                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
2609                         flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
2610                         goto action_of_vlan;
2611                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
2612                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
2613                         goto action_of_vlan;
2614                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
2615                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
2616                         goto action_of_vlan;
2617 action_of_vlan:
2618                         size += SZ_NLATTR_NEST + /* na_act_index. */
2619                                 SZ_NLATTR_STRZ_OF("vlan") +
2620                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2621                                 SZ_NLATTR_TYPE_OF(struct tc_vlan) +
2622                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2623                                 /* VLAN protocol. */
2624                                 SZ_NLATTR_TYPE_OF(uint16_t) + /* VLAN ID. */
2625                                 SZ_NLATTR_TYPE_OF(uint8_t); /* VLAN prio. */
2626                         break;
2627                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2628                         size += SZ_NLATTR_NEST + /* na_act_index. */
2629                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2630                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2631                                 SZ_NLATTR_TYPE_OF(uint8_t);
2632                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2633                         size += flow_tcf_vxlan_encap_size(actions) +
2634                                 RTE_ALIGN_CEIL /* preceding encap params. */
2635                                 (sizeof(struct flow_tcf_vxlan_encap),
2636                                 MNL_ALIGNTO);
2637                         flags |= MLX5_FLOW_ACTION_VXLAN_ENCAP;
2638                         break;
2639                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
2640                         size += SZ_NLATTR_NEST + /* na_act_index. */
2641                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2642                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2643                                 SZ_NLATTR_TYPE_OF(uint8_t);
2644                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2645                         size += RTE_ALIGN_CEIL /* preceding decap params. */
2646                                 (sizeof(struct flow_tcf_vxlan_decap),
2647                                 MNL_ALIGNTO);
2648                         flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2649                         break;
2650                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
2651                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
2652                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
2653                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
2654                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
2655                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
2656                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
2657                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
2658                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
2659                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
2660                         size += flow_tcf_get_pedit_actions_size(&actions,
2661                                                                 &flags);
2662                         break;
2663                 default:
2664                         DRV_LOG(WARNING,
2665                                 "unsupported action %p type %d,"
2666                                 " items must be validated before flow creation",
2667                                 (const void *)actions, actions->type);
2668                         break;
2669                 }
2670         }
2671         *action_flags = flags;
2672         return size;
2673 }
2674
2675 /**
2676  * Brand rtnetlink buffer with unique handle.
2677  *
2678  * This handle should be unique for a given network interface to avoid
2679  * collisions.
2680  *
2681  * @param nlh
2682  *   Pointer to Netlink message.
2683  * @param handle
2684  *   Unique 32-bit handle to use.
2685  */
2686 static void
2687 flow_tcf_nl_brand(struct nlmsghdr *nlh, uint32_t handle)
2688 {
2689         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
2690
2691         tcm->tcm_handle = handle;
2692         DRV_LOG(DEBUG, "Netlink msg %p is branded with handle %x",
2693                 (void *)nlh, handle);
2694 }
2695
2696 /**
2697  * Prepare a flow object for Linux TC flower. It calculates the maximum size of
2698  * memory required, allocates the memory, initializes Netlink message headers
2699  * and set unique TC message handle.
2700  *
2701  * @param[in] attr
2702  *   Pointer to the flow attributes.
2703  * @param[in] items
2704  *   Pointer to the list of items.
2705  * @param[in] actions
2706  *   Pointer to the list of actions.
2707  * @param[out] error
2708  *   Pointer to the error structure.
2709  *
2710  * @return
2711  *   Pointer to mlx5_flow object on success,
2712  *   otherwise NULL and rte_errno is set.
2713  */
2714 static struct mlx5_flow *
2715 flow_tcf_prepare(const struct rte_flow_attr *attr,
2716                  const struct rte_flow_item items[],
2717                  const struct rte_flow_action actions[],
2718                  struct rte_flow_error *error)
2719 {
2720         size_t size = RTE_ALIGN_CEIL
2721                         (sizeof(struct mlx5_flow),
2722                          alignof(struct flow_tcf_tunnel_hdr)) +
2723                       MNL_ALIGN(sizeof(struct nlmsghdr)) +
2724                       MNL_ALIGN(sizeof(struct tcmsg));
2725         struct mlx5_flow *dev_flow;
2726         uint64_t action_flags = 0;
2727         struct nlmsghdr *nlh;
2728         struct tcmsg *tcm;
2729         uint8_t *sp, *tun = NULL;
2730
2731         size += flow_tcf_get_items_size(attr, items, &action_flags);
2732         size += flow_tcf_get_actions_and_size(actions, &action_flags);
2733         dev_flow = rte_zmalloc(__func__, size, MNL_ALIGNTO);
2734         if (!dev_flow) {
2735                 rte_flow_error_set(error, ENOMEM,
2736                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2737                                    "not enough memory to create E-Switch flow");
2738                 return NULL;
2739         }
2740         sp = (uint8_t *)(dev_flow + 1);
2741         if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) {
2742                 sp = RTE_PTR_ALIGN
2743                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2744                 tun = sp;
2745                 sp += RTE_ALIGN_CEIL
2746                         (sizeof(struct flow_tcf_vxlan_encap),
2747                         MNL_ALIGNTO);
2748 #ifndef NDEBUG
2749                 size -= RTE_ALIGN_CEIL
2750                         (sizeof(struct flow_tcf_vxlan_encap),
2751                         MNL_ALIGNTO);
2752 #endif
2753         } else if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2754                 sp = RTE_PTR_ALIGN
2755                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2756                 tun = sp;
2757                 sp += RTE_ALIGN_CEIL
2758                         (sizeof(struct flow_tcf_vxlan_decap),
2759                         MNL_ALIGNTO);
2760 #ifndef NDEBUG
2761                 size -= RTE_ALIGN_CEIL
2762                         (sizeof(struct flow_tcf_vxlan_decap),
2763                         MNL_ALIGNTO);
2764 #endif
2765         } else {
2766                 sp = RTE_PTR_ALIGN(sp, MNL_ALIGNTO);
2767         }
2768         nlh = mnl_nlmsg_put_header(sp);
2769         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
2770         *dev_flow = (struct mlx5_flow){
2771                 .tcf = (struct mlx5_flow_tcf){
2772 #ifndef NDEBUG
2773                         .nlsize = size - RTE_ALIGN_CEIL
2774                                 (sizeof(struct mlx5_flow),
2775                                  alignof(struct flow_tcf_tunnel_hdr)),
2776 #endif
2777                         .tunnel = (struct flow_tcf_tunnel_hdr *)tun,
2778                         .nlh = nlh,
2779                         .tcm = tcm,
2780                 },
2781         };
2782         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP)
2783                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_DECAP;
2784         else if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP)
2785                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_ENCAP;
2786         /*
2787          * Generate a reasonably unique handle based on the address of the
2788          * target buffer.
2789          *
2790          * This is straightforward on 32-bit systems where the flow pointer can
2791          * be used directly. Otherwise, its least significant part is taken
2792          * after shifting it by the previous power of two of the pointed buffer
2793          * size.
2794          */
2795         if (sizeof(dev_flow) <= 4)
2796                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow);
2797         else
2798                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow >>
2799                                        rte_log2_u32(rte_align32prevpow2(size)));
2800         return dev_flow;
2801 }
2802
2803 /**
2804  * Make adjustments for supporting count actions.
2805  *
2806  * @param[in] dev
2807  *   Pointer to the Ethernet device structure.
2808  * @param[in] dev_flow
2809  *   Pointer to mlx5_flow.
2810  * @param[out] error
2811  *   Pointer to error structure.
2812  *
2813  * @return
2814  *   0 On success else a negative errno value is returned and rte_errno is set.
2815  */
2816 static int
2817 flow_tcf_translate_action_count(struct rte_eth_dev *dev __rte_unused,
2818                                   struct mlx5_flow *dev_flow,
2819                                   struct rte_flow_error *error)
2820 {
2821         struct rte_flow *flow = dev_flow->flow;
2822
2823         if (!flow->counter) {
2824                 flow->counter = flow_tcf_counter_new();
2825                 if (!flow->counter)
2826                         return rte_flow_error_set(error, rte_errno,
2827                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2828                                                   NULL,
2829                                                   "cannot get counter"
2830                                                   " context.");
2831         }
2832         return 0;
2833 }
2834
2835 /**
2836  * Convert VXLAN VNI to 32-bit integer.
2837  *
2838  * @param[in] vni
2839  *   VXLAN VNI in 24-bit wire format.
2840  *
2841  * @return
2842  *   VXLAN VNI as a 32-bit integer value in network endian.
2843  */
2844 static inline rte_be32_t
2845 vxlan_vni_as_be32(const uint8_t vni[3])
2846 {
2847         union {
2848                 uint8_t vni[4];
2849                 rte_be32_t dword;
2850         } ret = {
2851                 .vni = { 0, vni[0], vni[1], vni[2] },
2852         };
2853         return ret.dword;
2854 }
2855
2856 /**
2857  * Helper function to process RTE_FLOW_ITEM_TYPE_ETH entry in configuration
2858  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the MAC address fields
2859  * in the encapsulation parameters structure. The item must be prevalidated,
2860  * no any validation checks performed by function.
2861  *
2862  * @param[in] spec
2863  *   RTE_FLOW_ITEM_TYPE_ETH entry specification.
2864  * @param[in] mask
2865  *   RTE_FLOW_ITEM_TYPE_ETH entry mask.
2866  * @param[out] encap
2867  *   Structure to fill the gathered MAC address data.
2868  */
2869 static void
2870 flow_tcf_parse_vxlan_encap_eth(const struct rte_flow_item_eth *spec,
2871                                const struct rte_flow_item_eth *mask,
2872                                struct flow_tcf_vxlan_encap *encap)
2873 {
2874         /* Item must be validated before. No redundant checks. */
2875         assert(spec);
2876         if (!mask || !memcmp(&mask->dst,
2877                              &rte_flow_item_eth_mask.dst,
2878                              sizeof(rte_flow_item_eth_mask.dst))) {
2879                 /*
2880                  * Ethernet addresses are not supported by
2881                  * tc as tunnel_key parameters. Destination
2882                  * address is needed to form encap packet
2883                  * header and retrieved by kernel from
2884                  * implicit sources (ARP table, etc),
2885                  * address masks are not supported at all.
2886                  */
2887                 encap->eth.dst = spec->dst;
2888                 encap->mask |= FLOW_TCF_ENCAP_ETH_DST;
2889         }
2890         if (!mask || !memcmp(&mask->src,
2891                              &rte_flow_item_eth_mask.src,
2892                              sizeof(rte_flow_item_eth_mask.src))) {
2893                 /*
2894                  * Ethernet addresses are not supported by
2895                  * tc as tunnel_key parameters. Source ethernet
2896                  * address is ignored anyway.
2897                  */
2898                 encap->eth.src = spec->src;
2899                 encap->mask |= FLOW_TCF_ENCAP_ETH_SRC;
2900         }
2901 }
2902
2903 /**
2904  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV4 entry in configuration
2905  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV4 address fields
2906  * in the encapsulation parameters structure. The item must be prevalidated,
2907  * no any validation checks performed by function.
2908  *
2909  * @param[in] spec
2910  *   RTE_FLOW_ITEM_TYPE_IPV4 entry specification.
2911  * @param[out] encap
2912  *   Structure to fill the gathered IPV4 address data.
2913  */
2914 static void
2915 flow_tcf_parse_vxlan_encap_ipv4(const struct rte_flow_item_ipv4 *spec,
2916                                 struct flow_tcf_vxlan_encap *encap)
2917 {
2918         /* Item must be validated before. No redundant checks. */
2919         assert(spec);
2920         encap->ipv4.dst = spec->hdr.dst_addr;
2921         encap->ipv4.src = spec->hdr.src_addr;
2922         encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC |
2923                        FLOW_TCF_ENCAP_IPV4_DST;
2924 }
2925
2926 /**
2927  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV6 entry in configuration
2928  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV6 address fields
2929  * in the encapsulation parameters structure. The item must be prevalidated,
2930  * no any validation checks performed by function.
2931  *
2932  * @param[in] spec
2933  *   RTE_FLOW_ITEM_TYPE_IPV6 entry specification.
2934  * @param[out] encap
2935  *   Structure to fill the gathered IPV6 address data.
2936  */
2937 static void
2938 flow_tcf_parse_vxlan_encap_ipv6(const struct rte_flow_item_ipv6 *spec,
2939                                 struct flow_tcf_vxlan_encap *encap)
2940 {
2941         /* Item must be validated before. No redundant checks. */
2942         assert(spec);
2943         memcpy(encap->ipv6.dst, spec->hdr.dst_addr, IPV6_ADDR_LEN);
2944         memcpy(encap->ipv6.src, spec->hdr.src_addr, IPV6_ADDR_LEN);
2945         encap->mask |= FLOW_TCF_ENCAP_IPV6_SRC |
2946                        FLOW_TCF_ENCAP_IPV6_DST;
2947 }
2948
2949 /**
2950  * Helper function to process RTE_FLOW_ITEM_TYPE_UDP entry in configuration
2951  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the UDP port fields
2952  * in the encapsulation parameters structure. The item must be prevalidated,
2953  * no any validation checks performed by function.
2954  *
2955  * @param[in] spec
2956  *   RTE_FLOW_ITEM_TYPE_UDP entry specification.
2957  * @param[in] mask
2958  *   RTE_FLOW_ITEM_TYPE_UDP entry mask.
2959  * @param[out] encap
2960  *   Structure to fill the gathered UDP port data.
2961  */
2962 static void
2963 flow_tcf_parse_vxlan_encap_udp(const struct rte_flow_item_udp *spec,
2964                                const struct rte_flow_item_udp *mask,
2965                                struct flow_tcf_vxlan_encap *encap)
2966 {
2967         assert(spec);
2968         encap->udp.dst = spec->hdr.dst_port;
2969         encap->mask |= FLOW_TCF_ENCAP_UDP_DST;
2970         if (!mask || mask->hdr.src_port != RTE_BE16(0x0000)) {
2971                 encap->udp.src = spec->hdr.src_port;
2972                 encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC;
2973         }
2974 }
2975
2976 /**
2977  * Helper function to process RTE_FLOW_ITEM_TYPE_VXLAN entry in configuration
2978  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the VNI fields
2979  * in the encapsulation parameters structure. The item must be prevalidated,
2980  * no any validation checks performed by function.
2981  *
2982  * @param[in] spec
2983  *   RTE_FLOW_ITEM_TYPE_VXLAN entry specification.
2984  * @param[out] encap
2985  *   Structure to fill the gathered VNI address data.
2986  */
2987 static void
2988 flow_tcf_parse_vxlan_encap_vni(const struct rte_flow_item_vxlan *spec,
2989                                struct flow_tcf_vxlan_encap *encap)
2990 {
2991         /* Item must be validated before. Do not redundant checks. */
2992         assert(spec);
2993         memcpy(encap->vxlan.vni, spec->vni, sizeof(encap->vxlan.vni));
2994         encap->mask |= FLOW_TCF_ENCAP_VXLAN_VNI;
2995 }
2996
2997 /**
2998  * Populate consolidated encapsulation object from list of pattern items.
2999  *
3000  * Helper function to process configuration of action such as
3001  * RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. The item list should be
3002  * validated, there is no way to return an meaningful error.
3003  *
3004  * @param[in] action
3005  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
3006  *   List of pattern items to gather data from.
3007  * @param[out] src
3008  *   Structure to fill gathered data.
3009  */
3010 static void
3011 flow_tcf_vxlan_encap_parse(const struct rte_flow_action *action,
3012                            struct flow_tcf_vxlan_encap *encap)
3013 {
3014         union {
3015                 const struct rte_flow_item_eth *eth;
3016                 const struct rte_flow_item_ipv4 *ipv4;
3017                 const struct rte_flow_item_ipv6 *ipv6;
3018                 const struct rte_flow_item_udp *udp;
3019                 const struct rte_flow_item_vxlan *vxlan;
3020         } spec, mask;
3021         const struct rte_flow_item *items;
3022
3023         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
3024         assert(action->conf);
3025
3026         items = ((const struct rte_flow_action_vxlan_encap *)
3027                                         action->conf)->definition;
3028         assert(items);
3029         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3030                 switch (items->type) {
3031                 case RTE_FLOW_ITEM_TYPE_VOID:
3032                         break;
3033                 case RTE_FLOW_ITEM_TYPE_ETH:
3034                         mask.eth = items->mask;
3035                         spec.eth = items->spec;
3036                         flow_tcf_parse_vxlan_encap_eth(spec.eth, mask.eth,
3037                                                        encap);
3038                         break;
3039                 case RTE_FLOW_ITEM_TYPE_IPV4:
3040                         spec.ipv4 = items->spec;
3041                         flow_tcf_parse_vxlan_encap_ipv4(spec.ipv4, encap);
3042                         break;
3043                 case RTE_FLOW_ITEM_TYPE_IPV6:
3044                         spec.ipv6 = items->spec;
3045                         flow_tcf_parse_vxlan_encap_ipv6(spec.ipv6, encap);
3046                         break;
3047                 case RTE_FLOW_ITEM_TYPE_UDP:
3048                         mask.udp = items->mask;
3049                         spec.udp = items->spec;
3050                         flow_tcf_parse_vxlan_encap_udp(spec.udp, mask.udp,
3051                                                        encap);
3052                         break;
3053                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3054                         spec.vxlan = items->spec;
3055                         flow_tcf_parse_vxlan_encap_vni(spec.vxlan, encap);
3056                         break;
3057                 default:
3058                         assert(false);
3059                         DRV_LOG(WARNING,
3060                                 "unsupported item %p type %d,"
3061                                 " items must be validated"
3062                                 " before flow creation",
3063                                 (const void *)items, items->type);
3064                         encap->mask = 0;
3065                         return;
3066                 }
3067         }
3068 }
3069
3070 /**
3071  * Translate flow for Linux TC flower and construct Netlink message.
3072  *
3073  * @param[in] priv
3074  *   Pointer to the priv structure.
3075  * @param[in, out] flow
3076  *   Pointer to the sub flow.
3077  * @param[in] attr
3078  *   Pointer to the flow attributes.
3079  * @param[in] items
3080  *   Pointer to the list of items.
3081  * @param[in] actions
3082  *   Pointer to the list of actions.
3083  * @param[out] error
3084  *   Pointer to the error structure.
3085  *
3086  * @return
3087  *   0 on success, a negative errno value otherwise and rte_errno is set.
3088  */
3089 static int
3090 flow_tcf_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3091                    const struct rte_flow_attr *attr,
3092                    const struct rte_flow_item items[],
3093                    const struct rte_flow_action actions[],
3094                    struct rte_flow_error *error)
3095 {
3096         union {
3097                 const struct rte_flow_item_port_id *port_id;
3098                 const struct rte_flow_item_eth *eth;
3099                 const struct rte_flow_item_vlan *vlan;
3100                 const struct rte_flow_item_ipv4 *ipv4;
3101                 const struct rte_flow_item_ipv6 *ipv6;
3102                 const struct rte_flow_item_tcp *tcp;
3103                 const struct rte_flow_item_udp *udp;
3104                 const struct rte_flow_item_vxlan *vxlan;
3105         } spec, mask;
3106         union {
3107                 const struct rte_flow_action_port_id *port_id;
3108                 const struct rte_flow_action_jump *jump;
3109                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
3110                 const struct rte_flow_action_of_set_vlan_vid *
3111                         of_set_vlan_vid;
3112                 const struct rte_flow_action_of_set_vlan_pcp *
3113                         of_set_vlan_pcp;
3114         } conf;
3115         union {
3116                 struct flow_tcf_tunnel_hdr *hdr;
3117                 struct flow_tcf_vxlan_decap *vxlan;
3118         } decap = {
3119                 .hdr = NULL,
3120         };
3121         union {
3122                 struct flow_tcf_tunnel_hdr *hdr;
3123                 struct flow_tcf_vxlan_encap *vxlan;
3124         } encap = {
3125                 .hdr = NULL,
3126         };
3127         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
3128         struct nlmsghdr *nlh = dev_flow->tcf.nlh;
3129         struct tcmsg *tcm = dev_flow->tcf.tcm;
3130         uint32_t na_act_index_cur;
3131         rte_be16_t inner_etype = RTE_BE16(ETH_P_ALL);
3132         rte_be16_t outer_etype = RTE_BE16(ETH_P_ALL);
3133         rte_be16_t vlan_etype = RTE_BE16(ETH_P_ALL);
3134         bool ip_proto_set = 0;
3135         bool tunnel_outer = 0;
3136         struct nlattr *na_flower;
3137         struct nlattr *na_flower_act;
3138         struct nlattr *na_vlan_id = NULL;
3139         struct nlattr *na_vlan_priority = NULL;
3140         uint64_t item_flags = 0;
3141         int ret;
3142
3143         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
3144                                                 PTOI_TABLE_SZ_MAX(dev)));
3145         if (dev_flow->tcf.tunnel) {
3146                 switch (dev_flow->tcf.tunnel->type) {
3147                 case FLOW_TCF_TUNACT_VXLAN_DECAP:
3148                         decap.vxlan = dev_flow->tcf.vxlan_decap;
3149                         tunnel_outer = 1;
3150                         break;
3151                 case FLOW_TCF_TUNACT_VXLAN_ENCAP:
3152                         encap.vxlan = dev_flow->tcf.vxlan_encap;
3153                         break;
3154                 /* New tunnel actions can be added here. */
3155                 default:
3156                         assert(false);
3157                         break;
3158                 }
3159         }
3160         nlh = dev_flow->tcf.nlh;
3161         tcm = dev_flow->tcf.tcm;
3162         /* Prepare API must have been called beforehand. */
3163         assert(nlh != NULL && tcm != NULL);
3164         tcm->tcm_family = AF_UNSPEC;
3165         tcm->tcm_ifindex = ptoi[0].ifindex;
3166         tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
3167         /*
3168          * Priority cannot be zero to prevent the kernel from picking one
3169          * automatically.
3170          */
3171         tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16, outer_etype);
3172         if (attr->group > 0)
3173                 mnl_attr_put_u32(nlh, TCA_CHAIN, attr->group);
3174         mnl_attr_put_strz(nlh, TCA_KIND, "flower");
3175         na_flower = mnl_attr_nest_start(nlh, TCA_OPTIONS);
3176         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3177                 unsigned int i;
3178
3179                 switch (items->type) {
3180                 case RTE_FLOW_ITEM_TYPE_VOID:
3181                         break;
3182                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3183                         mask.port_id = flow_tcf_item_mask
3184                                 (items, &rte_flow_item_port_id_mask,
3185                                  &flow_tcf_mask_supported.port_id,
3186                                  &flow_tcf_mask_empty.port_id,
3187                                  sizeof(flow_tcf_mask_supported.port_id),
3188                                  error);
3189                         assert(mask.port_id);
3190                         if (mask.port_id == &flow_tcf_mask_empty.port_id)
3191                                 break;
3192                         spec.port_id = items->spec;
3193                         if (!mask.port_id->id)
3194                                 i = 0;
3195                         else
3196                                 for (i = 0; ptoi[i].ifindex; ++i)
3197                                         if (ptoi[i].port_id == spec.port_id->id)
3198                                                 break;
3199                         assert(ptoi[i].ifindex);
3200                         tcm->tcm_ifindex = ptoi[i].ifindex;
3201                         break;
3202                 case RTE_FLOW_ITEM_TYPE_ETH:
3203                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3204                                       MLX5_FLOW_LAYER_INNER_L2 :
3205                                       MLX5_FLOW_LAYER_OUTER_L2;
3206                         mask.eth = flow_tcf_item_mask
3207                                 (items, &rte_flow_item_eth_mask,
3208                                  &flow_tcf_mask_supported.eth,
3209                                  &flow_tcf_mask_empty.eth,
3210                                  sizeof(flow_tcf_mask_supported.eth),
3211                                  error);
3212                         assert(mask.eth);
3213                         if (mask.eth == &flow_tcf_mask_empty.eth)
3214                                 break;
3215                         spec.eth = items->spec;
3216                         if (mask.eth->type) {
3217                                 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3218                                         inner_etype = spec.eth->type;
3219                                 else
3220                                         outer_etype = spec.eth->type;
3221                         }
3222                         if (tunnel_outer) {
3223                                 DRV_LOG(WARNING,
3224                                         "outer L2 addresses cannot be"
3225                                         " forced is outer ones for tunnel,"
3226                                         " parameter is ignored");
3227                                 break;
3228                         }
3229                         if (!is_zero_ether_addr(&mask.eth->dst)) {
3230                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST,
3231                                              ETHER_ADDR_LEN,
3232                                              spec.eth->dst.addr_bytes);
3233                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST_MASK,
3234                                              ETHER_ADDR_LEN,
3235                                              mask.eth->dst.addr_bytes);
3236                         }
3237                         if (!is_zero_ether_addr(&mask.eth->src)) {
3238                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC,
3239                                              ETHER_ADDR_LEN,
3240                                              spec.eth->src.addr_bytes);
3241                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC_MASK,
3242                                              ETHER_ADDR_LEN,
3243                                              mask.eth->src.addr_bytes);
3244                         }
3245                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3246                         break;
3247                 case RTE_FLOW_ITEM_TYPE_VLAN:
3248                         assert(!encap.hdr);
3249                         assert(!decap.hdr);
3250                         assert(!tunnel_outer);
3251                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
3252                         mask.vlan = flow_tcf_item_mask
3253                                 (items, &rte_flow_item_vlan_mask,
3254                                  &flow_tcf_mask_supported.vlan,
3255                                  &flow_tcf_mask_empty.vlan,
3256                                  sizeof(flow_tcf_mask_supported.vlan),
3257                                  error);
3258                         assert(mask.vlan);
3259                         if (mask.vlan == &flow_tcf_mask_empty.vlan)
3260                                 break;
3261                         spec.vlan = items->spec;
3262                         assert(outer_etype == RTE_BE16(ETH_P_ALL) ||
3263                                outer_etype == RTE_BE16(ETH_P_8021Q));
3264                         outer_etype = RTE_BE16(ETH_P_8021Q);
3265                         if (mask.vlan->inner_type)
3266                                 vlan_etype = spec.vlan->inner_type;
3267                         if (mask.vlan->tci & RTE_BE16(0xe000))
3268                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_VLAN_PRIO,
3269                                                 (rte_be_to_cpu_16
3270                                                  (spec.vlan->tci) >> 13) & 0x7);
3271                         if (mask.vlan->tci & RTE_BE16(0x0fff))
3272                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_VLAN_ID,
3273                                                  rte_be_to_cpu_16
3274                                                  (spec.vlan->tci &
3275                                                   RTE_BE16(0x0fff)));
3276                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3277                         break;
3278                 case RTE_FLOW_ITEM_TYPE_IPV4:
3279                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3280                                       MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3281                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3282                         mask.ipv4 = flow_tcf_item_mask
3283                                 (items, &rte_flow_item_ipv4_mask,
3284                                  &flow_tcf_mask_supported.ipv4,
3285                                  &flow_tcf_mask_empty.ipv4,
3286                                  sizeof(flow_tcf_mask_supported.ipv4),
3287                                  error);
3288                         assert(mask.ipv4);
3289                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
3290                                 assert(inner_etype == RTE_BE16(ETH_P_ALL) ||
3291                                        inner_etype == RTE_BE16(ETH_P_IP));
3292                                 inner_etype = RTE_BE16(ETH_P_IP);
3293                         } else if (outer_etype == RTE_BE16(ETH_P_8021Q)) {
3294                                 assert(vlan_etype == RTE_BE16(ETH_P_ALL) ||
3295                                        vlan_etype == RTE_BE16(ETH_P_IP));
3296                                 vlan_etype = RTE_BE16(ETH_P_IP);
3297                         } else {
3298                                 assert(outer_etype == RTE_BE16(ETH_P_ALL) ||
3299                                        outer_etype == RTE_BE16(ETH_P_IP));
3300                                 outer_etype = RTE_BE16(ETH_P_IP);
3301                         }
3302                         spec.ipv4 = items->spec;
3303                         if (!tunnel_outer && mask.ipv4->hdr.next_proto_id) {
3304                                 /*
3305                                  * No way to set IP protocol for outer tunnel
3306                                  * layers. Usually it is fixed, for example,
3307                                  * to UDP for VXLAN/GPE.
3308                                  */
3309                                 assert(spec.ipv4); /* Mask is not empty. */
3310                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3311                                                 spec.ipv4->hdr.next_proto_id);
3312                                 ip_proto_set = 1;
3313                         }
3314                         if (mask.ipv4 == &flow_tcf_mask_empty.ipv4 ||
3315                              (!mask.ipv4->hdr.src_addr &&
3316                               !mask.ipv4->hdr.dst_addr)) {
3317                                 if (!tunnel_outer)
3318                                         break;
3319                                 /*
3320                                  * For tunnel outer we must set outer IP key
3321                                  * anyway, even if the specification/mask is
3322                                  * empty. There is no another way to tell
3323                                  * kernel about he outer layer protocol.
3324                                  */
3325                                 mnl_attr_put_u32
3326                                         (nlh, TCA_FLOWER_KEY_ENC_IPV4_SRC,
3327                                          mask.ipv4->hdr.src_addr);
3328                                 mnl_attr_put_u32
3329                                         (nlh, TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
3330                                          mask.ipv4->hdr.src_addr);
3331                                 assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3332                                 break;
3333                         }
3334                         if (mask.ipv4->hdr.src_addr) {
3335                                 mnl_attr_put_u32
3336                                         (nlh, tunnel_outer ?
3337                                          TCA_FLOWER_KEY_ENC_IPV4_SRC :
3338                                          TCA_FLOWER_KEY_IPV4_SRC,
3339                                          spec.ipv4->hdr.src_addr);
3340                                 mnl_attr_put_u32
3341                                         (nlh, tunnel_outer ?
3342                                          TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK :
3343                                          TCA_FLOWER_KEY_IPV4_SRC_MASK,
3344                                          mask.ipv4->hdr.src_addr);
3345                         }
3346                         if (mask.ipv4->hdr.dst_addr) {
3347                                 mnl_attr_put_u32
3348                                         (nlh, tunnel_outer ?
3349                                          TCA_FLOWER_KEY_ENC_IPV4_DST :
3350                                          TCA_FLOWER_KEY_IPV4_DST,
3351                                          spec.ipv4->hdr.dst_addr);
3352                                 mnl_attr_put_u32
3353                                         (nlh, tunnel_outer ?
3354                                          TCA_FLOWER_KEY_ENC_IPV4_DST_MASK :
3355                                          TCA_FLOWER_KEY_IPV4_DST_MASK,
3356                                          mask.ipv4->hdr.dst_addr);
3357                         }
3358                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3359                         break;
3360                 case RTE_FLOW_ITEM_TYPE_IPV6: {
3361                         bool ipv6_src, ipv6_dst;
3362
3363                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3364                                       MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3365                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3366                         mask.ipv6 = flow_tcf_item_mask
3367                                 (items, &rte_flow_item_ipv6_mask,
3368                                  &flow_tcf_mask_supported.ipv6,
3369                                  &flow_tcf_mask_empty.ipv6,
3370                                  sizeof(flow_tcf_mask_supported.ipv6),
3371                                  error);
3372                         assert(mask.ipv6);
3373                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
3374                                 assert(inner_etype == RTE_BE16(ETH_P_ALL) ||
3375                                        inner_etype == RTE_BE16(ETH_P_IPV6));
3376                                 inner_etype = RTE_BE16(ETH_P_IPV6);
3377                         } else if (outer_etype == RTE_BE16(ETH_P_8021Q)) {
3378                                 assert(vlan_etype == RTE_BE16(ETH_P_ALL) ||
3379                                        vlan_etype == RTE_BE16(ETH_P_IPV6));
3380                                 vlan_etype = RTE_BE16(ETH_P_IPV6);
3381                         } else {
3382                                 assert(outer_etype == RTE_BE16(ETH_P_ALL) ||
3383                                        outer_etype == RTE_BE16(ETH_P_IPV6));
3384                                 outer_etype = RTE_BE16(ETH_P_IPV6);
3385                         }
3386                         spec.ipv6 = items->spec;
3387                         if (!tunnel_outer && mask.ipv6->hdr.proto) {
3388                                 /*
3389                                  * No way to set IP protocol for outer tunnel
3390                                  * layers. Usually it is fixed, for example,
3391                                  * to UDP for VXLAN/GPE.
3392                                  */
3393                                 assert(spec.ipv6); /* Mask is not empty. */
3394                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3395                                                 spec.ipv6->hdr.proto);
3396                                 ip_proto_set = 1;
3397                         }
3398                         ipv6_dst = !IN6_IS_ADDR_UNSPECIFIED
3399                                                 (mask.ipv6->hdr.dst_addr);
3400                         ipv6_src = !IN6_IS_ADDR_UNSPECIFIED
3401                                                 (mask.ipv6->hdr.src_addr);
3402                         if (mask.ipv6 == &flow_tcf_mask_empty.ipv6 ||
3403                              (!ipv6_dst && !ipv6_src)) {
3404                                 if (!tunnel_outer)
3405                                         break;
3406                                 /*
3407                                  * For tunnel outer we must set outer IP key
3408                                  * anyway, even if the specification/mask is
3409                                  * empty. There is no another way to tell
3410                                  * kernel about he outer layer protocol.
3411                                  */
3412                                 mnl_attr_put(nlh,
3413                                              TCA_FLOWER_KEY_ENC_IPV6_SRC,
3414                                              IPV6_ADDR_LEN,
3415                                              mask.ipv6->hdr.src_addr);
3416                                 mnl_attr_put(nlh,
3417                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
3418                                              IPV6_ADDR_LEN,
3419                                              mask.ipv6->hdr.src_addr);
3420                                 assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3421                                 break;
3422                         }
3423                         if (ipv6_src) {
3424                                 mnl_attr_put(nlh, tunnel_outer ?
3425                                              TCA_FLOWER_KEY_ENC_IPV6_SRC :
3426                                              TCA_FLOWER_KEY_IPV6_SRC,
3427                                              IPV6_ADDR_LEN,
3428                                              spec.ipv6->hdr.src_addr);
3429                                 mnl_attr_put(nlh, tunnel_outer ?
3430                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK :
3431                                              TCA_FLOWER_KEY_IPV6_SRC_MASK,
3432                                              IPV6_ADDR_LEN,
3433                                              mask.ipv6->hdr.src_addr);
3434                         }
3435                         if (ipv6_dst) {
3436                                 mnl_attr_put(nlh, tunnel_outer ?
3437                                              TCA_FLOWER_KEY_ENC_IPV6_DST :
3438                                              TCA_FLOWER_KEY_IPV6_DST,
3439                                              IPV6_ADDR_LEN,
3440                                              spec.ipv6->hdr.dst_addr);
3441                                 mnl_attr_put(nlh, tunnel_outer ?
3442                                              TCA_FLOWER_KEY_ENC_IPV6_DST_MASK :
3443                                              TCA_FLOWER_KEY_IPV6_DST_MASK,
3444                                              IPV6_ADDR_LEN,
3445                                              mask.ipv6->hdr.dst_addr);
3446                         }
3447                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3448                         break;
3449                 }
3450                 case RTE_FLOW_ITEM_TYPE_UDP:
3451                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3452                                       MLX5_FLOW_LAYER_INNER_L4_UDP :
3453                                       MLX5_FLOW_LAYER_OUTER_L4_UDP;
3454                         mask.udp = flow_tcf_item_mask
3455                                 (items, &rte_flow_item_udp_mask,
3456                                  &flow_tcf_mask_supported.udp,
3457                                  &flow_tcf_mask_empty.udp,
3458                                  sizeof(flow_tcf_mask_supported.udp),
3459                                  error);
3460                         assert(mask.udp);
3461                         spec.udp = items->spec;
3462                         if (!tunnel_outer) {
3463                                 if (!ip_proto_set)
3464                                         mnl_attr_put_u8
3465                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3466                                                 IPPROTO_UDP);
3467                                 if (mask.udp == &flow_tcf_mask_empty.udp)
3468                                         break;
3469                         } else {
3470                                 assert(mask.udp != &flow_tcf_mask_empty.udp);
3471                                 decap.vxlan->udp_port =
3472                                         rte_be_to_cpu_16
3473                                                 (spec.udp->hdr.dst_port);
3474                         }
3475                         if (mask.udp->hdr.src_port) {
3476                                 mnl_attr_put_u16
3477                                         (nlh, tunnel_outer ?
3478                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT :
3479                                          TCA_FLOWER_KEY_UDP_SRC,
3480                                          spec.udp->hdr.src_port);
3481                                 mnl_attr_put_u16
3482                                         (nlh, tunnel_outer ?
3483                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK :
3484                                          TCA_FLOWER_KEY_UDP_SRC_MASK,
3485                                          mask.udp->hdr.src_port);
3486                         }
3487                         if (mask.udp->hdr.dst_port) {
3488                                 mnl_attr_put_u16
3489                                         (nlh, tunnel_outer ?
3490                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT :
3491                                          TCA_FLOWER_KEY_UDP_DST,
3492                                          spec.udp->hdr.dst_port);
3493                                 mnl_attr_put_u16
3494                                         (nlh, tunnel_outer ?
3495                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK :
3496                                          TCA_FLOWER_KEY_UDP_DST_MASK,
3497                                          mask.udp->hdr.dst_port);
3498                         }
3499                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3500                         break;
3501                 case RTE_FLOW_ITEM_TYPE_TCP:
3502                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3503                                       MLX5_FLOW_LAYER_INNER_L4_TCP :
3504                                       MLX5_FLOW_LAYER_OUTER_L4_TCP;
3505                         mask.tcp = flow_tcf_item_mask
3506                                 (items, &rte_flow_item_tcp_mask,
3507                                  &flow_tcf_mask_supported.tcp,
3508                                  &flow_tcf_mask_empty.tcp,
3509                                  sizeof(flow_tcf_mask_supported.tcp),
3510                                  error);
3511                         assert(mask.tcp);
3512                         if (!ip_proto_set)
3513                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3514                                                 IPPROTO_TCP);
3515                         if (mask.tcp == &flow_tcf_mask_empty.tcp)
3516                                 break;
3517                         spec.tcp = items->spec;
3518                         if (mask.tcp->hdr.src_port) {
3519                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_SRC,
3520                                                  spec.tcp->hdr.src_port);
3521                                 mnl_attr_put_u16(nlh,
3522                                                  TCA_FLOWER_KEY_TCP_SRC_MASK,
3523                                                  mask.tcp->hdr.src_port);
3524                         }
3525                         if (mask.tcp->hdr.dst_port) {
3526                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_DST,
3527                                                  spec.tcp->hdr.dst_port);
3528                                 mnl_attr_put_u16(nlh,
3529                                                  TCA_FLOWER_KEY_TCP_DST_MASK,
3530                                                  mask.tcp->hdr.dst_port);
3531                         }
3532                         if (mask.tcp->hdr.tcp_flags) {
3533                                 mnl_attr_put_u16
3534                                         (nlh,
3535                                          TCA_FLOWER_KEY_TCP_FLAGS,
3536                                          rte_cpu_to_be_16
3537                                                 (spec.tcp->hdr.tcp_flags));
3538                                 mnl_attr_put_u16
3539                                         (nlh,
3540                                          TCA_FLOWER_KEY_TCP_FLAGS_MASK,
3541                                          rte_cpu_to_be_16
3542                                                 (mask.tcp->hdr.tcp_flags));
3543                         }
3544                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3545                         break;
3546                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3547                         assert(decap.vxlan);
3548                         tunnel_outer = 0;
3549                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
3550                         spec.vxlan = items->spec;
3551                         mnl_attr_put_u32(nlh,
3552                                          TCA_FLOWER_KEY_ENC_KEY_ID,
3553                                          vxlan_vni_as_be32(spec.vxlan->vni));
3554                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3555                         break;
3556                 default:
3557                         return rte_flow_error_set(error, ENOTSUP,
3558                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3559                                                   NULL, "item not supported");
3560                 }
3561         }
3562         /*
3563          * Set the ether_type flower key and tc rule protocol:
3564          * - if there is nor VLAN neither VXLAN the key is taken from
3565          *   eth item directly or deduced from L3 items.
3566          * - if there is vlan item then key is fixed to 802.1q.
3567          * - if there is vxlan item then key is set to inner tunnel type.
3568          * - simultaneous vlan and vxlan items are prohibited.
3569          */
3570         if (outer_etype != RTE_BE16(ETH_P_ALL)) {
3571                 tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
3572                                            outer_etype);
3573                 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
3574                         if (inner_etype != RTE_BE16(ETH_P_ALL))
3575                                 mnl_attr_put_u16(nlh,
3576                                                  TCA_FLOWER_KEY_ETH_TYPE,
3577                                                  inner_etype);
3578                 } else {
3579                         mnl_attr_put_u16(nlh,
3580                                          TCA_FLOWER_KEY_ETH_TYPE,
3581                                          outer_etype);
3582                         if (outer_etype == RTE_BE16(ETH_P_8021Q) &&
3583                             vlan_etype != RTE_BE16(ETH_P_ALL))
3584                                 mnl_attr_put_u16(nlh,
3585                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE,
3586                                                  vlan_etype);
3587                 }
3588                 assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3589         }
3590         na_flower_act = mnl_attr_nest_start(nlh, TCA_FLOWER_ACT);
3591         na_act_index_cur = 1;
3592         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3593                 struct nlattr *na_act_index;
3594                 struct nlattr *na_act;
3595                 unsigned int vlan_act;
3596                 unsigned int i;
3597
3598                 switch (actions->type) {
3599                 case RTE_FLOW_ACTION_TYPE_VOID:
3600                         break;
3601                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3602                         conf.port_id = actions->conf;
3603                         if (conf.port_id->original)
3604                                 i = 0;
3605                         else
3606                                 for (i = 0; ptoi[i].ifindex; ++i)
3607                                         if (ptoi[i].port_id == conf.port_id->id)
3608                                                 break;
3609                         assert(ptoi[i].ifindex);
3610                         na_act_index =
3611                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3612                         assert(na_act_index);
3613                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "mirred");
3614                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3615                         assert(na_act);
3616                         if (encap.hdr) {
3617                                 assert(dev_flow->tcf.tunnel);
3618                                 dev_flow->tcf.tunnel->ifindex_ptr =
3619                                         &((struct tc_mirred *)
3620                                         mnl_attr_get_payload
3621                                         (mnl_nlmsg_get_payload_tail
3622                                                 (nlh)))->ifindex;
3623                         }
3624                         mnl_attr_put(nlh, TCA_MIRRED_PARMS,
3625                                      sizeof(struct tc_mirred),
3626                                      &(struct tc_mirred){
3627                                         .action = TC_ACT_STOLEN,
3628                                         .eaction = TCA_EGRESS_REDIR,
3629                                         .ifindex = ptoi[i].ifindex,
3630                                      });
3631                         mnl_attr_nest_end(nlh, na_act);
3632                         mnl_attr_nest_end(nlh, na_act_index);
3633                         break;
3634                 case RTE_FLOW_ACTION_TYPE_JUMP:
3635                         conf.jump = actions->conf;
3636                         na_act_index =
3637                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3638                         assert(na_act_index);
3639                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3640                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3641                         assert(na_act);
3642                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3643                                      sizeof(struct tc_gact),
3644                                      &(struct tc_gact){
3645                                         .action = TC_ACT_GOTO_CHAIN |
3646                                                   conf.jump->group,
3647                                      });
3648                         mnl_attr_nest_end(nlh, na_act);
3649                         mnl_attr_nest_end(nlh, na_act_index);
3650                         break;
3651                 case RTE_FLOW_ACTION_TYPE_DROP:
3652                         na_act_index =
3653                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3654                         assert(na_act_index);
3655                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3656                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3657                         assert(na_act);
3658                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3659                                      sizeof(struct tc_gact),
3660                                      &(struct tc_gact){
3661                                         .action = TC_ACT_SHOT,
3662                                      });
3663                         mnl_attr_nest_end(nlh, na_act);
3664                         mnl_attr_nest_end(nlh, na_act_index);
3665                         break;
3666                 case RTE_FLOW_ACTION_TYPE_COUNT:
3667                         /*
3668                          * Driver adds the count action implicitly for
3669                          * each rule it creates.
3670                          */
3671                         ret = flow_tcf_translate_action_count(dev,
3672                                                               dev_flow, error);
3673                         if (ret < 0)
3674                                 return ret;
3675                         break;
3676                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3677                         conf.of_push_vlan = NULL;
3678                         vlan_act = TCA_VLAN_ACT_POP;
3679                         goto action_of_vlan;
3680                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3681                         conf.of_push_vlan = actions->conf;
3682                         vlan_act = TCA_VLAN_ACT_PUSH;
3683                         goto action_of_vlan;
3684                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3685                         conf.of_set_vlan_vid = actions->conf;
3686                         if (na_vlan_id)
3687                                 goto override_na_vlan_id;
3688                         vlan_act = TCA_VLAN_ACT_MODIFY;
3689                         goto action_of_vlan;
3690                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3691                         conf.of_set_vlan_pcp = actions->conf;
3692                         if (na_vlan_priority)
3693                                 goto override_na_vlan_priority;
3694                         vlan_act = TCA_VLAN_ACT_MODIFY;
3695                         goto action_of_vlan;
3696 action_of_vlan:
3697                         na_act_index =
3698                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3699                         assert(na_act_index);
3700                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "vlan");
3701                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3702                         assert(na_act);
3703                         mnl_attr_put(nlh, TCA_VLAN_PARMS,
3704                                      sizeof(struct tc_vlan),
3705                                      &(struct tc_vlan){
3706                                         .action = TC_ACT_PIPE,
3707                                         .v_action = vlan_act,
3708                                      });
3709                         if (vlan_act == TCA_VLAN_ACT_POP) {
3710                                 mnl_attr_nest_end(nlh, na_act);
3711                                 mnl_attr_nest_end(nlh, na_act_index);
3712                                 break;
3713                         }
3714                         if (vlan_act == TCA_VLAN_ACT_PUSH)
3715                                 mnl_attr_put_u16(nlh,
3716                                                  TCA_VLAN_PUSH_VLAN_PROTOCOL,
3717                                                  conf.of_push_vlan->ethertype);
3718                         na_vlan_id = mnl_nlmsg_get_payload_tail(nlh);
3719                         mnl_attr_put_u16(nlh, TCA_VLAN_PAD, 0);
3720                         na_vlan_priority = mnl_nlmsg_get_payload_tail(nlh);
3721                         mnl_attr_put_u8(nlh, TCA_VLAN_PAD, 0);
3722                         mnl_attr_nest_end(nlh, na_act);
3723                         mnl_attr_nest_end(nlh, na_act_index);
3724                         if (actions->type ==
3725                             RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
3726 override_na_vlan_id:
3727                                 na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
3728                                 *(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
3729                                         rte_be_to_cpu_16
3730                                         (conf.of_set_vlan_vid->vlan_vid);
3731                         } else if (actions->type ==
3732                                    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
3733 override_na_vlan_priority:
3734                                 na_vlan_priority->nla_type =
3735                                         TCA_VLAN_PUSH_VLAN_PRIORITY;
3736                                 *(uint8_t *)mnl_attr_get_payload
3737                                         (na_vlan_priority) =
3738                                         conf.of_set_vlan_pcp->vlan_pcp;
3739                         }
3740                         break;
3741                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3742                         assert(decap.vxlan);
3743                         assert(dev_flow->tcf.tunnel);
3744                         dev_flow->tcf.tunnel->ifindex_ptr =
3745                                 (unsigned int *)&tcm->tcm_ifindex;
3746                         na_act_index =
3747                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3748                         assert(na_act_index);
3749                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3750                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3751                         assert(na_act);
3752                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3753                                 sizeof(struct tc_tunnel_key),
3754                                 &(struct tc_tunnel_key){
3755                                         .action = TC_ACT_PIPE,
3756                                         .t_action = TCA_TUNNEL_KEY_ACT_RELEASE,
3757                                         });
3758                         mnl_attr_nest_end(nlh, na_act);
3759                         mnl_attr_nest_end(nlh, na_act_index);
3760                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3761                         break;
3762                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3763                         assert(encap.vxlan);
3764                         flow_tcf_vxlan_encap_parse(actions, encap.vxlan);
3765                         na_act_index =
3766                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3767                         assert(na_act_index);
3768                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3769                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3770                         assert(na_act);
3771                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3772                                 sizeof(struct tc_tunnel_key),
3773                                 &(struct tc_tunnel_key){
3774                                         .action = TC_ACT_PIPE,
3775                                         .t_action = TCA_TUNNEL_KEY_ACT_SET,
3776                                         });
3777                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_UDP_DST)
3778                                 mnl_attr_put_u16(nlh,
3779                                          TCA_TUNNEL_KEY_ENC_DST_PORT,
3780                                          encap.vxlan->udp.dst);
3781                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_SRC)
3782                                 mnl_attr_put_u32(nlh,
3783                                          TCA_TUNNEL_KEY_ENC_IPV4_SRC,
3784                                          encap.vxlan->ipv4.src);
3785                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_DST)
3786                                 mnl_attr_put_u32(nlh,
3787                                          TCA_TUNNEL_KEY_ENC_IPV4_DST,
3788                                          encap.vxlan->ipv4.dst);
3789                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_SRC)
3790                                 mnl_attr_put(nlh,
3791                                          TCA_TUNNEL_KEY_ENC_IPV6_SRC,
3792                                          sizeof(encap.vxlan->ipv6.src),
3793                                          &encap.vxlan->ipv6.src);
3794                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_DST)
3795                                 mnl_attr_put(nlh,
3796                                          TCA_TUNNEL_KEY_ENC_IPV6_DST,
3797                                          sizeof(encap.vxlan->ipv6.dst),
3798                                          &encap.vxlan->ipv6.dst);
3799                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_VXLAN_VNI)
3800                                 mnl_attr_put_u32(nlh,
3801                                          TCA_TUNNEL_KEY_ENC_KEY_ID,
3802                                          vxlan_vni_as_be32
3803                                                 (encap.vxlan->vxlan.vni));
3804                         mnl_attr_put_u8(nlh, TCA_TUNNEL_KEY_NO_CSUM, 0);
3805                         mnl_attr_nest_end(nlh, na_act);
3806                         mnl_attr_nest_end(nlh, na_act_index);
3807                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3808                         break;
3809                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3810                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3811                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3812                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3813                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3814                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3815                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3816                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3817                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3818                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3819                         na_act_index =
3820                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3821                         flow_tcf_create_pedit_mnl_msg(nlh,
3822                                                       &actions, item_flags);
3823                         mnl_attr_nest_end(nlh, na_act_index);
3824                         break;
3825                 default:
3826                         return rte_flow_error_set(error, ENOTSUP,
3827                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3828                                                   actions,
3829                                                   "action not supported");
3830                 }
3831         }
3832         assert(na_flower);
3833         assert(na_flower_act);
3834         mnl_attr_nest_end(nlh, na_flower_act);
3835         dev_flow->tcf.ptc_flags = mnl_attr_get_payload
3836                                         (mnl_nlmsg_get_payload_tail(nlh));
3837         mnl_attr_put_u32(nlh, TCA_FLOWER_FLAGS, decap.vxlan ?
3838                                                 0 : TCA_CLS_FLAGS_SKIP_SW);
3839         mnl_attr_nest_end(nlh, na_flower);
3840         if (dev_flow->tcf.tunnel && dev_flow->tcf.tunnel->ifindex_ptr)
3841                 dev_flow->tcf.tunnel->ifindex_org =
3842                         *dev_flow->tcf.tunnel->ifindex_ptr;
3843         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3844         return 0;
3845 }
3846
3847 /**
3848  * Send Netlink message with acknowledgment.
3849  *
3850  * @param tcf
3851  *   Flow context to use.
3852  * @param nlh
3853  *   Message to send. This function always raises the NLM_F_ACK flag before
3854  *   sending.
3855  * @param[in] cb
3856  *   Callback handler for received message.
3857  * @param[in] arg
3858  *   Context pointer for callback handler.
3859  *
3860  * @return
3861  *   0 on success, a negative errno value otherwise and rte_errno is set.
3862  */
3863 static int
3864 flow_tcf_nl_ack(struct mlx5_flow_tcf_context *tcf,
3865                 struct nlmsghdr *nlh,
3866                 mnl_cb_t cb, void *arg)
3867 {
3868         unsigned int portid = mnl_socket_get_portid(tcf->nl);
3869         uint32_t seq = tcf->seq++;
3870         int ret, err = 0;
3871
3872         assert(tcf->nl);
3873         assert(tcf->buf);
3874         if (!seq) {
3875                 /* seq 0 is reserved for kernel event-driven notifications. */
3876                 seq = tcf->seq++;
3877         }
3878         nlh->nlmsg_seq = seq;
3879         nlh->nlmsg_flags |= NLM_F_ACK;
3880         ret = mnl_socket_sendto(tcf->nl, nlh, nlh->nlmsg_len);
3881         if (ret <= 0) {
3882                 /* Message send error occurres. */
3883                 rte_errno = errno;
3884                 return -rte_errno;
3885         }
3886         nlh = (struct nlmsghdr *)(tcf->buf);
3887         /*
3888          * The following loop postpones non-fatal errors until multipart
3889          * messages are complete.
3890          */
3891         while (true) {
3892                 ret = mnl_socket_recvfrom(tcf->nl, tcf->buf, tcf->buf_size);
3893                 if (ret < 0) {
3894                         err = errno;
3895                         /*
3896                          * In case of overflow Will receive till
3897                          * end of multipart message. We may lost part
3898                          * of reply messages but mark and return an error.
3899                          */
3900                         if (err != ENOSPC ||
3901                             !(nlh->nlmsg_flags & NLM_F_MULTI) ||
3902                             nlh->nlmsg_type == NLMSG_DONE)
3903                                 break;
3904                 } else {
3905                         ret = mnl_cb_run(nlh, ret, seq, portid, cb, arg);
3906                         if (!ret) {
3907                                 /*
3908                                  * libmnl returns 0 if DONE or
3909                                  * success ACK message found.
3910                                  */
3911                                 break;
3912                         }
3913                         if (ret < 0) {
3914                                 /*
3915                                  * ACK message with error found
3916                                  * or some error occurred.
3917                                  */
3918                                 err = errno;
3919                                 break;
3920                         }
3921                         /* We should continue receiving. */
3922                 }
3923         }
3924         if (!err)
3925                 return 0;
3926         rte_errno = err;
3927         return -err;
3928 }
3929
3930 #define MNL_BUF_EXTRA_SPACE 16
3931 #define MNL_REQUEST_SIZE_MIN 256
3932 #define MNL_REQUEST_SIZE_MAX 2048
3933 #define MNL_REQUEST_SIZE RTE_MIN(RTE_MAX(sysconf(_SC_PAGESIZE), \
3934                                  MNL_REQUEST_SIZE_MIN), MNL_REQUEST_SIZE_MAX)
3935
3936 /* Data structures used by flow_tcf_xxx_cb() routines. */
3937 struct tcf_nlcb_buf {
3938         LIST_ENTRY(tcf_nlcb_buf) next;
3939         uint32_t size;
3940         alignas(struct nlmsghdr)
3941         uint8_t msg[]; /**< Netlink message data. */
3942 };
3943
3944 struct tcf_nlcb_context {
3945         unsigned int ifindex; /**< Base interface index. */
3946         uint32_t bufsize;
3947         LIST_HEAD(, tcf_nlcb_buf) nlbuf;
3948 };
3949
3950 /**
3951  * Allocate space for netlink command in buffer list
3952  *
3953  * @param[in, out] ctx
3954  *   Pointer to callback context with command buffers list.
3955  * @param[in] size
3956  *   Required size of data buffer to be allocated.
3957  *
3958  * @return
3959  *   Pointer to allocated memory, aligned as message header.
3960  *   NULL if some error occurred.
3961  */
3962 static struct nlmsghdr *
3963 flow_tcf_alloc_nlcmd(struct tcf_nlcb_context *ctx, uint32_t size)
3964 {
3965         struct tcf_nlcb_buf *buf;
3966         struct nlmsghdr *nlh;
3967
3968         size = NLMSG_ALIGN(size);
3969         buf = LIST_FIRST(&ctx->nlbuf);
3970         if (buf && (buf->size + size) <= ctx->bufsize) {
3971                 nlh = (struct nlmsghdr *)&buf->msg[buf->size];
3972                 buf->size += size;
3973                 return nlh;
3974         }
3975         if (size > ctx->bufsize) {
3976                 DRV_LOG(WARNING, "netlink: too long command buffer requested");
3977                 return NULL;
3978         }
3979         buf = rte_malloc(__func__,
3980                         ctx->bufsize + sizeof(struct tcf_nlcb_buf),
3981                         alignof(struct tcf_nlcb_buf));
3982         if (!buf) {
3983                 DRV_LOG(WARNING, "netlink: no memory for command buffer");
3984                 return NULL;
3985         }
3986         LIST_INSERT_HEAD(&ctx->nlbuf, buf, next);
3987         buf->size = size;
3988         nlh = (struct nlmsghdr *)&buf->msg[0];
3989         return nlh;
3990 }
3991
3992 /**
3993  * Send the buffers with prepared netlink commands. Scans the list and
3994  * sends all found buffers. Buffers are sent and freed anyway in order
3995  * to prevent memory leakage if some every message in received packet.
3996  *
3997  * @param[in] tcf
3998  *   Context object initialized by mlx5_flow_tcf_context_create().
3999  * @param[in, out] ctx
4000  *   Pointer to callback context with command buffers list.
4001  *
4002  * @return
4003  *   Zero value on success, negative errno value otherwise
4004  *   and rte_errno is set.
4005  */
4006 static int
4007 flow_tcf_send_nlcmd(struct mlx5_flow_tcf_context *tcf,
4008                     struct tcf_nlcb_context *ctx)
4009 {
4010         struct tcf_nlcb_buf *bc = LIST_FIRST(&ctx->nlbuf);
4011         int ret = 0;
4012
4013         while (bc) {
4014                 struct tcf_nlcb_buf *bn = LIST_NEXT(bc, next);
4015                 struct nlmsghdr *nlh;
4016                 uint32_t msg = 0;
4017                 int rc;
4018
4019                 while (msg < bc->size) {
4020                         /*
4021                          * Send Netlink commands from buffer in one by one
4022                          * fashion. If we send multiple rule deletion commands
4023                          * in one Netlink message and some error occurs it may
4024                          * cause multiple ACK error messages and break sequence
4025                          * numbers of Netlink communication, because we expect
4026                          * the only one ACK reply.
4027                          */
4028                         assert((bc->size - msg) >= sizeof(struct nlmsghdr));
4029                         nlh = (struct nlmsghdr *)&bc->msg[msg];
4030                         assert((bc->size - msg) >= nlh->nlmsg_len);
4031                         msg += nlh->nlmsg_len;
4032                         rc = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4033                         if (rc) {
4034                                 DRV_LOG(WARNING,
4035                                         "netlink: cleanup error %d", rc);
4036                                 if (!ret)
4037                                         ret = rc;
4038                         }
4039                 }
4040                 rte_free(bc);
4041                 bc = bn;
4042         }
4043         LIST_INIT(&ctx->nlbuf);
4044         return ret;
4045 }
4046
4047 /**
4048  * Collect local IP address rules with scope link attribute  on specified
4049  * network device. This is callback routine called by libmnl mnl_cb_run()
4050  * in loop for every message in received packet.
4051  *
4052  * @param[in] nlh
4053  *   Pointer to reply header.
4054  * @param[in, out] arg
4055  *   Opaque data pointer for this callback.
4056  *
4057  * @return
4058  *   A positive, nonzero value on success, negative errno value otherwise
4059  *   and rte_errno is set.
4060  */
4061 static int
4062 flow_tcf_collect_local_cb(const struct nlmsghdr *nlh, void *arg)
4063 {
4064         struct tcf_nlcb_context *ctx = arg;
4065         struct nlmsghdr *cmd;
4066         struct ifaddrmsg *ifa;
4067         struct nlattr *na;
4068         struct nlattr *na_local = NULL;
4069         struct nlattr *na_peer = NULL;
4070         unsigned char family;
4071         uint32_t size;
4072
4073         if (nlh->nlmsg_type != RTM_NEWADDR) {
4074                 rte_errno = EINVAL;
4075                 return -rte_errno;
4076         }
4077         ifa = mnl_nlmsg_get_payload(nlh);
4078         family = ifa->ifa_family;
4079         if (ifa->ifa_index != ctx->ifindex ||
4080             ifa->ifa_scope != RT_SCOPE_LINK ||
4081             !(ifa->ifa_flags & IFA_F_PERMANENT) ||
4082             (family != AF_INET && family != AF_INET6))
4083                 return 1;
4084         mnl_attr_for_each(na, nlh, sizeof(*ifa)) {
4085                 switch (mnl_attr_get_type(na)) {
4086                 case IFA_LOCAL:
4087                         na_local = na;
4088                         break;
4089                 case IFA_ADDRESS:
4090                         na_peer = na;
4091                         break;
4092                 }
4093                 if (na_local && na_peer)
4094                         break;
4095         }
4096         if (!na_local || !na_peer)
4097                 return 1;
4098         /* Local rule found with scope link, permanent and assigned peer. */
4099         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4100                MNL_ALIGN(sizeof(struct ifaddrmsg)) +
4101                (family == AF_INET6 ? 2 * SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
4102                                    : 2 * SZ_NLATTR_TYPE_OF(uint32_t));
4103         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4104         if (!cmd) {
4105                 rte_errno = ENOMEM;
4106                 return -rte_errno;
4107         }
4108         cmd = mnl_nlmsg_put_header(cmd);
4109         cmd->nlmsg_type = RTM_DELADDR;
4110         cmd->nlmsg_flags = NLM_F_REQUEST;
4111         ifa = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifa));
4112         ifa->ifa_flags = IFA_F_PERMANENT;
4113         ifa->ifa_scope = RT_SCOPE_LINK;
4114         ifa->ifa_index = ctx->ifindex;
4115         if (family == AF_INET) {
4116                 ifa->ifa_family = AF_INET;
4117                 ifa->ifa_prefixlen = 32;
4118                 mnl_attr_put_u32(cmd, IFA_LOCAL, mnl_attr_get_u32(na_local));
4119                 mnl_attr_put_u32(cmd, IFA_ADDRESS, mnl_attr_get_u32(na_peer));
4120         } else {
4121                 ifa->ifa_family = AF_INET6;
4122                 ifa->ifa_prefixlen = 128;
4123                 mnl_attr_put(cmd, IFA_LOCAL, IPV6_ADDR_LEN,
4124                         mnl_attr_get_payload(na_local));
4125                 mnl_attr_put(cmd, IFA_ADDRESS, IPV6_ADDR_LEN,
4126                         mnl_attr_get_payload(na_peer));
4127         }
4128         assert(size == cmd->nlmsg_len);
4129         return 1;
4130 }
4131
4132 /**
4133  * Cleanup the local IP addresses on outer interface.
4134  *
4135  * @param[in] tcf
4136  *   Context object initialized by mlx5_flow_tcf_context_create().
4137  * @param[in] ifindex
4138  *   Network inferface index to perform cleanup.
4139  */
4140 static void
4141 flow_tcf_encap_local_cleanup(struct mlx5_flow_tcf_context *tcf,
4142                             unsigned int ifindex)
4143 {
4144         struct nlmsghdr *nlh;
4145         struct ifaddrmsg *ifa;
4146         struct tcf_nlcb_context ctx = {
4147                 .ifindex = ifindex,
4148                 .bufsize = MNL_REQUEST_SIZE,
4149                 .nlbuf = LIST_HEAD_INITIALIZER(),
4150         };
4151         int ret;
4152
4153         assert(ifindex);
4154         /*
4155          * Seek and destroy leftovers of local IP addresses with
4156          * matching properties "scope link".
4157          */
4158         nlh = mnl_nlmsg_put_header(tcf->buf);
4159         nlh->nlmsg_type = RTM_GETADDR;
4160         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4161         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4162         ifa->ifa_family = AF_UNSPEC;
4163         ifa->ifa_index = ifindex;
4164         ifa->ifa_scope = RT_SCOPE_LINK;
4165         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_local_cb, &ctx);
4166         if (ret)
4167                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4168         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4169         if (ret)
4170                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4171 }
4172
4173 /**
4174  * Collect neigh permament rules on specified network device.
4175  * This is callback routine called by libmnl mnl_cb_run() in loop for
4176  * every message in received packet.
4177  *
4178  * @param[in] nlh
4179  *   Pointer to reply header.
4180  * @param[in, out] arg
4181  *   Opaque data pointer for this callback.
4182  *
4183  * @return
4184  *   A positive, nonzero value on success, negative errno value otherwise
4185  *   and rte_errno is set.
4186  */
4187 static int
4188 flow_tcf_collect_neigh_cb(const struct nlmsghdr *nlh, void *arg)
4189 {
4190         struct tcf_nlcb_context *ctx = arg;
4191         struct nlmsghdr *cmd;
4192         struct ndmsg *ndm;
4193         struct nlattr *na;
4194         struct nlattr *na_ip = NULL;
4195         struct nlattr *na_mac = NULL;
4196         unsigned char family;
4197         uint32_t size;
4198
4199         if (nlh->nlmsg_type != RTM_NEWNEIGH) {
4200                 rte_errno = EINVAL;
4201                 return -rte_errno;
4202         }
4203         ndm = mnl_nlmsg_get_payload(nlh);
4204         family = ndm->ndm_family;
4205         if (ndm->ndm_ifindex != (int)ctx->ifindex ||
4206            !(ndm->ndm_state & NUD_PERMANENT) ||
4207            (family != AF_INET && family != AF_INET6))
4208                 return 1;
4209         mnl_attr_for_each(na, nlh, sizeof(*ndm)) {
4210                 switch (mnl_attr_get_type(na)) {
4211                 case NDA_DST:
4212                         na_ip = na;
4213                         break;
4214                 case NDA_LLADDR:
4215                         na_mac = na;
4216                         break;
4217                 }
4218                 if (na_mac && na_ip)
4219                         break;
4220         }
4221         if (!na_mac || !na_ip)
4222                 return 1;
4223         /* Neigh rule with permenent attribute found. */
4224         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4225                MNL_ALIGN(sizeof(struct ndmsg)) +
4226                SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) +
4227                (family == AF_INET6 ? SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
4228                                    : SZ_NLATTR_TYPE_OF(uint32_t));
4229         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4230         if (!cmd) {
4231                 rte_errno = ENOMEM;
4232                 return -rte_errno;
4233         }
4234         cmd = mnl_nlmsg_put_header(cmd);
4235         cmd->nlmsg_type = RTM_DELNEIGH;
4236         cmd->nlmsg_flags = NLM_F_REQUEST;
4237         ndm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ndm));
4238         ndm->ndm_ifindex = ctx->ifindex;
4239         ndm->ndm_state = NUD_PERMANENT;
4240         ndm->ndm_flags = 0;
4241         ndm->ndm_type = 0;
4242         if (family == AF_INET) {
4243                 ndm->ndm_family = AF_INET;
4244                 mnl_attr_put_u32(cmd, NDA_DST, mnl_attr_get_u32(na_ip));
4245         } else {
4246                 ndm->ndm_family = AF_INET6;
4247                 mnl_attr_put(cmd, NDA_DST, IPV6_ADDR_LEN,
4248                              mnl_attr_get_payload(na_ip));
4249         }
4250         mnl_attr_put(cmd, NDA_LLADDR, ETHER_ADDR_LEN,
4251                      mnl_attr_get_payload(na_mac));
4252         assert(size == cmd->nlmsg_len);
4253         return 1;
4254 }
4255
4256 /**
4257  * Cleanup the neigh rules on outer interface.
4258  *
4259  * @param[in] tcf
4260  *   Context object initialized by mlx5_flow_tcf_context_create().
4261  * @param[in] ifindex
4262  *   Network inferface index to perform cleanup.
4263  */
4264 static void
4265 flow_tcf_encap_neigh_cleanup(struct mlx5_flow_tcf_context *tcf,
4266                             unsigned int ifindex)
4267 {
4268         struct nlmsghdr *nlh;
4269         struct ndmsg *ndm;
4270         struct tcf_nlcb_context ctx = {
4271                 .ifindex = ifindex,
4272                 .bufsize = MNL_REQUEST_SIZE,
4273                 .nlbuf = LIST_HEAD_INITIALIZER(),
4274         };
4275         int ret;
4276
4277         assert(ifindex);
4278         /* Seek and destroy leftovers of neigh rules. */
4279         nlh = mnl_nlmsg_put_header(tcf->buf);
4280         nlh->nlmsg_type = RTM_GETNEIGH;
4281         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4282         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4283         ndm->ndm_family = AF_UNSPEC;
4284         ndm->ndm_ifindex = ifindex;
4285         ndm->ndm_state = NUD_PERMANENT;
4286         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_neigh_cb, &ctx);
4287         if (ret)
4288                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4289         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4290         if (ret)
4291                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4292 }
4293
4294 /**
4295  * Collect indices of VXLAN encap/decap interfaces associated with device.
4296  * This is callback routine called by libmnl mnl_cb_run() in loop for
4297  * every message in received packet.
4298  *
4299  * @param[in] nlh
4300  *   Pointer to reply header.
4301  * @param[in, out] arg
4302  *   Opaque data pointer for this callback.
4303  *
4304  * @return
4305  *   A positive, nonzero value on success, negative errno value otherwise
4306  *   and rte_errno is set.
4307  */
4308 static int
4309 flow_tcf_collect_vxlan_cb(const struct nlmsghdr *nlh, void *arg)
4310 {
4311         struct tcf_nlcb_context *ctx = arg;
4312         struct nlmsghdr *cmd;
4313         struct ifinfomsg *ifm;
4314         struct nlattr *na;
4315         struct nlattr *na_info = NULL;
4316         struct nlattr *na_vxlan = NULL;
4317         bool found = false;
4318         unsigned int vxindex;
4319         uint32_t size;
4320
4321         if (nlh->nlmsg_type != RTM_NEWLINK) {
4322                 rte_errno = EINVAL;
4323                 return -rte_errno;
4324         }
4325         ifm = mnl_nlmsg_get_payload(nlh);
4326         if (!ifm->ifi_index) {
4327                 rte_errno = EINVAL;
4328                 return -rte_errno;
4329         }
4330         mnl_attr_for_each(na, nlh, sizeof(*ifm))
4331                 if (mnl_attr_get_type(na) == IFLA_LINKINFO) {
4332                         na_info = na;
4333                         break;
4334                 }
4335         if (!na_info)
4336                 return 1;
4337         mnl_attr_for_each_nested(na, na_info) {
4338                 switch (mnl_attr_get_type(na)) {
4339                 case IFLA_INFO_KIND:
4340                         if (!strncmp("vxlan", mnl_attr_get_str(na),
4341                                      mnl_attr_get_len(na)))
4342                                 found = true;
4343                         break;
4344                 case IFLA_INFO_DATA:
4345                         na_vxlan = na;
4346                         break;
4347                 }
4348                 if (found && na_vxlan)
4349                         break;
4350         }
4351         if (!found || !na_vxlan)
4352                 return 1;
4353         found = false;
4354         mnl_attr_for_each_nested(na, na_vxlan) {
4355                 if (mnl_attr_get_type(na) == IFLA_VXLAN_LINK &&
4356                     mnl_attr_get_u32(na) == ctx->ifindex) {
4357                         found = true;
4358                         break;
4359                 }
4360         }
4361         if (!found)
4362                 return 1;
4363         /* Attached VXLAN device found, store the command to delete. */
4364         vxindex = ifm->ifi_index;
4365         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4366                MNL_ALIGN(sizeof(struct ifinfomsg));
4367         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4368         if (!cmd) {
4369                 rte_errno = ENOMEM;
4370                 return -rte_errno;
4371         }
4372         cmd = mnl_nlmsg_put_header(cmd);
4373         cmd->nlmsg_type = RTM_DELLINK;
4374         cmd->nlmsg_flags = NLM_F_REQUEST;
4375         ifm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifm));
4376         ifm->ifi_family = AF_UNSPEC;
4377         ifm->ifi_index = vxindex;
4378         assert(size == cmd->nlmsg_len);
4379         return 1;
4380 }
4381
4382 /**
4383  * Cleanup the outer interface. Removes all found vxlan devices
4384  * attached to specified index, flushes the neigh and local IP
4385  * database.
4386  *
4387  * @param[in] tcf
4388  *   Context object initialized by mlx5_flow_tcf_context_create().
4389  * @param[in] ifindex
4390  *   Network inferface index to perform cleanup.
4391  */
4392 static void
4393 flow_tcf_encap_iface_cleanup(struct mlx5_flow_tcf_context *tcf,
4394                             unsigned int ifindex)
4395 {
4396         struct nlmsghdr *nlh;
4397         struct ifinfomsg *ifm;
4398         struct tcf_nlcb_context ctx = {
4399                 .ifindex = ifindex,
4400                 .bufsize = MNL_REQUEST_SIZE,
4401                 .nlbuf = LIST_HEAD_INITIALIZER(),
4402         };
4403         int ret;
4404
4405         assert(ifindex);
4406         /*
4407          * Seek and destroy leftover VXLAN encap/decap interfaces with
4408          * matching properties.
4409          */
4410         nlh = mnl_nlmsg_put_header(tcf->buf);
4411         nlh->nlmsg_type = RTM_GETLINK;
4412         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4413         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4414         ifm->ifi_family = AF_UNSPEC;
4415         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_vxlan_cb, &ctx);
4416         if (ret)
4417                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4418         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4419         if (ret)
4420                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4421 }
4422
4423 /**
4424  * Emit Netlink message to add/remove local address to the outer device.
4425  * The address being added is visible within the link only (scope link).
4426  *
4427  * Note that an implicit route is maintained by the kernel due to the
4428  * presence of a peer address (IFA_ADDRESS).
4429  *
4430  * These rules are used for encapsultion only and allow to assign
4431  * the outer tunnel source IP address.
4432  *
4433  * @param[in] tcf
4434  *   Libmnl socket context object.
4435  * @param[in] encap
4436  *   Encapsulation properties (source address and its peer).
4437  * @param[in] ifindex
4438  *   Network interface to apply rule.
4439  * @param[in] enable
4440  *   Toggle between add and remove.
4441  * @param[out] error
4442  *   Perform verbose error reporting if not NULL.
4443  *
4444  * @return
4445  *   0 on success, a negative errno value otherwise and rte_errno is set.
4446  */
4447 static int
4448 flow_tcf_rule_local(struct mlx5_flow_tcf_context *tcf,
4449                     const struct flow_tcf_vxlan_encap *encap,
4450                     unsigned int ifindex,
4451                     bool enable,
4452                     struct rte_flow_error *error)
4453 {
4454         struct nlmsghdr *nlh;
4455         struct ifaddrmsg *ifa;
4456         alignas(struct nlmsghdr)
4457         uint8_t buf[mnl_nlmsg_size(sizeof(*ifa) + 128)];
4458
4459         nlh = mnl_nlmsg_put_header(buf);
4460         nlh->nlmsg_type = enable ? RTM_NEWADDR : RTM_DELADDR;
4461         nlh->nlmsg_flags =
4462                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4463         nlh->nlmsg_seq = 0;
4464         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4465         ifa->ifa_flags = IFA_F_PERMANENT;
4466         ifa->ifa_scope = RT_SCOPE_LINK;
4467         ifa->ifa_index = ifindex;
4468         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4469                 ifa->ifa_family = AF_INET;
4470                 ifa->ifa_prefixlen = 32;
4471                 mnl_attr_put_u32(nlh, IFA_LOCAL, encap->ipv4.src);
4472                 if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST)
4473                         mnl_attr_put_u32(nlh, IFA_ADDRESS,
4474                                               encap->ipv4.dst);
4475         } else {
4476                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4477                 ifa->ifa_family = AF_INET6;
4478                 ifa->ifa_prefixlen = 128;
4479                 mnl_attr_put(nlh, IFA_LOCAL,
4480                                   sizeof(encap->ipv6.src),
4481                                   &encap->ipv6.src);
4482                 if (encap->mask & FLOW_TCF_ENCAP_IPV6_DST)
4483                         mnl_attr_put(nlh, IFA_ADDRESS,
4484                                           sizeof(encap->ipv6.dst),
4485                                           &encap->ipv6.dst);
4486         }
4487         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4488                 return 0;
4489         return rte_flow_error_set(error, rte_errno,
4490                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4491                                   "netlink: cannot complete IFA request"
4492                                   " (ip addr add)");
4493 }
4494
4495 /**
4496  * Emit Netlink message to add/remove neighbor.
4497  *
4498  * @param[in] tcf
4499  *   Libmnl socket context object.
4500  * @param[in] encap
4501  *   Encapsulation properties (destination address).
4502  * @param[in] ifindex
4503  *   Network interface.
4504  * @param[in] enable
4505  *   Toggle between add and remove.
4506  * @param[out] error
4507  *   Perform verbose error reporting if not NULL.
4508  *
4509  * @return
4510  *   0 on success, a negative errno value otherwise and rte_errno is set.
4511  */
4512 static int
4513 flow_tcf_rule_neigh(struct mlx5_flow_tcf_context *tcf,
4514                      const struct flow_tcf_vxlan_encap *encap,
4515                      unsigned int ifindex,
4516                      bool enable,
4517                      struct rte_flow_error *error)
4518 {
4519         struct nlmsghdr *nlh;
4520         struct ndmsg *ndm;
4521         alignas(struct nlmsghdr)
4522         uint8_t buf[mnl_nlmsg_size(sizeof(*ndm) + 128)];
4523
4524         nlh = mnl_nlmsg_put_header(buf);
4525         nlh->nlmsg_type = enable ? RTM_NEWNEIGH : RTM_DELNEIGH;
4526         nlh->nlmsg_flags =
4527                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4528         nlh->nlmsg_seq = 0;
4529         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4530         ndm->ndm_ifindex = ifindex;
4531         ndm->ndm_state = NUD_PERMANENT;
4532         ndm->ndm_flags = 0;
4533         ndm->ndm_type = 0;
4534         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4535                 ndm->ndm_family = AF_INET;
4536                 mnl_attr_put_u32(nlh, NDA_DST, encap->ipv4.dst);
4537         } else {
4538                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4539                 ndm->ndm_family = AF_INET6;
4540                 mnl_attr_put(nlh, NDA_DST, sizeof(encap->ipv6.dst),
4541                                                  &encap->ipv6.dst);
4542         }
4543         if (encap->mask & FLOW_TCF_ENCAP_ETH_SRC && enable)
4544                 DRV_LOG(WARNING,
4545                         "outer ethernet source address cannot be "
4546                         "forced for VXLAN encapsulation");
4547         if (encap->mask & FLOW_TCF_ENCAP_ETH_DST)
4548                 mnl_attr_put(nlh, NDA_LLADDR, sizeof(encap->eth.dst),
4549                                                     &encap->eth.dst);
4550         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4551                 return 0;
4552         return rte_flow_error_set(error, rte_errno,
4553                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4554                                   "netlink: cannot complete ND request"
4555                                   " (ip neigh)");
4556 }
4557
4558 /**
4559  * Manage the local IP addresses and their peers IP addresses on the
4560  * outer interface for encapsulation purposes. The kernel searches the
4561  * appropriate device for tunnel egress traffic using the outer source
4562  * IP, this IP should be assigned to the outer network device, otherwise
4563  * kernel rejects the rule.
4564  *
4565  * Adds or removes the addresses using the Netlink command like this:
4566  *   ip addr add <src_ip> peer <dst_ip> scope link dev <ifouter>
4567  *
4568  * The addresses are local to the netdev ("scope link"), this reduces
4569  * the risk of conflicts. Note that an implicit route is maintained by
4570  * the kernel due to the presence of a peer address (IFA_ADDRESS).
4571  *
4572  * @param[in] tcf
4573  *   Libmnl socket context object.
4574  * @param[in] iface
4575  *   Object, contains rule database and ifouter index.
4576  * @param[in] dev_flow
4577  *   Flow object, contains the tunnel parameters (for encap only).
4578  * @param[in] enable
4579  *   Toggle between add and remove.
4580  * @param[out] error
4581  *   Perform verbose error reporting if not NULL.
4582  *
4583  * @return
4584  *   0 on success, a negative errno value otherwise and rte_errno is set.
4585  */
4586 static int
4587 flow_tcf_encap_local(struct mlx5_flow_tcf_context *tcf,
4588                      struct tcf_irule *iface,
4589                      struct mlx5_flow *dev_flow,
4590                      bool enable,
4591                      struct rte_flow_error *error)
4592 {
4593         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4594         struct tcf_local_rule *rule = NULL;
4595         int ret;
4596
4597         assert(encap);
4598         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4599         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4600                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_DST);
4601                 LIST_FOREACH(rule, &iface->local, next) {
4602                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_SRC &&
4603                             encap->ipv4.src == rule->ipv4.src &&
4604                             encap->ipv4.dst == rule->ipv4.dst) {
4605                                 break;
4606                         }
4607                 }
4608         } else {
4609                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4610                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4611                 LIST_FOREACH(rule, &iface->local, next) {
4612                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_SRC &&
4613                             !memcmp(&encap->ipv6.src, &rule->ipv6.src,
4614                                             sizeof(encap->ipv6.src)) &&
4615                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4616                                             sizeof(encap->ipv6.dst))) {
4617                                 break;
4618                         }
4619                 }
4620         }
4621         if (rule) {
4622                 if (enable) {
4623                         rule->refcnt++;
4624                         return 0;
4625                 }
4626                 if (!rule->refcnt || !--rule->refcnt) {
4627                         LIST_REMOVE(rule, next);
4628                         return flow_tcf_rule_local(tcf, encap,
4629                                         iface->ifouter, false, error);
4630                 }
4631                 return 0;
4632         }
4633         if (!enable) {
4634                 DRV_LOG(WARNING, "disabling not existing local rule");
4635                 rte_flow_error_set(error, ENOENT,
4636                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4637                                    "disabling not existing local rule");
4638                 return -ENOENT;
4639         }
4640         rule = rte_zmalloc(__func__, sizeof(struct tcf_local_rule),
4641                                 alignof(struct tcf_local_rule));
4642         if (!rule) {
4643                 rte_flow_error_set(error, ENOMEM,
4644                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4645                                    "unable to allocate memory for local rule");
4646                 return -rte_errno;
4647         }
4648         *rule = (struct tcf_local_rule){.refcnt = 0,
4649                                         .mask = 0,
4650                                         };
4651         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4652                 rule->mask = FLOW_TCF_ENCAP_IPV4_SRC
4653                            | FLOW_TCF_ENCAP_IPV4_DST;
4654                 rule->ipv4.src = encap->ipv4.src;
4655                 rule->ipv4.dst = encap->ipv4.dst;
4656         } else {
4657                 rule->mask = FLOW_TCF_ENCAP_IPV6_SRC
4658                            | FLOW_TCF_ENCAP_IPV6_DST;
4659                 memcpy(&rule->ipv6.src, &encap->ipv6.src, IPV6_ADDR_LEN);
4660                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4661         }
4662         ret = flow_tcf_rule_local(tcf, encap, iface->ifouter, true, error);
4663         if (ret) {
4664                 rte_free(rule);
4665                 return ret;
4666         }
4667         rule->refcnt++;
4668         LIST_INSERT_HEAD(&iface->local, rule, next);
4669         return 0;
4670 }
4671
4672 /**
4673  * Manage the destination MAC/IP addresses neigh database, kernel uses
4674  * this one to determine the destination MAC address within encapsulation
4675  * header. Adds or removes the entries using the Netlink command like this:
4676  *   ip neigh add dev <ifouter> lladdr <dst_mac> to <dst_ip> nud permanent
4677  *
4678  * @param[in] tcf
4679  *   Libmnl socket context object.
4680  * @param[in] iface
4681  *   Object, contains rule database and ifouter index.
4682  * @param[in] dev_flow
4683  *   Flow object, contains the tunnel parameters (for encap only).
4684  * @param[in] enable
4685  *   Toggle between add and remove.
4686  * @param[out] error
4687  *   Perform verbose error reporting if not NULL.
4688  *
4689  * @return
4690  *   0 on success, a negative errno value otherwise and rte_errno is set.
4691  */
4692 static int
4693 flow_tcf_encap_neigh(struct mlx5_flow_tcf_context *tcf,
4694                      struct tcf_irule *iface,
4695                      struct mlx5_flow *dev_flow,
4696                      bool enable,
4697                      struct rte_flow_error *error)
4698 {
4699         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4700         struct tcf_neigh_rule *rule = NULL;
4701         int ret;
4702
4703         assert(encap);
4704         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4705         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4706                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_SRC);
4707                 LIST_FOREACH(rule, &iface->neigh, next) {
4708                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_DST &&
4709                             encap->ipv4.dst == rule->ipv4.dst) {
4710                                 break;
4711                         }
4712                 }
4713         } else {
4714                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4715                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4716                 LIST_FOREACH(rule, &iface->neigh, next) {
4717                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_DST &&
4718                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4719                                                 sizeof(encap->ipv6.dst))) {
4720                                 break;
4721                         }
4722                 }
4723         }
4724         if (rule) {
4725                 if (memcmp(&encap->eth.dst, &rule->eth,
4726                            sizeof(encap->eth.dst))) {
4727                         DRV_LOG(WARNING, "Destination MAC differs"
4728                                          " in neigh rule");
4729                         rte_flow_error_set(error, EEXIST,
4730                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4731                                            NULL, "Different MAC address"
4732                                            " neigh rule for the same"
4733                                            " destination IP");
4734                                         return -EEXIST;
4735                 }
4736                 if (enable) {
4737                         rule->refcnt++;
4738                         return 0;
4739                 }
4740                 if (!rule->refcnt || !--rule->refcnt) {
4741                         LIST_REMOVE(rule, next);
4742                         return flow_tcf_rule_neigh(tcf, encap,
4743                                                    iface->ifouter,
4744                                                    false, error);
4745                 }
4746                 return 0;
4747         }
4748         if (!enable) {
4749                 DRV_LOG(WARNING, "Disabling not existing neigh rule");
4750                 rte_flow_error_set(error, ENOENT,
4751                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4752                                    "unable to allocate memory for neigh rule");
4753                 return -ENOENT;
4754         }
4755         rule = rte_zmalloc(__func__, sizeof(struct tcf_neigh_rule),
4756                                 alignof(struct tcf_neigh_rule));
4757         if (!rule) {
4758                 rte_flow_error_set(error, ENOMEM,
4759                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4760                                    "unable to allocate memory for neigh rule");
4761                 return -rte_errno;
4762         }
4763         *rule = (struct tcf_neigh_rule){.refcnt = 0,
4764                                         .mask = 0,
4765                                         };
4766         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4767                 rule->mask = FLOW_TCF_ENCAP_IPV4_DST;
4768                 rule->ipv4.dst = encap->ipv4.dst;
4769         } else {
4770                 rule->mask = FLOW_TCF_ENCAP_IPV6_DST;
4771                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4772         }
4773         memcpy(&rule->eth, &encap->eth.dst, sizeof(rule->eth));
4774         ret = flow_tcf_rule_neigh(tcf, encap, iface->ifouter, true, error);
4775         if (ret) {
4776                 rte_free(rule);
4777                 return ret;
4778         }
4779         rule->refcnt++;
4780         LIST_INSERT_HEAD(&iface->neigh, rule, next);
4781         return 0;
4782 }
4783
4784 /* VXLAN encap rule database for outer interfaces. */
4785 static  LIST_HEAD(, tcf_irule) iface_list_vxlan = LIST_HEAD_INITIALIZER();
4786
4787 /* VTEP device list is shared between PMD port instances. */
4788 static LIST_HEAD(, tcf_vtep) vtep_list_vxlan = LIST_HEAD_INITIALIZER();
4789 static pthread_mutex_t vtep_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4790
4791 /**
4792  * Acquire the VXLAN encap rules container for specified interface.
4793  * First looks for the container in the existing ones list, creates
4794  * and initializes the new container if existing not found.
4795  *
4796  * @param[in] tcf
4797  *   Context object initialized by mlx5_flow_tcf_context_create().
4798  * @param[in] ifouter
4799  *   Network interface index to create VXLAN encap rules on.
4800  * @param[out] error
4801  *   Perform verbose error reporting if not NULL.
4802  * @return
4803  *   Rule container pointer on success,
4804  *   NULL otherwise and rte_errno is set.
4805  */
4806 static struct tcf_irule*
4807 flow_tcf_encap_irule_acquire(struct mlx5_flow_tcf_context *tcf,
4808                              unsigned int ifouter,
4809                              struct rte_flow_error *error)
4810 {
4811         struct tcf_irule *iface;
4812
4813         /* Look whether the container for encap rules is created. */
4814         assert(ifouter);
4815         LIST_FOREACH(iface, &iface_list_vxlan, next) {
4816                 if (iface->ifouter == ifouter)
4817                         break;
4818         }
4819         if (iface) {
4820                 /* Container already exists, just increment the reference. */
4821                 iface->refcnt++;
4822                 return iface;
4823         }
4824         /* Not found, we should create the new container. */
4825         iface = rte_zmalloc(__func__, sizeof(*iface),
4826                             alignof(struct tcf_irule));
4827         if (!iface) {
4828                 rte_flow_error_set(error, ENOMEM,
4829                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4830                                    "unable to allocate memory for container");
4831                 return NULL;
4832         }
4833         *iface = (struct tcf_irule){
4834                         .local = LIST_HEAD_INITIALIZER(),
4835                         .neigh = LIST_HEAD_INITIALIZER(),
4836                         .ifouter = ifouter,
4837                         .refcnt = 1,
4838         };
4839         /* Interface cleanup for new container created. */
4840         flow_tcf_encap_iface_cleanup(tcf, ifouter);
4841         flow_tcf_encap_local_cleanup(tcf, ifouter);
4842         flow_tcf_encap_neigh_cleanup(tcf, ifouter);
4843         LIST_INSERT_HEAD(&iface_list_vxlan, iface, next);
4844         return iface;
4845 }
4846
4847 /**
4848  * Releases VXLAN encap rules container by pointer. Decrements the
4849  * reference cointer and deletes the container if counter is zero.
4850  *
4851  * @param[in] irule
4852  *   VXLAN rule container pointer to release.
4853  */
4854 static void
4855 flow_tcf_encap_irule_release(struct tcf_irule *iface)
4856 {
4857         assert(iface->refcnt);
4858         if (--iface->refcnt == 0) {
4859                 /* Reference counter is zero, delete the container. */
4860                 assert(LIST_EMPTY(&iface->local));
4861                 assert(LIST_EMPTY(&iface->neigh));
4862                 LIST_REMOVE(iface, next);
4863                 rte_free(iface);
4864         }
4865 }
4866
4867 /**
4868  * Deletes VTEP network device.
4869  *
4870  * @param[in] tcf
4871  *   Context object initialized by mlx5_flow_tcf_context_create().
4872  * @param[in] vtep
4873  *   Object represinting the network device to delete. Memory
4874  *   allocated for this object is freed by routine.
4875  */
4876 static void
4877 flow_tcf_vtep_delete(struct mlx5_flow_tcf_context *tcf,
4878                      struct tcf_vtep *vtep)
4879 {
4880         struct nlmsghdr *nlh;
4881         struct ifinfomsg *ifm;
4882         alignas(struct nlmsghdr)
4883         uint8_t buf[mnl_nlmsg_size(MNL_ALIGN(sizeof(*ifm))) +
4884                     MNL_BUF_EXTRA_SPACE];
4885         int ret;
4886
4887         assert(!vtep->refcnt);
4888         /* Delete only ifaces those we actually created. */
4889         if (vtep->created && vtep->ifindex) {
4890                 DRV_LOG(INFO, "VTEP delete (%d)", vtep->ifindex);
4891                 nlh = mnl_nlmsg_put_header(buf);
4892                 nlh->nlmsg_type = RTM_DELLINK;
4893                 nlh->nlmsg_flags = NLM_F_REQUEST;
4894                 ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4895                 ifm->ifi_family = AF_UNSPEC;
4896                 ifm->ifi_index = vtep->ifindex;
4897                 assert(sizeof(buf) >= nlh->nlmsg_len);
4898                 ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4899                 if (ret)
4900                         DRV_LOG(WARNING, "netlink: error deleting vxlan"
4901                                          " encap/decap ifindex %u",
4902                                          ifm->ifi_index);
4903         }
4904         rte_free(vtep);
4905 }
4906
4907 /**
4908  * Creates VTEP network device.
4909  *
4910  * @param[in] tcf
4911  *   Context object initialized by mlx5_flow_tcf_context_create().
4912  * @param[in] port
4913  *   UDP port of created VTEP device.
4914  * @param[out] error
4915  *   Perform verbose error reporting if not NULL.
4916  *
4917  * @return
4918  * Pointer to created device structure on success,
4919  * NULL otherwise and rte_errno is set.
4920  */
4921 #ifdef HAVE_IFLA_VXLAN_COLLECT_METADATA
4922 static struct tcf_vtep*
4923 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf,
4924                      uint16_t port, struct rte_flow_error *error)
4925 {
4926         struct tcf_vtep *vtep;
4927         struct nlmsghdr *nlh;
4928         struct ifinfomsg *ifm;
4929         char name[sizeof(MLX5_VXLAN_DEVICE_PFX) + 24];
4930         alignas(struct nlmsghdr)
4931         uint8_t buf[mnl_nlmsg_size(sizeof(*ifm)) +
4932                     SZ_NLATTR_DATA_OF(sizeof(name)) +
4933                     SZ_NLATTR_NEST * 2 +
4934                     SZ_NLATTR_STRZ_OF("vxlan") +
4935                     SZ_NLATTR_DATA_OF(sizeof(uint32_t)) +
4936                     SZ_NLATTR_DATA_OF(sizeof(uint16_t)) +
4937                     SZ_NLATTR_DATA_OF(sizeof(uint8_t)) * 3 +
4938                     MNL_BUF_EXTRA_SPACE];
4939         struct nlattr *na_info;
4940         struct nlattr *na_vxlan;
4941         rte_be16_t vxlan_port = rte_cpu_to_be_16(port);
4942         int ret;
4943
4944         vtep = rte_zmalloc(__func__, sizeof(*vtep), alignof(struct tcf_vtep));
4945         if (!vtep) {
4946                 rte_flow_error_set(error, ENOMEM,
4947                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4948                                    "unable to allocate memory for VTEP");
4949                 return NULL;
4950         }
4951         *vtep = (struct tcf_vtep){
4952                         .port = port,
4953         };
4954         memset(buf, 0, sizeof(buf));
4955         nlh = mnl_nlmsg_put_header(buf);
4956         nlh->nlmsg_type = RTM_NEWLINK;
4957         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE  | NLM_F_EXCL;
4958         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4959         ifm->ifi_family = AF_UNSPEC;
4960         ifm->ifi_type = 0;
4961         ifm->ifi_index = 0;
4962         ifm->ifi_flags = IFF_UP;
4963         ifm->ifi_change = 0xffffffff;
4964         snprintf(name, sizeof(name), "%s%u", MLX5_VXLAN_DEVICE_PFX, port);
4965         mnl_attr_put_strz(nlh, IFLA_IFNAME, name);
4966         na_info = mnl_attr_nest_start(nlh, IFLA_LINKINFO);
4967         assert(na_info);
4968         mnl_attr_put_strz(nlh, IFLA_INFO_KIND, "vxlan");
4969         na_vxlan = mnl_attr_nest_start(nlh, IFLA_INFO_DATA);
4970         assert(na_vxlan);
4971         mnl_attr_put_u8(nlh, IFLA_VXLAN_COLLECT_METADATA, 1);
4972         mnl_attr_put_u8(nlh, IFLA_VXLAN_UDP_ZERO_CSUM6_RX, 1);
4973         mnl_attr_put_u8(nlh, IFLA_VXLAN_LEARNING, 0);
4974         mnl_attr_put_u16(nlh, IFLA_VXLAN_PORT, vxlan_port);
4975         mnl_attr_nest_end(nlh, na_vxlan);
4976         mnl_attr_nest_end(nlh, na_info);
4977         assert(sizeof(buf) >= nlh->nlmsg_len);
4978         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4979         if (ret) {
4980                 DRV_LOG(WARNING,
4981                         "netlink: VTEP %s create failure (%d)",
4982                         name, rte_errno);
4983                 if (rte_errno != EEXIST)
4984                         /*
4985                          * Some unhandled error occurred or device is
4986                          * for encapsulation and cannot be shared.
4987                          */
4988                         goto error;
4989         } else {
4990                 /*
4991                  * Mark device we actually created.
4992                  * We should explicitly delete
4993                  * when we do not need it anymore.
4994                  */
4995                 vtep->created = 1;
4996         }
4997         /* Try to get ifindex of created of pre-existing device. */
4998         ret = if_nametoindex(name);
4999         if (!ret) {
5000                 DRV_LOG(WARNING,
5001                         "VTEP %s failed to get index (%d)", name, errno);
5002                 rte_flow_error_set
5003                         (error, -errno,
5004                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5005                          "netlink: failed to retrieve VTEP ifindex");
5006                 goto error;
5007         }
5008         vtep->ifindex = ret;
5009         memset(buf, 0, sizeof(buf));
5010         nlh = mnl_nlmsg_put_header(buf);
5011         nlh->nlmsg_type = RTM_NEWLINK;
5012         nlh->nlmsg_flags = NLM_F_REQUEST;
5013         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
5014         ifm->ifi_family = AF_UNSPEC;
5015         ifm->ifi_type = 0;
5016         ifm->ifi_index = vtep->ifindex;
5017         ifm->ifi_flags = IFF_UP;
5018         ifm->ifi_change = IFF_UP;
5019         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
5020         if (ret) {
5021                 rte_flow_error_set(error, -errno,
5022                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5023                                    "netlink: failed to set VTEP link up");
5024                 DRV_LOG(WARNING, "netlink: VTEP %s set link up failure (%d)",
5025                         name, rte_errno);
5026                 goto clean;
5027         }
5028         ret = mlx5_flow_tcf_init(tcf, vtep->ifindex, error);
5029         if (ret) {
5030                 DRV_LOG(WARNING, "VTEP %s init failure (%d)", name, rte_errno);
5031                 goto clean;
5032         }
5033         DRV_LOG(INFO, "VTEP create (%d, %d)", vtep->port, vtep->ifindex);
5034         vtep->refcnt = 1;
5035         return vtep;
5036 clean:
5037         flow_tcf_vtep_delete(tcf, vtep);
5038         return NULL;
5039 error:
5040         rte_free(vtep);
5041         return NULL;
5042 }
5043 #else
5044 static struct tcf_vtep*
5045 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf __rte_unused,
5046                      uint16_t port __rte_unused,
5047                      struct rte_flow_error *error)
5048 {
5049         rte_flow_error_set(error, ENOTSUP,
5050                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5051                            "netlink: failed to create VTEP, "
5052                            "vxlan metadata are not supported by kernel");
5053         return NULL;
5054 }
5055 #endif /* HAVE_IFLA_VXLAN_COLLECT_METADATA */
5056
5057 /**
5058  * Acquire target interface index for VXLAN tunneling decapsulation.
5059  * In order to share the UDP port within the other interfaces the
5060  * VXLAN device created as not attached to any interface (if created).
5061  *
5062  * @param[in] tcf
5063  *   Context object initialized by mlx5_flow_tcf_context_create().
5064  * @param[in] dev_flow
5065  *   Flow tcf object with tunnel structure pointer set.
5066  * @param[out] error
5067  *   Perform verbose error reporting if not NULL.
5068  * @return
5069  *   Interface descriptor pointer on success,
5070  *   NULL otherwise and rte_errno is set.
5071  */
5072 static struct tcf_vtep*
5073 flow_tcf_decap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
5074                             struct mlx5_flow *dev_flow,
5075                             struct rte_flow_error *error)
5076 {
5077         struct tcf_vtep *vtep;
5078         uint16_t port = dev_flow->tcf.vxlan_decap->udp_port;
5079
5080         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
5081                 if (vtep->port == port)
5082                         break;
5083         }
5084         if (vtep) {
5085                 /* Device exists, just increment the reference counter. */
5086                 vtep->refcnt++;
5087                 assert(vtep->ifindex);
5088                 return vtep;
5089         }
5090         /* No decapsulation device exists, try to create the new one. */
5091         vtep = flow_tcf_vtep_create(tcf, port, error);
5092         if (vtep)
5093                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
5094         return vtep;
5095 }
5096
5097 /**
5098  * Aqcuire target interface index for VXLAN tunneling encapsulation.
5099  *
5100  * @param[in] tcf
5101  *   Context object initialized by mlx5_flow_tcf_context_create().
5102  * @param[in] ifouter
5103  *   Network interface index to attach VXLAN encap device to.
5104  * @param[in] dev_flow
5105  *   Flow tcf object with tunnel structure pointer set.
5106  * @param[out] error
5107  *   Perform verbose error reporting if not NULL.
5108  * @return
5109  *   Interface descriptor pointer on success,
5110  *   NULL otherwise and rte_errno is set.
5111  */
5112 static struct tcf_vtep*
5113 flow_tcf_encap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
5114                             unsigned int ifouter,
5115                             struct mlx5_flow *dev_flow,
5116                             struct rte_flow_error *error)
5117 {
5118         static uint16_t port;
5119         struct tcf_vtep *vtep;
5120         struct tcf_irule *iface;
5121         int ret;
5122
5123         assert(ifouter);
5124         /* Look whether the VTEP for specified port is created. */
5125         port = rte_be_to_cpu_16(dev_flow->tcf.vxlan_encap->udp.dst);
5126         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
5127                 if (vtep->port == port)
5128                         break;
5129         }
5130         if (vtep) {
5131                 /* VTEP already exists, just increment the reference. */
5132                 vtep->refcnt++;
5133         } else {
5134                 /* Not found, we should create the new VTEP. */
5135                 vtep = flow_tcf_vtep_create(tcf, port, error);
5136                 if (!vtep)
5137                         return NULL;
5138                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
5139         }
5140         assert(vtep->ifindex);
5141         iface = flow_tcf_encap_irule_acquire(tcf, ifouter, error);
5142         if (!iface) {
5143                 if (--vtep->refcnt == 0)
5144                         flow_tcf_vtep_delete(tcf, vtep);
5145                 return NULL;
5146         }
5147         dev_flow->tcf.vxlan_encap->iface = iface;
5148         /* Create local ipaddr with peer to specify the outer IPs. */
5149         ret = flow_tcf_encap_local(tcf, iface, dev_flow, true, error);
5150         if (!ret) {
5151                 /* Create neigh rule to specify outer destination MAC. */
5152                 ret = flow_tcf_encap_neigh(tcf, iface, dev_flow, true, error);
5153                 if (ret)
5154                         flow_tcf_encap_local(tcf, iface,
5155                                              dev_flow, false, error);
5156         }
5157         if (ret) {
5158                 dev_flow->tcf.vxlan_encap->iface = NULL;
5159                 flow_tcf_encap_irule_release(iface);
5160                 if (--vtep->refcnt == 0)
5161                         flow_tcf_vtep_delete(tcf, vtep);
5162                 return NULL;
5163         }
5164         return vtep;
5165 }
5166
5167 /**
5168  * Acquires target interface index for tunneling of any type.
5169  * Creates the new VTEP if needed.
5170  *
5171  * @param[in] tcf
5172  *   Context object initialized by mlx5_flow_tcf_context_create().
5173  * @param[in] ifouter
5174  *   Network interface index to create VXLAN encap rules on.
5175  * @param[in] dev_flow
5176  *   Flow tcf object with tunnel structure pointer set.
5177  * @param[out] error
5178  *   Perform verbose error reporting if not NULL.
5179  * @return
5180  *   Interface descriptor pointer on success,
5181  *   NULL otherwise and rte_errno is set.
5182  */
5183 static struct tcf_vtep*
5184 flow_tcf_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
5185                       unsigned int ifouter,
5186                       struct mlx5_flow *dev_flow,
5187                       struct rte_flow_error *error)
5188 {
5189         struct tcf_vtep *vtep = NULL;
5190
5191         assert(dev_flow->tcf.tunnel);
5192         pthread_mutex_lock(&vtep_list_mutex);
5193         switch (dev_flow->tcf.tunnel->type) {
5194         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5195                 vtep = flow_tcf_encap_vtep_acquire(tcf, ifouter,
5196                                                   dev_flow, error);
5197                 break;
5198         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5199                 vtep = flow_tcf_decap_vtep_acquire(tcf, dev_flow, error);
5200                 break;
5201         default:
5202                 rte_flow_error_set(error, ENOTSUP,
5203                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5204                                    "unsupported tunnel type");
5205                 break;
5206         }
5207         pthread_mutex_unlock(&vtep_list_mutex);
5208         return vtep;
5209 }
5210
5211 /**
5212  * Release tunneling interface by ifindex. Decrements reference
5213  * counter and actually removes the device if counter is zero.
5214  *
5215  * @param[in] tcf
5216  *   Context object initialized by mlx5_flow_tcf_context_create().
5217  * @param[in] vtep
5218  *   VTEP device descriptor structure.
5219  * @param[in] dev_flow
5220  *   Flow tcf object with tunnel structure pointer set.
5221  */
5222 static void
5223 flow_tcf_vtep_release(struct mlx5_flow_tcf_context *tcf,
5224                       struct tcf_vtep *vtep,
5225                       struct mlx5_flow *dev_flow)
5226 {
5227         assert(dev_flow->tcf.tunnel);
5228         pthread_mutex_lock(&vtep_list_mutex);
5229         switch (dev_flow->tcf.tunnel->type) {
5230         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5231                 break;
5232         case FLOW_TCF_TUNACT_VXLAN_ENCAP: {
5233                 struct tcf_irule *iface;
5234
5235                 /* Remove the encap ancillary rules first. */
5236                 iface = dev_flow->tcf.vxlan_encap->iface;
5237                 assert(iface);
5238                 flow_tcf_encap_neigh(tcf, iface, dev_flow, false, NULL);
5239                 flow_tcf_encap_local(tcf, iface, dev_flow, false, NULL);
5240                 flow_tcf_encap_irule_release(iface);
5241                 dev_flow->tcf.vxlan_encap->iface = NULL;
5242                 break;
5243         }
5244         default:
5245                 assert(false);
5246                 DRV_LOG(WARNING, "Unsupported tunnel type");
5247                 break;
5248         }
5249         assert(vtep->refcnt);
5250         if (--vtep->refcnt == 0) {
5251                 LIST_REMOVE(vtep, next);
5252                 flow_tcf_vtep_delete(tcf, vtep);
5253         }
5254         pthread_mutex_unlock(&vtep_list_mutex);
5255 }
5256
5257 struct tcf_nlcb_query {
5258         uint32_t handle;
5259         uint32_t tc_flags;
5260         uint32_t flags_valid:1;
5261 };
5262
5263 /**
5264  * Collect queried rule attributes. This is callback routine called by
5265  * libmnl mnl_cb_run() in loop for every message in received packet.
5266  * Current implementation collects the flower flags only.
5267  *
5268  * @param[in] nlh
5269  *   Pointer to reply header.
5270  * @param[in, out] arg
5271  *   Context pointer for this callback.
5272  *
5273  * @return
5274  *   A positive, nonzero value on success (required by libmnl
5275  *   to continue messages processing).
5276  */
5277 static int
5278 flow_tcf_collect_query_cb(const struct nlmsghdr *nlh, void *arg)
5279 {
5280         struct tcf_nlcb_query *query = arg;
5281         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
5282         struct nlattr *na, *na_opt;
5283         bool flower = false;
5284
5285         if (nlh->nlmsg_type != RTM_NEWTFILTER ||
5286             tcm->tcm_handle != query->handle)
5287                 return 1;
5288         mnl_attr_for_each(na, nlh, sizeof(*tcm)) {
5289                 switch (mnl_attr_get_type(na)) {
5290                 case TCA_KIND:
5291                         if (strcmp(mnl_attr_get_payload(na), "flower")) {
5292                                 /* Not flower filter, drop entire message. */
5293                                 return 1;
5294                         }
5295                         flower = true;
5296                         break;
5297                 case TCA_OPTIONS:
5298                         if (!flower) {
5299                                 /* Not flower options, drop entire message. */
5300                                 return 1;
5301                         }
5302                         /* Check nested flower options. */
5303                         mnl_attr_for_each_nested(na_opt, na) {
5304                                 switch (mnl_attr_get_type(na_opt)) {
5305                                 case TCA_FLOWER_FLAGS:
5306                                         query->flags_valid = 1;
5307                                         query->tc_flags =
5308                                                 mnl_attr_get_u32(na_opt);
5309                                         break;
5310                                 }
5311                         }
5312                         break;
5313                 }
5314         }
5315         return 1;
5316 }
5317
5318 /**
5319  * Query a TC flower rule flags via netlink.
5320  *
5321  * @param[in] tcf
5322  *   Context object initialized by mlx5_flow_tcf_context_create().
5323  * @param[in] dev_flow
5324  *   Pointer to the flow.
5325  * @param[out] pflags
5326  *   pointer to the data retrieved by the query.
5327  *
5328  * @return
5329  *   0 on success, a negative errno value otherwise.
5330  */
5331 static int
5332 flow_tcf_query_flags(struct mlx5_flow_tcf_context *tcf,
5333                      struct mlx5_flow *dev_flow,
5334                      uint32_t *pflags)
5335 {
5336         struct nlmsghdr *nlh;
5337         struct tcmsg *tcm;
5338         struct tcf_nlcb_query query = {
5339                 .handle = dev_flow->tcf.tcm->tcm_handle,
5340         };
5341
5342         nlh = mnl_nlmsg_put_header(tcf->buf);
5343         nlh->nlmsg_type = RTM_GETTFILTER;
5344         nlh->nlmsg_flags = NLM_F_REQUEST;
5345         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5346         memcpy(tcm, dev_flow->tcf.tcm, sizeof(*tcm));
5347         /*
5348          * Ignore Netlink error for filter query operations.
5349          * The reply length is sent by kernel as errno.
5350          * Just check we got the flags option.
5351          */
5352         flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_query_cb, &query);
5353         if (!query.flags_valid) {
5354                 *pflags = 0;
5355                 return -ENOENT;
5356         }
5357         *pflags = query.tc_flags;
5358         return 0;
5359 }
5360
5361 /**
5362  * Query and check the in_hw set for specified rule.
5363  *
5364  * @param[in] tcf
5365  *   Context object initialized by mlx5_flow_tcf_context_create().
5366  * @param[in] dev_flow
5367  *   Pointer to the flow to check.
5368  *
5369  * @return
5370  *   0 on success, a negative errno value otherwise.
5371  */
5372 static int
5373 flow_tcf_check_inhw(struct mlx5_flow_tcf_context *tcf,
5374                     struct mlx5_flow *dev_flow)
5375 {
5376         uint32_t flags;
5377         int ret;
5378
5379         ret = flow_tcf_query_flags(tcf, dev_flow, &flags);
5380         if (ret)
5381                 return ret;
5382         return  (flags & TCA_CLS_FLAGS_IN_HW) ? 0 : -ENOENT;
5383 }
5384
5385 /**
5386  * Remove flow from E-Switch by sending Netlink message.
5387  *
5388  * @param[in] dev
5389  *   Pointer to Ethernet device.
5390  * @param[in, out] flow
5391  *   Pointer to the sub flow.
5392  */
5393 static void
5394 flow_tcf_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5395 {
5396         struct priv *priv = dev->data->dev_private;
5397         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5398         struct mlx5_flow *dev_flow;
5399         struct nlmsghdr *nlh;
5400
5401         if (!flow)
5402                 return;
5403         dev_flow = LIST_FIRST(&flow->dev_flows);
5404         if (!dev_flow)
5405                 return;
5406         /* E-Switch flow can't be expanded. */
5407         assert(!LIST_NEXT(dev_flow, next));
5408         if (dev_flow->tcf.applied) {
5409                 nlh = dev_flow->tcf.nlh;
5410                 nlh->nlmsg_type = RTM_DELTFILTER;
5411                 nlh->nlmsg_flags = NLM_F_REQUEST;
5412                 flow_tcf_nl_ack(ctx, nlh, NULL, NULL);
5413                 if (dev_flow->tcf.tunnel) {
5414                         assert(dev_flow->tcf.tunnel->vtep);
5415                         flow_tcf_vtep_release(ctx,
5416                                 dev_flow->tcf.tunnel->vtep,
5417                                 dev_flow);
5418                         dev_flow->tcf.tunnel->vtep = NULL;
5419                 }
5420                 dev_flow->tcf.applied = 0;
5421         }
5422 }
5423
5424 /**
5425  * Apply flow to E-Switch by sending Netlink message.
5426  *
5427  * @param[in] dev
5428  *   Pointer to Ethernet device.
5429  * @param[in, out] flow
5430  *   Pointer to the sub flow.
5431  * @param[out] error
5432  *   Pointer to the error structure.
5433  *
5434  * @return
5435  *   0 on success, a negative errno value otherwise and rte_errno is set.
5436  */
5437 static int
5438 flow_tcf_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5439                struct rte_flow_error *error)
5440 {
5441         struct priv *priv = dev->data->dev_private;
5442         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5443         struct mlx5_flow *dev_flow;
5444         struct nlmsghdr *nlh;
5445
5446         dev_flow = LIST_FIRST(&flow->dev_flows);
5447         /* E-Switch flow can't be expanded. */
5448         assert(!LIST_NEXT(dev_flow, next));
5449         if (dev_flow->tcf.applied)
5450                 return 0;
5451         nlh = dev_flow->tcf.nlh;
5452         nlh->nlmsg_type = RTM_NEWTFILTER;
5453         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5454         if (dev_flow->tcf.tunnel) {
5455                 /*
5456                  * Replace the interface index, target for
5457                  * encapsulation, source for decapsulation.
5458                  */
5459                 assert(!dev_flow->tcf.tunnel->vtep);
5460                 assert(dev_flow->tcf.tunnel->ifindex_ptr);
5461                 /* Acquire actual VTEP device when rule is being applied. */
5462                 dev_flow->tcf.tunnel->vtep =
5463                         flow_tcf_vtep_acquire(ctx,
5464                                         dev_flow->tcf.tunnel->ifindex_org,
5465                                         dev_flow, error);
5466                 if (!dev_flow->tcf.tunnel->vtep)
5467                         return -rte_errno;
5468                 DRV_LOG(INFO, "Replace ifindex: %d->%d",
5469                                 dev_flow->tcf.tunnel->vtep->ifindex,
5470                                 dev_flow->tcf.tunnel->ifindex_org);
5471                 *dev_flow->tcf.tunnel->ifindex_ptr =
5472                         dev_flow->tcf.tunnel->vtep->ifindex;
5473         }
5474         if (!flow_tcf_nl_ack(ctx, nlh, NULL, NULL)) {
5475                 dev_flow->tcf.applied = 1;
5476                 if (*dev_flow->tcf.ptc_flags & TCA_CLS_FLAGS_SKIP_SW)
5477                         return 0;
5478                 /*
5479                  * Rule was applied without skip_sw flag set.
5480                  * We should check whether the rule was acctually
5481                  * accepted by hardware (have look at in_hw flag).
5482                  */
5483                 if (flow_tcf_check_inhw(ctx, dev_flow)) {
5484                         flow_tcf_remove(dev, flow);
5485                         return rte_flow_error_set
5486                                 (error, ENOENT,
5487                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5488                                  "netlink: rule has no in_hw flag set");
5489                 }
5490                 return 0;
5491         }
5492         if (dev_flow->tcf.tunnel) {
5493                 /* Rollback the VTEP configuration if rule apply failed. */
5494                 assert(dev_flow->tcf.tunnel->vtep);
5495                 flow_tcf_vtep_release(ctx, dev_flow->tcf.tunnel->vtep,
5496                                       dev_flow);
5497                 dev_flow->tcf.tunnel->vtep = NULL;
5498         }
5499         return rte_flow_error_set(error, rte_errno,
5500                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5501                                   "netlink: failed to create TC flow rule");
5502 }
5503
5504 /**
5505  * Remove flow from E-Switch and release resources of the device flow.
5506  *
5507  * @param[in] dev
5508  *   Pointer to Ethernet device.
5509  * @param[in, out] flow
5510  *   Pointer to the sub flow.
5511  */
5512 static void
5513 flow_tcf_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5514 {
5515         struct mlx5_flow *dev_flow;
5516
5517         if (!flow)
5518                 return;
5519         flow_tcf_remove(dev, flow);
5520         if (flow->counter) {
5521                 if (--flow->counter->ref_cnt == 0) {
5522                         rte_free(flow->counter);
5523                         flow->counter = NULL;
5524                 }
5525         }
5526         dev_flow = LIST_FIRST(&flow->dev_flows);
5527         if (!dev_flow)
5528                 return;
5529         /* E-Switch flow can't be expanded. */
5530         assert(!LIST_NEXT(dev_flow, next));
5531         LIST_REMOVE(dev_flow, next);
5532         rte_free(dev_flow);
5533 }
5534
5535 /**
5536  * Helper routine for figuring the space size required for a parse buffer.
5537  *
5538  * @param array
5539  *   array of values to use.
5540  * @param idx
5541  *   Current location in array.
5542  * @param value
5543  *   Value to compare with.
5544  *
5545  * @return
5546  *   The maximum between the given value and the array value on index.
5547  */
5548 static uint16_t
5549 flow_tcf_arr_val_max(uint16_t array[], int idx, uint16_t value)
5550 {
5551         return idx < 0 ? (value) : RTE_MAX((array)[idx], value);
5552 }
5553
5554 /**
5555  * Parse rtnetlink message attributes filling the attribute table with the info
5556  * retrieved.
5557  *
5558  * @param tb
5559  *   Attribute table to be filled.
5560  * @param[out] max
5561  *   Maxinum entry in the attribute table.
5562  * @param rte
5563  *   The attributes section in the message to be parsed.
5564  * @param len
5565  *   The length of the attributes section in the message.
5566  */
5567 static void
5568 flow_tcf_nl_parse_rtattr(struct rtattr *tb[], int max,
5569                          struct rtattr *rta, int len)
5570 {
5571         unsigned short type;
5572         memset(tb, 0, sizeof(struct rtattr *) * (max + 1));
5573         while (RTA_OK(rta, len)) {
5574                 type = rta->rta_type;
5575                 if (type <= max && !tb[type])
5576                         tb[type] = rta;
5577                 rta = RTA_NEXT(rta, len);
5578         }
5579 }
5580
5581 /**
5582  * Extract flow counters from flower action.
5583  *
5584  * @param rta
5585  *   flower action stats properties in the Netlink message received.
5586  * @param rta_type
5587  *   The backward sequence of rta_types, as written in the attribute table,
5588  *   we need to traverse in order to get to the requested object.
5589  * @param idx
5590  *   Current location in rta_type table.
5591  * @param[out] data
5592  *   data holding the count statistics of the rte_flow retrieved from
5593  *   the message.
5594  *
5595  * @return
5596  *   0 if data was found and retrieved, -1 otherwise.
5597  */
5598 static int
5599 flow_tcf_nl_action_stats_parse_and_get(struct rtattr *rta,
5600                                        uint16_t rta_type[], int idx,
5601                                        struct gnet_stats_basic *data)
5602 {
5603         int tca_stats_max = flow_tcf_arr_val_max(rta_type, idx,
5604                                                  TCA_STATS_BASIC);
5605         struct rtattr *tbs[tca_stats_max + 1];
5606
5607         if (rta == NULL || idx < 0)
5608                 return -1;
5609         flow_tcf_nl_parse_rtattr(tbs, tca_stats_max,
5610                                  RTA_DATA(rta), RTA_PAYLOAD(rta));
5611         switch (rta_type[idx]) {
5612         case TCA_STATS_BASIC:
5613                 if (tbs[TCA_STATS_BASIC]) {
5614                         memcpy(data, RTA_DATA(tbs[TCA_STATS_BASIC]),
5615                                RTE_MIN(RTA_PAYLOAD(tbs[TCA_STATS_BASIC]),
5616                                sizeof(*data)));
5617                         return 0;
5618                 }
5619                 break;
5620         default:
5621                 break;
5622         }
5623         return -1;
5624 }
5625
5626 /**
5627  * Parse flower single action retrieving the requested action attribute,
5628  * if found.
5629  *
5630  * @param arg
5631  *   flower action properties in the Netlink message received.
5632  * @param rta_type
5633  *   The backward sequence of rta_types, as written in the attribute table,
5634  *   we need to traverse in order to get to the requested object.
5635  * @param idx
5636  *   Current location in rta_type table.
5637  * @param[out] data
5638  *   Count statistics retrieved from the message query.
5639  *
5640  * @return
5641  *   0 if data was found and retrieved, -1 otherwise.
5642  */
5643 static int
5644 flow_tcf_nl_parse_one_action_and_get(struct rtattr *arg,
5645                                      uint16_t rta_type[], int idx, void *data)
5646 {
5647         int tca_act_max = flow_tcf_arr_val_max(rta_type, idx, TCA_ACT_STATS);
5648         struct rtattr *tb[tca_act_max + 1];
5649
5650         if (arg == NULL || idx < 0)
5651                 return -1;
5652         flow_tcf_nl_parse_rtattr(tb, tca_act_max,
5653                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5654         if (tb[TCA_ACT_KIND] == NULL)
5655                 return -1;
5656         switch (rta_type[idx]) {
5657         case TCA_ACT_STATS:
5658                 if (tb[TCA_ACT_STATS])
5659                         return flow_tcf_nl_action_stats_parse_and_get
5660                                         (tb[TCA_ACT_STATS],
5661                                          rta_type, --idx,
5662                                          (struct gnet_stats_basic *)data);
5663                 break;
5664         default:
5665                 break;
5666         }
5667         return -1;
5668 }
5669
5670 /**
5671  * Parse flower action section in the message retrieving the requested
5672  * attribute from the first action that provides it.
5673  *
5674  * @param opt
5675  *   flower section in the Netlink message received.
5676  * @param rta_type
5677  *   The backward sequence of rta_types, as written in the attribute table,
5678  *   we need to traverse in order to get to the requested object.
5679  * @param idx
5680  *   Current location in rta_type table.
5681  * @param[out] data
5682  *   data retrieved from the message query.
5683  *
5684  * @return
5685  *   0 if data was found and retrieved, -1 otherwise.
5686  */
5687 static int
5688 flow_tcf_nl_action_parse_and_get(struct rtattr *arg,
5689                                  uint16_t rta_type[], int idx, void *data)
5690 {
5691         struct rtattr *tb[TCA_ACT_MAX_PRIO + 1];
5692         int i;
5693
5694         if (arg == NULL || idx < 0)
5695                 return -1;
5696         flow_tcf_nl_parse_rtattr(tb, TCA_ACT_MAX_PRIO,
5697                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5698         switch (rta_type[idx]) {
5699         /*
5700          * flow counters are stored in the actions defined by the flow
5701          * and not in the flow itself, therefore we need to traverse the
5702          * flower chain of actions in search for them.
5703          *
5704          * Note that the index is not decremented here.
5705          */
5706         case TCA_ACT_STATS:
5707                 for (i = 0; i <= TCA_ACT_MAX_PRIO; i++) {
5708                         if (tb[i] &&
5709                         !flow_tcf_nl_parse_one_action_and_get(tb[i],
5710                                                               rta_type,
5711                                                               idx, data))
5712                                 return 0;
5713                 }
5714                 break;
5715         default:
5716                 break;
5717         }
5718         return -1;
5719 }
5720
5721 /**
5722  * Parse flower classifier options in the message, retrieving the requested
5723  * attribute if found.
5724  *
5725  * @param opt
5726  *   flower section in the Netlink message received.
5727  * @param rta_type
5728  *   The backward sequence of rta_types, as written in the attribute table,
5729  *   we need to traverse in order to get to the requested object.
5730  * @param idx
5731  *   Current location in rta_type table.
5732  * @param[out] data
5733  *   data retrieved from the message query.
5734  *
5735  * @return
5736  *   0 if data was found and retrieved, -1 otherwise.
5737  */
5738 static int
5739 flow_tcf_nl_opts_parse_and_get(struct rtattr *opt,
5740                                uint16_t rta_type[], int idx, void *data)
5741 {
5742         int tca_flower_max = flow_tcf_arr_val_max(rta_type, idx,
5743                                                   TCA_FLOWER_ACT);
5744         struct rtattr *tb[tca_flower_max + 1];
5745
5746         if (!opt || idx < 0)
5747                 return -1;
5748         flow_tcf_nl_parse_rtattr(tb, tca_flower_max,
5749                                  RTA_DATA(opt), RTA_PAYLOAD(opt));
5750         switch (rta_type[idx]) {
5751         case TCA_FLOWER_ACT:
5752                 if (tb[TCA_FLOWER_ACT])
5753                         return flow_tcf_nl_action_parse_and_get
5754                                                         (tb[TCA_FLOWER_ACT],
5755                                                          rta_type, --idx, data);
5756                 break;
5757         default:
5758                 break;
5759         }
5760         return -1;
5761 }
5762
5763 /**
5764  * Parse Netlink reply on filter query, retrieving the flow counters.
5765  *
5766  * @param nlh
5767  *   Message received from Netlink.
5768  * @param rta_type
5769  *   The backward sequence of rta_types, as written in the attribute table,
5770  *   we need to traverse in order to get to the requested object.
5771  * @param idx
5772  *   Current location in rta_type table.
5773  * @param[out] data
5774  *   data retrieved from the message query.
5775  *
5776  * @return
5777  *   0 if data was found and retrieved, -1 otherwise.
5778  */
5779 static int
5780 flow_tcf_nl_filter_parse_and_get(struct nlmsghdr *cnlh,
5781                                  uint16_t rta_type[], int idx, void *data)
5782 {
5783         struct nlmsghdr *nlh = cnlh;
5784         struct tcmsg *t = NLMSG_DATA(nlh);
5785         int len = nlh->nlmsg_len;
5786         int tca_max = flow_tcf_arr_val_max(rta_type, idx, TCA_OPTIONS);
5787         struct rtattr *tb[tca_max + 1];
5788
5789         if (idx < 0)
5790                 return -1;
5791         if (nlh->nlmsg_type != RTM_NEWTFILTER &&
5792             nlh->nlmsg_type != RTM_GETTFILTER &&
5793             nlh->nlmsg_type != RTM_DELTFILTER)
5794                 return -1;
5795         len -= NLMSG_LENGTH(sizeof(*t));
5796         if (len < 0)
5797                 return -1;
5798         flow_tcf_nl_parse_rtattr(tb, tca_max, TCA_RTA(t), len);
5799         /* Not a TC flower flow - bail out */
5800         if (!tb[TCA_KIND] ||
5801             strcmp(RTA_DATA(tb[TCA_KIND]), "flower"))
5802                 return -1;
5803         switch (rta_type[idx]) {
5804         case TCA_OPTIONS:
5805                 if (tb[TCA_OPTIONS])
5806                         return flow_tcf_nl_opts_parse_and_get(tb[TCA_OPTIONS],
5807                                                               rta_type,
5808                                                               --idx, data);
5809                 break;
5810         default:
5811                 break;
5812         }
5813         return -1;
5814 }
5815
5816 /**
5817  * A callback to parse Netlink reply on TC flower query.
5818  *
5819  * @param nlh
5820  *   Message received from Netlink.
5821  * @param[out] data
5822  *   Pointer to data area to be filled by the parsing routine.
5823  *   assumed to be a pointer to struct flow_tcf_stats_basic.
5824  *
5825  * @return
5826  *   MNL_CB_OK value.
5827  */
5828 static int
5829 flow_tcf_nl_message_get_stats_basic(const struct nlmsghdr *nlh, void *data)
5830 {
5831         /*
5832          * The backward sequence of rta_types to pass in order to get
5833          *  to the counters.
5834          */
5835         uint16_t rta_type[] = { TCA_STATS_BASIC, TCA_ACT_STATS,
5836                                 TCA_FLOWER_ACT, TCA_OPTIONS };
5837         struct flow_tcf_stats_basic *sb_data = data;
5838         union {
5839                 const struct nlmsghdr *c;
5840                 struct nlmsghdr *nc;
5841         } tnlh = { .c = nlh };
5842
5843         if (!flow_tcf_nl_filter_parse_and_get(tnlh.nc, rta_type,
5844                                               RTE_DIM(rta_type) - 1,
5845                                               (void *)&sb_data->counters))
5846                 sb_data->valid = true;
5847         return MNL_CB_OK;
5848 }
5849
5850 /**
5851  * Query a TC flower rule for its statistics via netlink.
5852  *
5853  * @param[in] dev
5854  *   Pointer to Ethernet device.
5855  * @param[in] flow
5856  *   Pointer to the sub flow.
5857  * @param[out] data
5858  *   data retrieved by the query.
5859  * @param[out] error
5860  *   Perform verbose error reporting if not NULL.
5861  *
5862  * @return
5863  *   0 on success, a negative errno value otherwise and rte_errno is set.
5864  */
5865 static int
5866 flow_tcf_query_count(struct rte_eth_dev *dev,
5867                           struct rte_flow *flow,
5868                           void *data,
5869                           struct rte_flow_error *error)
5870 {
5871         struct flow_tcf_stats_basic sb_data;
5872         struct rte_flow_query_count *qc = data;
5873         struct priv *priv = dev->data->dev_private;
5874         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5875         struct mnl_socket *nl = ctx->nl;
5876         struct mlx5_flow *dev_flow;
5877         struct nlmsghdr *nlh;
5878         uint32_t seq = priv->tcf_context->seq++;
5879         ssize_t ret;
5880         assert(qc);
5881
5882         memset(&sb_data, 0, sizeof(sb_data));
5883         dev_flow = LIST_FIRST(&flow->dev_flows);
5884         /* E-Switch flow can't be expanded. */
5885         assert(!LIST_NEXT(dev_flow, next));
5886         if (!dev_flow->flow->counter)
5887                 goto notsup_exit;
5888         nlh = dev_flow->tcf.nlh;
5889         nlh->nlmsg_type = RTM_GETTFILTER;
5890         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_ECHO;
5891         nlh->nlmsg_seq = seq;
5892         if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) == -1)
5893                 goto error_exit;
5894         do {
5895                 ret = mnl_socket_recvfrom(nl, ctx->buf, ctx->buf_size);
5896                 if (ret <= 0)
5897                         break;
5898                 ret = mnl_cb_run(ctx->buf, ret, seq,
5899                                  mnl_socket_get_portid(nl),
5900                                  flow_tcf_nl_message_get_stats_basic,
5901                                  (void *)&sb_data);
5902         } while (ret > 0);
5903         /* Return the delta from last reset. */
5904         if (sb_data.valid) {
5905                 /* Return the delta from last reset. */
5906                 qc->hits_set = 1;
5907                 qc->bytes_set = 1;
5908                 qc->hits = sb_data.counters.packets - flow->counter->hits;
5909                 qc->bytes = sb_data.counters.bytes - flow->counter->bytes;
5910                 if (qc->reset) {
5911                         flow->counter->hits = sb_data.counters.packets;
5912                         flow->counter->bytes = sb_data.counters.bytes;
5913                 }
5914                 return 0;
5915         }
5916         return rte_flow_error_set(error, EINVAL,
5917                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5918                                   NULL,
5919                                   "flow does not have counter");
5920 error_exit:
5921         return rte_flow_error_set
5922                         (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5923                          NULL, "netlink: failed to read flow rule counters");
5924 notsup_exit:
5925         return rte_flow_error_set
5926                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5927                          NULL, "counters are not available.");
5928 }
5929
5930 /**
5931  * Query a flow.
5932  *
5933  * @see rte_flow_query()
5934  * @see rte_flow_ops
5935  */
5936 static int
5937 flow_tcf_query(struct rte_eth_dev *dev,
5938                struct rte_flow *flow,
5939                const struct rte_flow_action *actions,
5940                void *data,
5941                struct rte_flow_error *error)
5942 {
5943         int ret = -EINVAL;
5944
5945         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5946                 switch (actions->type) {
5947                 case RTE_FLOW_ACTION_TYPE_VOID:
5948                         break;
5949                 case RTE_FLOW_ACTION_TYPE_COUNT:
5950                         ret = flow_tcf_query_count(dev, flow, data, error);
5951                         break;
5952                 default:
5953                         return rte_flow_error_set(error, ENOTSUP,
5954                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5955                                                   actions,
5956                                                   "action not supported");
5957                 }
5958         }
5959         return ret;
5960 }
5961
5962 const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops = {
5963         .validate = flow_tcf_validate,
5964         .prepare = flow_tcf_prepare,
5965         .translate = flow_tcf_translate,
5966         .apply = flow_tcf_apply,
5967         .remove = flow_tcf_remove,
5968         .destroy = flow_tcf_destroy,
5969         .query = flow_tcf_query,
5970 };
5971
5972 /**
5973  * Create and configure a libmnl socket for Netlink flow rules.
5974  *
5975  * @return
5976  *   A valid libmnl socket object pointer on success, NULL otherwise and
5977  *   rte_errno is set.
5978  */
5979 static struct mnl_socket *
5980 flow_tcf_mnl_socket_create(void)
5981 {
5982         struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
5983
5984         if (nl) {
5985                 mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
5986                                       sizeof(int));
5987                 if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
5988                         return nl;
5989         }
5990         rte_errno = errno;
5991         if (nl)
5992                 mnl_socket_close(nl);
5993         return NULL;
5994 }
5995
5996 /**
5997  * Destroy a libmnl socket.
5998  *
5999  * @param nl
6000  *   Libmnl socket of the @p NETLINK_ROUTE kind.
6001  */
6002 static void
6003 flow_tcf_mnl_socket_destroy(struct mnl_socket *nl)
6004 {
6005         if (nl)
6006                 mnl_socket_close(nl);
6007 }
6008
6009 /**
6010  * Initialize ingress qdisc of a given network interface.
6011  *
6012  * @param ctx
6013  *   Pointer to tc-flower context to use.
6014  * @param ifindex
6015  *   Index of network interface to initialize.
6016  * @param[out] error
6017  *   Perform verbose error reporting if not NULL.
6018  *
6019  * @return
6020  *   0 on success, a negative errno value otherwise and rte_errno is set.
6021  */
6022 int
6023 mlx5_flow_tcf_init(struct mlx5_flow_tcf_context *ctx,
6024                    unsigned int ifindex, struct rte_flow_error *error)
6025 {
6026         struct nlmsghdr *nlh;
6027         struct tcmsg *tcm;
6028         alignas(struct nlmsghdr)
6029         uint8_t buf[mnl_nlmsg_size(sizeof(*tcm)) +
6030                     SZ_NLATTR_STRZ_OF("ingress") +
6031                     MNL_BUF_EXTRA_SPACE];
6032
6033         /* Destroy existing ingress qdisc and everything attached to it. */
6034         nlh = mnl_nlmsg_put_header(buf);
6035         nlh->nlmsg_type = RTM_DELQDISC;
6036         nlh->nlmsg_flags = NLM_F_REQUEST;
6037         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
6038         tcm->tcm_family = AF_UNSPEC;
6039         tcm->tcm_ifindex = ifindex;
6040         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
6041         tcm->tcm_parent = TC_H_INGRESS;
6042         assert(sizeof(buf) >= nlh->nlmsg_len);
6043         /* Ignore errors when qdisc is already absent. */
6044         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL) &&
6045             rte_errno != EINVAL && rte_errno != ENOENT)
6046                 return rte_flow_error_set(error, rte_errno,
6047                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6048                                           "netlink: failed to remove ingress"
6049                                           " qdisc");
6050         /* Create fresh ingress qdisc. */
6051         nlh = mnl_nlmsg_put_header(buf);
6052         nlh->nlmsg_type = RTM_NEWQDISC;
6053         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
6054         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
6055         tcm->tcm_family = AF_UNSPEC;
6056         tcm->tcm_ifindex = ifindex;
6057         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
6058         tcm->tcm_parent = TC_H_INGRESS;
6059         mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
6060         assert(sizeof(buf) >= nlh->nlmsg_len);
6061         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL))
6062                 return rte_flow_error_set(error, rte_errno,
6063                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6064                                           "netlink: failed to create ingress"
6065                                           " qdisc");
6066         return 0;
6067 }
6068
6069 /**
6070  * Create libmnl context for Netlink flow rules.
6071  *
6072  * @return
6073  *   A valid libmnl socket object pointer on success, NULL otherwise and
6074  *   rte_errno is set.
6075  */
6076 struct mlx5_flow_tcf_context *
6077 mlx5_flow_tcf_context_create(void)
6078 {
6079         struct mlx5_flow_tcf_context *ctx = rte_zmalloc(__func__,
6080                                                         sizeof(*ctx),
6081                                                         sizeof(uint32_t));
6082         if (!ctx)
6083                 goto error;
6084         ctx->nl = flow_tcf_mnl_socket_create();
6085         if (!ctx->nl)
6086                 goto error;
6087         ctx->buf_size = MNL_SOCKET_BUFFER_SIZE;
6088         ctx->buf = rte_zmalloc(__func__,
6089                                ctx->buf_size, sizeof(uint32_t));
6090         if (!ctx->buf)
6091                 goto error;
6092         ctx->seq = random();
6093         return ctx;
6094 error:
6095         mlx5_flow_tcf_context_destroy(ctx);
6096         return NULL;
6097 }
6098
6099 /**
6100  * Destroy a libmnl context.
6101  *
6102  * @param ctx
6103  *   Libmnl socket of the @p NETLINK_ROUTE kind.
6104  */
6105 void
6106 mlx5_flow_tcf_context_destroy(struct mlx5_flow_tcf_context *ctx)
6107 {
6108         if (!ctx)
6109                 return;
6110         flow_tcf_mnl_socket_destroy(ctx->nl);
6111         rte_free(ctx->buf);
6112         rte_free(ctx);
6113 }