drivers/net/mlx5/mlx5_flow_tcf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <assert.h>
   7 #include <errno.h>
   8 #include <libmnl/libmnl.h>
   9 #include <linux/gen_stats.h>
  10 #include <linux/if_ether.h>
  11 #include <linux/netlink.h>
  12 #include <linux/pkt_cls.h>
  13 #include <linux/pkt_sched.h>
  14 #include <linux/rtnetlink.h>
  15 #include <linux/tc_act/tc_gact.h>
  16 #include <linux/tc_act/tc_mirred.h>
  17 #include <netinet/in.h>
  18 #include <stdalign.h>
  19 #include <stdbool.h>
  20 #include <stddef.h>
  21 #include <stdint.h>
  22 #include <stdlib.h>
  23 #include <sys/socket.h>
  24
  25 #include <rte_byteorder.h>
  26 #include <rte_errno.h>
  27 #include <rte_ether.h>
  28 #include <rte_flow.h>
  29 #include <rte_malloc.h>
  30 #include <rte_common.h>
  31
  32 #include "mlx5.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_autoconf.h"
  35
  36 #ifdef HAVE_TC_ACT_VLAN
  37
  38 #include <linux/tc_act/tc_vlan.h>
  39
  40 #else /* HAVE_TC_ACT_VLAN */
  41
  42 #define TCA_VLAN_ACT_POP 1
  43 #define TCA_VLAN_ACT_PUSH 2
  44 #define TCA_VLAN_ACT_MODIFY 3
  45 #define TCA_VLAN_PARMS 2
  46 #define TCA_VLAN_PUSH_VLAN_ID 3
  47 #define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
  48 #define TCA_VLAN_PAD 5
  49 #define TCA_VLAN_PUSH_VLAN_PRIORITY 6
  50
  51 struct tc_vlan {
  52         tc_gen;
  53         int v_action;
  54 };
  55
  56 #endif /* HAVE_TC_ACT_VLAN */
  57
  58 #ifdef HAVE_TC_ACT_PEDIT
  59
  60 #include <linux/tc_act/tc_pedit.h>
  61
  62 #else /* HAVE_TC_ACT_VLAN */
  63
  64 enum {
  65         TCA_PEDIT_UNSPEC,
  66         TCA_PEDIT_TM,
  67         TCA_PEDIT_PARMS,
  68         TCA_PEDIT_PAD,
  69         TCA_PEDIT_PARMS_EX,
  70         TCA_PEDIT_KEYS_EX,
  71         TCA_PEDIT_KEY_EX,
  72         __TCA_PEDIT_MAX
  73 };
  74
  75 enum {
  76         TCA_PEDIT_KEY_EX_HTYPE = 1,
  77         TCA_PEDIT_KEY_EX_CMD = 2,
  78         __TCA_PEDIT_KEY_EX_MAX
  79 };
  80
  81 enum pedit_header_type {
  82         TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK = 0,
  83         TCA_PEDIT_KEY_EX_HDR_TYPE_ETH = 1,
  84         TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 = 2,
  85         TCA_PEDIT_KEY_EX_HDR_TYPE_IP6 = 3,
  86         TCA_PEDIT_KEY_EX_HDR_TYPE_TCP = 4,
  87         TCA_PEDIT_KEY_EX_HDR_TYPE_UDP = 5,
  88         __PEDIT_HDR_TYPE_MAX,
  89 };
  90
  91 enum pedit_cmd {
  92         TCA_PEDIT_KEY_EX_CMD_SET = 0,
  93         TCA_PEDIT_KEY_EX_CMD_ADD = 1,
  94         __PEDIT_CMD_MAX,
  95 };
  96
  97 struct tc_pedit_key {
  98         __u32 mask; /* AND */
  99         __u32 val; /*XOR */
 100         __u32 off; /*offset */
 101         __u32 at;
 102         __u32 offmask;
 103         __u32 shift;
 104 };
 105
 106 __extension__
 107 struct tc_pedit_sel {
 108         tc_gen;
 109         unsigned char nkeys;
 110         unsigned char flags;
 111         struct tc_pedit_key keys[0];
 112 };
 113
 114 #endif /* HAVE_TC_ACT_VLAN */
 115
 116 #ifdef HAVE_TC_ACT_TUNNEL_KEY
 117
 118 #include <linux/tc_act/tc_tunnel_key.h>
 119
 120 #ifndef HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT
 121 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 122 #endif
 123
 124 #ifndef HAVE_TCA_TUNNEL_KEY_NO_CSUM
 125 #define TCA_TUNNEL_KEY_NO_CSUM 10
 126 #endif
 127
 128 #else /* HAVE_TC_ACT_TUNNEL_KEY */
 129
 130 #define TCA_ACT_TUNNEL_KEY 17
 131 #define TCA_TUNNEL_KEY_ACT_SET 1
 132 #define TCA_TUNNEL_KEY_ACT_RELEASE 2
 133 #define TCA_TUNNEL_KEY_PARMS 2
 134 #define TCA_TUNNEL_KEY_ENC_IPV4_SRC 3
 135 #define TCA_TUNNEL_KEY_ENC_IPV4_DST 4
 136 #define TCA_TUNNEL_KEY_ENC_IPV6_SRC 5
 137 #define TCA_TUNNEL_KEY_ENC_IPV6_DST 6
 138 #define TCA_TUNNEL_KEY_ENC_KEY_ID 7
 139 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 140 #define TCA_TUNNEL_KEY_NO_CSUM 10
 141
 142 struct tc_tunnel_key {
 143         tc_gen;
 144         int t_action;
 145 };
 146
 147 #endif /* HAVE_TC_ACT_TUNNEL_KEY */
 148
 149 /* Normally found in linux/netlink.h. */
 150 #ifndef NETLINK_CAP_ACK
 151 #define NETLINK_CAP_ACK 10
 152 #endif
 153
 154 /* Normally found in linux/pkt_sched.h. */
 155 #ifndef TC_H_MIN_INGRESS
 156 #define TC_H_MIN_INGRESS 0xfff2u
 157 #endif
 158
 159 /* Normally found in linux/pkt_cls.h. */
 160 #ifndef TCA_CLS_FLAGS_SKIP_SW
 161 #define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
 162 #endif
 163 #ifndef TCA_CLS_FLAGS_IN_HW
 164 #define TCA_CLS_FLAGS_IN_HW (1 << 2)
 165 #endif
 166 #ifndef HAVE_TCA_CHAIN
 167 #define TCA_CHAIN 11
 168 #endif
 169 #ifndef HAVE_TCA_FLOWER_ACT
 170 #define TCA_FLOWER_ACT 3
 171 #endif
 172 #ifndef HAVE_TCA_FLOWER_FLAGS
 173 #define TCA_FLOWER_FLAGS 22
 174 #endif
 175 #ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
 176 #define TCA_FLOWER_KEY_ETH_TYPE 8
 177 #endif
 178 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
 179 #define TCA_FLOWER_KEY_ETH_DST 4
 180 #endif
 181 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
 182 #define TCA_FLOWER_KEY_ETH_DST_MASK 5
 183 #endif
 184 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
 185 #define TCA_FLOWER_KEY_ETH_SRC 6
 186 #endif
 187 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
 188 #define TCA_FLOWER_KEY_ETH_SRC_MASK 7
 189 #endif
 190 #ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
 191 #define TCA_FLOWER_KEY_IP_PROTO 9
 192 #endif
 193 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
 194 #define TCA_FLOWER_KEY_IPV4_SRC 10
 195 #endif
 196 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
 197 #define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
 198 #endif
 199 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
 200 #define TCA_FLOWER_KEY_IPV4_DST 12
 201 #endif
 202 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
 203 #define TCA_FLOWER_KEY_IPV4_DST_MASK 13
 204 #endif
 205 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
 206 #define TCA_FLOWER_KEY_IPV6_SRC 14
 207 #endif
 208 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
 209 #define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
 210 #endif
 211 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
 212 #define TCA_FLOWER_KEY_IPV6_DST 16
 213 #endif
 214 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
 215 #define TCA_FLOWER_KEY_IPV6_DST_MASK 17
 216 #endif
 217 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
 218 #define TCA_FLOWER_KEY_TCP_SRC 18
 219 #endif
 220 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
 221 #define TCA_FLOWER_KEY_TCP_SRC_MASK 35
 222 #endif
 223 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
 224 #define TCA_FLOWER_KEY_TCP_DST 19
 225 #endif
 226 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
 227 #define TCA_FLOWER_KEY_TCP_DST_MASK 36
 228 #endif
 229 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
 230 #define TCA_FLOWER_KEY_UDP_SRC 20
 231 #endif
 232 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
 233 #define TCA_FLOWER_KEY_UDP_SRC_MASK 37
 234 #endif
 235 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
 236 #define TCA_FLOWER_KEY_UDP_DST 21
 237 #endif
 238 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
 239 #define TCA_FLOWER_KEY_UDP_DST_MASK 38
 240 #endif
 241 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
 242 #define TCA_FLOWER_KEY_VLAN_ID 23
 243 #endif
 244 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
 245 #define TCA_FLOWER_KEY_VLAN_PRIO 24
 246 #endif
 247 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
 248 #define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
 249 #endif
 250 #ifndef HAVE_TCA_FLOWER_KEY_ENC_KEY_ID
 251 #define TCA_FLOWER_KEY_ENC_KEY_ID 26
 252 #endif
 253 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC
 254 #define TCA_FLOWER_KEY_ENC_IPV4_SRC 27
 255 #endif
 256 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK
 257 #define TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK 28
 258 #endif
 259 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST
 260 #define TCA_FLOWER_KEY_ENC_IPV4_DST 29
 261 #endif
 262 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK
 263 #define TCA_FLOWER_KEY_ENC_IPV4_DST_MASK 30
 264 #endif
 265 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC
 266 #define TCA_FLOWER_KEY_ENC_IPV6_SRC 31
 267 #endif
 268 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK
 269 #define TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK 32
 270 #endif
 271 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST
 272 #define TCA_FLOWER_KEY_ENC_IPV6_DST 33
 273 #endif
 274 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK
 275 #define TCA_FLOWER_KEY_ENC_IPV6_DST_MASK 34
 276 #endif
 277 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT
 278 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT 43
 279 #endif
 280 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK
 281 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK 44
 282 #endif
 283 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT
 284 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT 45
 285 #endif
 286 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK
 287 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK 46
 288 #endif
 289 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS
 290 #define TCA_FLOWER_KEY_TCP_FLAGS 71
 291 #endif
 292 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK
 293 #define TCA_FLOWER_KEY_TCP_FLAGS_MASK 72
 294 #endif
 295 #ifndef HAVE_TC_ACT_GOTO_CHAIN
 296 #define TC_ACT_GOTO_CHAIN 0x20000000
 297 #endif
 298
 299 #ifndef IPV6_ADDR_LEN
 300 #define IPV6_ADDR_LEN 16
 301 #endif
 302
 303 #ifndef IPV4_ADDR_LEN
 304 #define IPV4_ADDR_LEN 4
 305 #endif
 306
 307 #ifndef TP_PORT_LEN
 308 #define TP_PORT_LEN 2 /* Transport Port (UDP/TCP) Length */
 309 #endif
 310
 311 #ifndef TTL_LEN
 312 #define TTL_LEN 1
 313 #endif
 314
 315 #ifndef TCA_ACT_MAX_PRIO
 316 #define TCA_ACT_MAX_PRIO 32
 317 #endif
 318
 319 /** UDP port range of VXLAN devices created by driver. */
 320 #define MLX5_VXLAN_PORT_MIN 30000
 321 #define MLX5_VXLAN_PORT_MAX 60000
 322 #define MLX5_VXLAN_DEVICE_PFX "vmlx_"
 323
 324 /** Tunnel action type, used for @p type in header structure. */
 325 enum flow_tcf_tunact_type {
 326         FLOW_TCF_TUNACT_VXLAN_DECAP,
 327         FLOW_TCF_TUNACT_VXLAN_ENCAP,
 328 };
 329
 330 /** Flags used for @p mask in tunnel action encap descriptors. */
 331 #define FLOW_TCF_ENCAP_ETH_SRC (1u << 0)
 332 #define FLOW_TCF_ENCAP_ETH_DST (1u << 1)
 333 #define FLOW_TCF_ENCAP_IPV4_SRC (1u << 2)
 334 #define FLOW_TCF_ENCAP_IPV4_DST (1u << 3)
 335 #define FLOW_TCF_ENCAP_IPV6_SRC (1u << 4)
 336 #define FLOW_TCF_ENCAP_IPV6_DST (1u << 5)
 337 #define FLOW_TCF_ENCAP_UDP_SRC (1u << 6)
 338 #define FLOW_TCF_ENCAP_UDP_DST (1u << 7)
 339 #define FLOW_TCF_ENCAP_VXLAN_VNI (1u << 8)
 340
 341 /**
 342  * Structure for holding netlink context.
 343  * Note the size of the message buffer which is MNL_SOCKET_BUFFER_SIZE.
 344  * Using this (8KB) buffer size ensures that netlink messages will never be
 345  * truncated.
 346  */
 347 struct mlx5_flow_tcf_context {
 348         struct mnl_socket *nl; /* NETLINK_ROUTE libmnl socket. */
 349         uint32_t seq; /* Message sequence number. */
 350         uint32_t buf_size; /* Message buffer size. */
 351         uint8_t *buf; /* Message buffer. */
 352 };
 353
 354 /**
 355  * Neigh rule structure. The neigh rule is applied via Netlink to
 356  * outer tunnel iface in order to provide destination MAC address
 357  * for the VXLAN encapsultion. The neigh rule is implicitly related
 358  * to the Flow itself and can be shared by multiple Flows.
 359  */
 360 struct tcf_neigh_rule {
 361         LIST_ENTRY(tcf_neigh_rule) next;
 362         uint32_t refcnt;
 363         struct ether_addr eth;
 364         uint16_t mask;
 365         union {
 366                 struct {
 367                         rte_be32_t dst;
 368                 } ipv4;
 369                 struct {
 370                         uint8_t dst[IPV6_ADDR_LEN];
 371                 } ipv6;
 372         };
 373 };
 374
 375 /**
 376  * Local rule structure. The local rule is applied via Netlink to
 377  * outer tunnel iface in order to provide local and peer IP addresses
 378  * of the VXLAN tunnel for encapsulation. The local rule is implicitly
 379  * related to the Flow itself and can be shared by multiple Flows.
 380  */
 381 struct tcf_local_rule {
 382         LIST_ENTRY(tcf_local_rule) next;
 383         uint32_t refcnt;
 384         uint16_t mask;
 385         union {
 386                 struct {
 387                         rte_be32_t dst;
 388                         rte_be32_t src;
 389                 } ipv4;
 390                 struct {
 391                         uint8_t dst[IPV6_ADDR_LEN];
 392                         uint8_t src[IPV6_ADDR_LEN];
 393                 } ipv6;
 394         };
 395 };
 396
 397 /** VXLAN virtual netdev. */
 398 struct tcf_vtep {
 399         LIST_ENTRY(tcf_vtep) next;
 400         LIST_HEAD(, tcf_neigh_rule) neigh;
 401         LIST_HEAD(, tcf_local_rule) local;
 402         uint32_t refcnt;
 403         unsigned int ifindex; /**< Own interface index. */
 404         unsigned int ifouter; /**< Index of device attached to. */
 405         uint16_t port;
 406         uint8_t created;
 407 };
 408
 409 /** Tunnel descriptor header, common for all tunnel types. */
 410 struct flow_tcf_tunnel_hdr {
 411         uint32_t type; /**< Tunnel action type. */
 412         struct tcf_vtep *vtep; /**< Virtual tunnel endpoint device. */
 413         unsigned int ifindex_org; /**< Original dst/src interface */
 414         unsigned int *ifindex_ptr; /**< Interface ptr in message. */
 415 };
 416
 417 struct flow_tcf_vxlan_decap {
 418         struct flow_tcf_tunnel_hdr hdr;
 419         uint16_t udp_port;
 420 };
 421
 422 struct flow_tcf_vxlan_encap {
 423         struct flow_tcf_tunnel_hdr hdr;
 424         uint32_t mask;
 425         struct {
 426                 struct ether_addr dst;
 427                 struct ether_addr src;
 428         } eth;
 429         union {
 430                 struct {
 431                         rte_be32_t dst;
 432                         rte_be32_t src;
 433                 } ipv4;
 434                 struct {
 435                         uint8_t dst[IPV6_ADDR_LEN];
 436                         uint8_t src[IPV6_ADDR_LEN];
 437                 } ipv6;
 438         };
 439 struct {
 440                 rte_be16_t src;
 441                 rte_be16_t dst;
 442         } udp;
 443         struct {
 444                 uint8_t vni[3];
 445         } vxlan;
 446 };
 447
 448 /** Structure used when extracting the values of a flow counters
 449  * from a netlink message.
 450  */
 451 struct flow_tcf_stats_basic {
 452         bool valid;
 453         struct gnet_stats_basic counters;
 454 };
 455
 456 /** Empty masks for known item types. */
 457 static const union {
 458         struct rte_flow_item_port_id port_id;
 459         struct rte_flow_item_eth eth;
 460         struct rte_flow_item_vlan vlan;
 461         struct rte_flow_item_ipv4 ipv4;
 462         struct rte_flow_item_ipv6 ipv6;
 463         struct rte_flow_item_tcp tcp;
 464         struct rte_flow_item_udp udp;
 465         struct rte_flow_item_vxlan vxlan;
 466 } flow_tcf_mask_empty = {
 467         {0},
 468 };
 469
 470 /** Supported masks for known item types. */
 471 static const struct {
 472         struct rte_flow_item_port_id port_id;
 473         struct rte_flow_item_eth eth;
 474         struct rte_flow_item_vlan vlan;
 475         struct rte_flow_item_ipv4 ipv4;
 476         struct rte_flow_item_ipv6 ipv6;
 477         struct rte_flow_item_tcp tcp;
 478         struct rte_flow_item_udp udp;
 479         struct rte_flow_item_vxlan vxlan;
 480 } flow_tcf_mask_supported = {
 481         .port_id = {
 482                 .id = 0xffffffff,
 483         },
 484         .eth = {
 485                 .type = RTE_BE16(0xffff),
 486                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 487                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 488         },
 489         .vlan = {
 490                 /* PCP and VID only, no DEI. */
 491                 .tci = RTE_BE16(0xefff),
 492                 .inner_type = RTE_BE16(0xffff),
 493         },
 494         .ipv4.hdr = {
 495                 .next_proto_id = 0xff,
 496                 .src_addr = RTE_BE32(0xffffffff),
 497                 .dst_addr = RTE_BE32(0xffffffff),
 498         },
 499         .ipv6.hdr = {
 500                 .proto = 0xff,
 501                 .src_addr =
 502                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 503                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 504                 .dst_addr =
 505                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 506                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 507         },
 508         .tcp.hdr = {
 509                 .src_port = RTE_BE16(0xffff),
 510                 .dst_port = RTE_BE16(0xffff),
 511                 .tcp_flags = 0xff,
 512         },
 513         .udp.hdr = {
 514                 .src_port = RTE_BE16(0xffff),
 515                 .dst_port = RTE_BE16(0xffff),
 516         },
 517         .vxlan = {
 518                .vni = "\xff\xff\xff",
 519         },
 520 };
 521
 522 #define SZ_NLATTR_HDR MNL_ALIGN(sizeof(struct nlattr))
 523 #define SZ_NLATTR_NEST SZ_NLATTR_HDR
 524 #define SZ_NLATTR_DATA_OF(len) MNL_ALIGN(SZ_NLATTR_HDR + (len))
 525 #define SZ_NLATTR_TYPE_OF(typ) SZ_NLATTR_DATA_OF(sizeof(typ))
 526 #define SZ_NLATTR_STRZ_OF(str) SZ_NLATTR_DATA_OF(strlen(str) + 1)
 527
 528 #define PTOI_TABLE_SZ_MAX(dev) (mlx5_dev_to_port_id((dev)->device, NULL, 0) + 2)
 529
 530 /** DPDK port to network interface index (ifindex) conversion. */
 531 struct flow_tcf_ptoi {
 532         uint16_t port_id; /**< DPDK port ID. */
 533         unsigned int ifindex; /**< Network interface index. */
 534 };
 535
 536 /* Due to a limitation on driver/FW. */
 537 #define MLX5_TCF_GROUP_ID_MAX 3
 538
 539 /*
 540  * Due to a limitation on driver/FW, priority ranges from 1 to 16 in kernel.
 541  * Priority in rte_flow attribute starts from 0 and is added by 1 in
 542  * translation. This is subject to be changed to determine the max priority
 543  * based on trial-and-error like Verbs driver once the restriction is lifted or
 544  * the range is extended.
 545  */
 546 #define MLX5_TCF_GROUP_PRIORITY_MAX 15
 547
 548 #define MLX5_TCF_FATE_ACTIONS \
 549         (MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
 550          MLX5_FLOW_ACTION_JUMP)
 551
 552 #define MLX5_TCF_VLAN_ACTIONS \
 553         (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN | \
 554          MLX5_FLOW_ACTION_OF_SET_VLAN_VID | MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
 555
 556 #define MLX5_TCF_VXLAN_ACTIONS \
 557         (MLX5_FLOW_ACTION_VXLAN_ENCAP | MLX5_FLOW_ACTION_VXLAN_DECAP)
 558
 559 #define MLX5_TCF_PEDIT_ACTIONS \
 560         (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST | \
 561          MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST | \
 562          MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST | \
 563          MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL | \
 564          MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)
 565
 566 #define MLX5_TCF_CONFIG_ACTIONS \
 567         (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_JUMP | \
 568          MLX5_FLOW_ACTION_OF_PUSH_VLAN | MLX5_FLOW_ACTION_OF_SET_VLAN_VID | \
 569          MLX5_FLOW_ACTION_OF_SET_VLAN_PCP | \
 570          (MLX5_TCF_PEDIT_ACTIONS & ~MLX5_FLOW_ACTION_DEC_TTL))
 571
 572 #define MAX_PEDIT_KEYS 128
 573 #define SZ_PEDIT_KEY_VAL 4
 574
 575 #define NUM_OF_PEDIT_KEYS(sz) \
 576         (((sz) / SZ_PEDIT_KEY_VAL) + (((sz) % SZ_PEDIT_KEY_VAL) ? 1 : 0))
 577
 578 struct pedit_key_ex {
 579         enum pedit_header_type htype;
 580         enum pedit_cmd cmd;
 581 };
 582
 583 struct pedit_parser {
 584         struct tc_pedit_sel sel;
 585         struct tc_pedit_key keys[MAX_PEDIT_KEYS];
 586         struct pedit_key_ex keys_ex[MAX_PEDIT_KEYS];
 587 };
 588
 589 /**
 590  * Create space for using the implicitly created TC flow counter.
 591  *
 592  * @param[in] dev
 593  *   Pointer to the Ethernet device structure.
 594  *
 595  * @return
 596  *   A pointer to the counter data structure, NULL otherwise and
 597  *   rte_errno is set.
 598  */
 599 static struct mlx5_flow_counter *
 600 flow_tcf_counter_new(void)
 601 {
 602         struct mlx5_flow_counter *cnt;
 603
 604         /*
 605          * eswitch counter cannot be shared and its id is unknown.
 606          * currently returning all with id 0.
 607          * in the future maybe better to switch to unique numbers.
 608          */
 609         struct mlx5_flow_counter tmpl = {
 610                 .ref_cnt = 1,
 611         };
 612         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 613         if (!cnt) {
 614                 rte_errno = ENOMEM;
 615                 return NULL;
 616         }
 617         *cnt = tmpl;
 618         /* Implicit counter, do not add to list. */
 619         return cnt;
 620 }
 621
 622 /**
 623  * Set pedit key of MAC address
 624  *
 625  * @param[in] actions
 626  *   pointer to action specification
 627  * @param[in,out] p_parser
 628  *   pointer to pedit_parser
 629  */
 630 static void
 631 flow_tcf_pedit_key_set_mac(const struct rte_flow_action *actions,
 632                            struct pedit_parser *p_parser)
 633 {
 634         int idx = p_parser->sel.nkeys;
 635         uint32_t off = actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
 636                                         offsetof(struct ether_hdr, s_addr) :
 637                                         offsetof(struct ether_hdr, d_addr);
 638         const struct rte_flow_action_set_mac *conf =
 639                 (const struct rte_flow_action_set_mac *)actions->conf;
 640
 641         p_parser->keys[idx].off = off;
 642         p_parser->keys[idx].mask = ~UINT32_MAX;
 643         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 644         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 645         memcpy(&p_parser->keys[idx].val,
 646                 conf->mac_addr, SZ_PEDIT_KEY_VAL);
 647         idx++;
 648         p_parser->keys[idx].off = off + SZ_PEDIT_KEY_VAL;
 649         p_parser->keys[idx].mask = 0xFFFF0000;
 650         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 651         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 652         memcpy(&p_parser->keys[idx].val,
 653                 conf->mac_addr + SZ_PEDIT_KEY_VAL,
 654                 ETHER_ADDR_LEN - SZ_PEDIT_KEY_VAL);
 655         p_parser->sel.nkeys = (++idx);
 656 }
 657
 658 /**
 659  * Set pedit key of decrease/set ttl
 660  *
 661  * @param[in] actions
 662  *   pointer to action specification
 663  * @param[in,out] p_parser
 664  *   pointer to pedit_parser
 665  * @param[in] item_flags
 666  *   flags of all items presented
 667  */
 668 static void
 669 flow_tcf_pedit_key_set_dec_ttl(const struct rte_flow_action *actions,
 670                                 struct pedit_parser *p_parser,
 671                                 uint64_t item_flags)
 672 {
 673         int idx = p_parser->sel.nkeys;
 674
 675         p_parser->keys[idx].mask = 0xFFFFFF00;
 676         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) {
 677                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 678                 p_parser->keys[idx].off =
 679                         offsetof(struct ipv4_hdr, time_to_live);
 680         }
 681         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6) {
 682                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 683                 p_parser->keys[idx].off =
 684                         offsetof(struct ipv6_hdr, hop_limits);
 685         }
 686         if (actions->type == RTE_FLOW_ACTION_TYPE_DEC_TTL) {
 687                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_ADD;
 688                 p_parser->keys[idx].val = 0x000000FF;
 689         } else {
 690                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 691                 p_parser->keys[idx].val =
 692                         (__u32)((const struct rte_flow_action_set_ttl *)
 693                          actions->conf)->ttl_value;
 694         }
 695         p_parser->sel.nkeys = (++idx);
 696 }
 697
 698 /**
 699  * Set pedit key of transport (TCP/UDP) port value
 700  *
 701  * @param[in] actions
 702  *   pointer to action specification
 703  * @param[in,out] p_parser
 704  *   pointer to pedit_parser
 705  * @param[in] item_flags
 706  *   flags of all items presented
 707  */
 708 static void
 709 flow_tcf_pedit_key_set_tp_port(const struct rte_flow_action *actions,
 710                                 struct pedit_parser *p_parser,
 711                                 uint64_t item_flags)
 712 {
 713         int idx = p_parser->sel.nkeys;
 714
 715         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 716                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_UDP;
 717         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 718                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_TCP;
 719         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 720         /* offset of src/dst port is same for TCP and UDP */
 721         p_parser->keys[idx].off =
 722                 actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
 723                 offsetof(struct tcp_hdr, src_port) :
 724                 offsetof(struct tcp_hdr, dst_port);
 725         p_parser->keys[idx].mask = 0xFFFF0000;
 726         p_parser->keys[idx].val =
 727                 (__u32)((const struct rte_flow_action_set_tp *)
 728                                 actions->conf)->port;
 729         p_parser->sel.nkeys = (++idx);
 730 }
 731
 732 /**
 733  * Set pedit key of ipv6 address
 734  *
 735  * @param[in] actions
 736  *   pointer to action specification
 737  * @param[in,out] p_parser
 738  *   pointer to pedit_parser
 739  */
 740 static void
 741 flow_tcf_pedit_key_set_ipv6_addr(const struct rte_flow_action *actions,
 742                                  struct pedit_parser *p_parser)
 743 {
 744         int idx = p_parser->sel.nkeys;
 745         int keys = NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 746         int off_base =
 747                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
 748                 offsetof(struct ipv6_hdr, src_addr) :
 749                 offsetof(struct ipv6_hdr, dst_addr);
 750         const struct rte_flow_action_set_ipv6 *conf =
 751                 (const struct rte_flow_action_set_ipv6 *)actions->conf;
 752
 753         for (int i = 0; i < keys; i++, idx++) {
 754                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 755                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 756                 p_parser->keys[idx].off = off_base + i * SZ_PEDIT_KEY_VAL;
 757                 p_parser->keys[idx].mask = ~UINT32_MAX;
 758                 memcpy(&p_parser->keys[idx].val,
 759                         conf->ipv6_addr + i *  SZ_PEDIT_KEY_VAL,
 760                         SZ_PEDIT_KEY_VAL);
 761         }
 762         p_parser->sel.nkeys += keys;
 763 }
 764
 765 /**
 766  * Set pedit key of ipv4 address
 767  *
 768  * @param[in] actions
 769  *   pointer to action specification
 770  * @param[in,out] p_parser
 771  *   pointer to pedit_parser
 772  */
 773 static void
 774 flow_tcf_pedit_key_set_ipv4_addr(const struct rte_flow_action *actions,
 775                                  struct pedit_parser *p_parser)
 776 {
 777         int idx = p_parser->sel.nkeys;
 778
 779         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 780         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 781         p_parser->keys[idx].off =
 782                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
 783                 offsetof(struct ipv4_hdr, src_addr) :
 784                 offsetof(struct ipv4_hdr, dst_addr);
 785         p_parser->keys[idx].mask = ~UINT32_MAX;
 786         p_parser->keys[idx].val =
 787                 ((const struct rte_flow_action_set_ipv4 *)
 788                  actions->conf)->ipv4_addr;
 789         p_parser->sel.nkeys = (++idx);
 790 }
 791
 792 /**
 793  * Create the pedit's na attribute in netlink message
 794  * on pre-allocate message buffer
 795  *
 796  * @param[in,out] nl
 797  *   pointer to pre-allocated netlink message buffer
 798  * @param[in,out] actions
 799  *   pointer to pointer of actions specification.
 800  * @param[in,out] action_flags
 801  *   pointer to actions flags
 802  * @param[in] item_flags
 803  *   flags of all item presented
 804  */
 805 static void
 806 flow_tcf_create_pedit_mnl_msg(struct nlmsghdr *nl,
 807                               const struct rte_flow_action **actions,
 808                               uint64_t item_flags)
 809 {
 810         struct pedit_parser p_parser;
 811         struct nlattr *na_act_options;
 812         struct nlattr *na_pedit_keys;
 813
 814         memset(&p_parser, 0, sizeof(p_parser));
 815         mnl_attr_put_strz(nl, TCA_ACT_KIND, "pedit");
 816         na_act_options = mnl_attr_nest_start(nl, TCA_ACT_OPTIONS);
 817         /* all modify header actions should be in one tc-pedit action */
 818         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 819                 switch ((*actions)->type) {
 820                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 821                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 822                         flow_tcf_pedit_key_set_ipv4_addr(*actions, &p_parser);
 823                         break;
 824                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 825                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 826                         flow_tcf_pedit_key_set_ipv6_addr(*actions, &p_parser);
 827                         break;
 828                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 829                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 830                         flow_tcf_pedit_key_set_tp_port(*actions,
 831                                                         &p_parser, item_flags);
 832                         break;
 833                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 834                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 835                         flow_tcf_pedit_key_set_dec_ttl(*actions,
 836                                                         &p_parser, item_flags);
 837                         break;
 838                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 839                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 840                         flow_tcf_pedit_key_set_mac(*actions, &p_parser);
 841                         break;
 842                 default:
 843                         goto pedit_mnl_msg_done;
 844                 }
 845         }
 846 pedit_mnl_msg_done:
 847         p_parser.sel.action = TC_ACT_PIPE;
 848         mnl_attr_put(nl, TCA_PEDIT_PARMS_EX,
 849                      sizeof(p_parser.sel) +
 850                      p_parser.sel.nkeys * sizeof(struct tc_pedit_key),
 851                      &p_parser);
 852         na_pedit_keys =
 853                 mnl_attr_nest_start(nl, TCA_PEDIT_KEYS_EX | NLA_F_NESTED);
 854         for (int i = 0; i < p_parser.sel.nkeys; i++) {
 855                 struct nlattr *na_pedit_key =
 856                         mnl_attr_nest_start(nl,
 857                                             TCA_PEDIT_KEY_EX | NLA_F_NESTED);
 858                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_HTYPE,
 859                                  p_parser.keys_ex[i].htype);
 860                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_CMD,
 861                                  p_parser.keys_ex[i].cmd);
 862                 mnl_attr_nest_end(nl, na_pedit_key);
 863         }
 864         mnl_attr_nest_end(nl, na_pedit_keys);
 865         mnl_attr_nest_end(nl, na_act_options);
 866         (*actions)--;
 867 }
 868
 869 /**
 870  * Calculate max memory size of one TC-pedit actions.
 871  * One TC-pedit action can contain set of keys each defining
 872  * a rewrite element (rte_flow action)
 873  *
 874  * @param[in,out] actions
 875  *   actions specification.
 876  * @param[in,out] action_flags
 877  *   actions flags
 878  * @param[in,out] size
 879  *   accumulated size
 880  * @return
 881  *   Max memory size of one TC-pedit action
 882  */
 883 static int
 884 flow_tcf_get_pedit_actions_size(const struct rte_flow_action **actions,
 885                                 uint64_t *action_flags)
 886 {
 887         int pedit_size = 0;
 888         int keys = 0;
 889         uint64_t flags = 0;
 890
 891         pedit_size += SZ_NLATTR_NEST + /* na_act_index. */
 892                       SZ_NLATTR_STRZ_OF("pedit") +
 893                       SZ_NLATTR_NEST; /* TCA_ACT_OPTIONS. */
 894         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 895                 switch ((*actions)->type) {
 896                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 897                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 898                         flags |= MLX5_FLOW_ACTION_SET_IPV4_SRC;
 899                         break;
 900                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 901                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 902                         flags |= MLX5_FLOW_ACTION_SET_IPV4_DST;
 903                         break;
 904                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 905                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 906                         flags |= MLX5_FLOW_ACTION_SET_IPV6_SRC;
 907                         break;
 908                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 909                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 910                         flags |= MLX5_FLOW_ACTION_SET_IPV6_DST;
 911                         break;
 912                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 913                         /* TCP is as same as UDP */
 914                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 915                         flags |= MLX5_FLOW_ACTION_SET_TP_SRC;
 916                         break;
 917                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 918                         /* TCP is as same as UDP */
 919                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 920                         flags |= MLX5_FLOW_ACTION_SET_TP_DST;
 921                         break;
 922                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 923                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 924                         flags |= MLX5_FLOW_ACTION_SET_TTL;
 925                         break;
 926                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 927                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 928                         flags |= MLX5_FLOW_ACTION_DEC_TTL;
 929                         break;
 930                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 931                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 932                         flags |= MLX5_FLOW_ACTION_SET_MAC_SRC;
 933                         break;
 934                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 935                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 936                         flags |= MLX5_FLOW_ACTION_SET_MAC_DST;
 937                         break;
 938                 default:
 939                         goto get_pedit_action_size_done;
 940                 }
 941         }
 942 get_pedit_action_size_done:
 943         /* TCA_PEDIT_PARAMS_EX */
 944         pedit_size +=
 945                 SZ_NLATTR_DATA_OF(sizeof(struct tc_pedit_sel) +
 946                                   keys * sizeof(struct tc_pedit_key));
 947         pedit_size += SZ_NLATTR_NEST; /* TCA_PEDIT_KEYS */
 948         pedit_size += keys *
 949                       /* TCA_PEDIT_KEY_EX + HTYPE + CMD */
 950                       (SZ_NLATTR_NEST + SZ_NLATTR_DATA_OF(2) +
 951                        SZ_NLATTR_DATA_OF(2));
 952         (*action_flags) |= flags;
 953         (*actions)--;
 954         return pedit_size;
 955 }
 956
 957 /**
 958  * Retrieve mask for pattern item.
 959  *
 960  * This function does basic sanity checks on a pattern item in order to
 961  * return the most appropriate mask for it.
 962  *
 963  * @param[in] item
 964  *   Item specification.
 965  * @param[in] mask_default
 966  *   Default mask for pattern item as specified by the flow API.
 967  * @param[in] mask_supported
 968  *   Mask fields supported by the implementation.
 969  * @param[in] mask_empty
 970  *   Empty mask to return when there is no specification.
 971  * @param[out] error
 972  *   Perform verbose error reporting if not NULL.
 973  *
 974  * @return
 975  *   Either @p item->mask or one of the mask parameters on success, NULL
 976  *   otherwise and rte_errno is set.
 977  */
 978 static const void *
 979 flow_tcf_item_mask(const struct rte_flow_item *item, const void *mask_default,
 980                    const void *mask_supported, const void *mask_empty,
 981                    size_t mask_size, struct rte_flow_error *error)
 982 {
 983         const uint8_t *mask;
 984         size_t i;
 985
 986         /* item->last and item->mask cannot exist without item->spec. */
 987         if (!item->spec && (item->mask || item->last)) {
 988                 rte_flow_error_set(error, EINVAL,
 989                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 990                                    "\"mask\" or \"last\" field provided without"
 991                                    " a corresponding \"spec\"");
 992                 return NULL;
 993         }
 994         /* No spec, no mask, no problem. */
 995         if (!item->spec)
 996                 return mask_empty;
 997         mask = item->mask ? item->mask : mask_default;
 998         assert(mask);
 999         /*
1000          * Single-pass check to make sure that:
1001          * - Mask is supported, no bits are set outside mask_supported.
1002          * - Both item->spec and item->last are included in mask.
1003          */
1004         for (i = 0; i != mask_size; ++i) {
1005                 if (!mask[i])
1006                         continue;
1007                 if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
1008                     ((const uint8_t *)mask_supported)[i]) {
1009                         rte_flow_error_set(error, ENOTSUP,
1010                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1011                                            "unsupported field found"
1012                                            " in \"mask\"");
1013                         return NULL;
1014                 }
1015                 if (item->last &&
1016                     (((const uint8_t *)item->spec)[i] & mask[i]) !=
1017                     (((const uint8_t *)item->last)[i] & mask[i])) {
1018                         rte_flow_error_set(error, EINVAL,
1019                                            RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1020                                            item->last,
1021                                            "range between \"spec\" and \"last\""
1022                                            " not comprised in \"mask\"");
1023                         return NULL;
1024                 }
1025         }
1026         return mask;
1027 }
1028
1029 /**
1030  * Build a conversion table between port ID and ifindex.
1031  *
1032  * @param[in] dev
1033  *   Pointer to Ethernet device.
1034  * @param[out] ptoi
1035  *   Pointer to ptoi table.
1036  * @param[in] len
1037  *   Size of ptoi table provided.
1038  *
1039  * @return
1040  *   Size of ptoi table filled.
1041  */
1042 static unsigned int
1043 flow_tcf_build_ptoi_table(struct rte_eth_dev *dev, struct flow_tcf_ptoi *ptoi,
1044                           unsigned int len)
1045 {
1046         unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
1047         uint16_t port_id[n + 1];
1048         unsigned int i;
1049         unsigned int own = 0;
1050
1051         /* At least one port is needed when no switch domain is present. */
1052         if (!n) {
1053                 n = 1;
1054                 port_id[0] = dev->data->port_id;
1055         } else {
1056                 n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
1057         }
1058         if (n > len)
1059                 return 0;
1060         for (i = 0; i != n; ++i) {
1061                 struct rte_eth_dev_info dev_info;
1062
1063                 rte_eth_dev_info_get(port_id[i], &dev_info);
1064                 if (port_id[i] == dev->data->port_id)
1065                         own = i;
1066                 ptoi[i].port_id = port_id[i];
1067                 ptoi[i].ifindex = dev_info.if_index;
1068         }
1069         /* Ensure first entry of ptoi[] is the current device. */
1070         if (own) {
1071                 ptoi[n] = ptoi[0];
1072                 ptoi[0] = ptoi[own];
1073                 ptoi[own] = ptoi[n];
1074         }
1075         /* An entry with zero ifindex terminates ptoi[]. */
1076         ptoi[n].port_id = 0;
1077         ptoi[n].ifindex = 0;
1078         return n;
1079 }
1080
1081 /**
1082  * Verify the @p attr will be correctly understood by the E-switch.
1083  *
1084  * @param[in] attr
1085  *   Pointer to flow attributes
1086  * @param[out] error
1087  *   Pointer to error structure.
1088  *
1089  * @return
1090  *   0 on success, a negative errno value otherwise and rte_errno is set.
1091  */
1092 static int
1093 flow_tcf_validate_attributes(const struct rte_flow_attr *attr,
1094                              struct rte_flow_error *error)
1095 {
1096         /*
1097          * Supported attributes: groups, some priorities and ingress only.
1098          * group is supported only if kernel supports chain. Don't care about
1099          * transfer as it is the caller's problem.
1100          */
1101         if (attr->group > MLX5_TCF_GROUP_ID_MAX)
1102                 return rte_flow_error_set(error, ENOTSUP,
1103                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1104                                           "group ID larger than "
1105                                           RTE_STR(MLX5_TCF_GROUP_ID_MAX)
1106                                           " isn't supported");
1107         else if (attr->priority > MLX5_TCF_GROUP_PRIORITY_MAX)
1108                 return rte_flow_error_set(error, ENOTSUP,
1109                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1110                                           attr,
1111                                           "priority more than "
1112                                           RTE_STR(MLX5_TCF_GROUP_PRIORITY_MAX)
1113                                           " is not supported");
1114         if (!attr->ingress)
1115                 return rte_flow_error_set(error, EINVAL,
1116                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1117                                           attr, "only ingress is supported");
1118         if (attr->egress)
1119                 return rte_flow_error_set(error, ENOTSUP,
1120                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1121                                           attr, "egress is not supported");
1122         return 0;
1123 }
1124
1125 /**
1126  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_ETH item for E-Switch.
1127  * The routine checks the L2 fields to be used in encapsulation header.
1128  *
1129  * @param[in] item
1130  *   Pointer to the item structure.
1131  * @param[out] error
1132  *   Pointer to the error structure.
1133  *
1134  * @return
1135  *   0 on success, a negative errno value otherwise and rte_errno is set.
1136  **/
1137 static int
1138 flow_tcf_validate_vxlan_encap_eth(const struct rte_flow_item *item,
1139                                   struct rte_flow_error *error)
1140 {
1141         const struct rte_flow_item_eth *spec = item->spec;
1142         const struct rte_flow_item_eth *mask = item->mask;
1143
1144         if (!spec) {
1145                 /*
1146                  * Specification for L2 addresses can be empty
1147                  * because these ones are optional and not
1148                  * required directly by tc rule. Kernel tries
1149                  * to resolve these ones on its own
1150                  */
1151                 return 0;
1152         }
1153         if (!mask) {
1154                 /* If mask is not specified use the default one. */
1155                 mask = &rte_flow_item_eth_mask;
1156         }
1157         if (memcmp(&mask->dst,
1158                    &flow_tcf_mask_empty.eth.dst,
1159                    sizeof(flow_tcf_mask_empty.eth.dst))) {
1160                 if (memcmp(&mask->dst,
1161                            &rte_flow_item_eth_mask.dst,
1162                            sizeof(rte_flow_item_eth_mask.dst)))
1163                         return rte_flow_error_set
1164                                 (error, ENOTSUP,
1165                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1166                                  "no support for partial mask on"
1167                                  " \"eth.dst\" field");
1168         }
1169         if (memcmp(&mask->src,
1170                    &flow_tcf_mask_empty.eth.src,
1171                    sizeof(flow_tcf_mask_empty.eth.src))) {
1172                 if (memcmp(&mask->src,
1173                            &rte_flow_item_eth_mask.src,
1174                            sizeof(rte_flow_item_eth_mask.src)))
1175                         return rte_flow_error_set
1176                                 (error, ENOTSUP,
1177                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1178                                  "no support for partial mask on"
1179                                  " \"eth.src\" field");
1180         }
1181         if (mask->type != RTE_BE16(0x0000)) {
1182                 if (mask->type != RTE_BE16(0xffff))
1183                         return rte_flow_error_set
1184                                 (error, ENOTSUP,
1185                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1186                                  "no support for partial mask on"
1187                                  " \"eth.type\" field");
1188                 DRV_LOG(WARNING,
1189                         "outer ethernet type field"
1190                         " cannot be forced for vxlan"
1191                         " encapsulation, parameter ignored");
1192         }
1193         return 0;
1194 }
1195
1196 /**
1197  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV4 item for E-Switch.
1198  * The routine checks the IPv4 fields to be used in encapsulation header.
1199  *
1200  * @param[in] item
1201  *   Pointer to the item structure.
1202  * @param[out] error
1203  *   Pointer to the error structure.
1204  *
1205  * @return
1206  *   0 on success, a negative errno value otherwise and rte_errno is set.
1207  **/
1208 static int
1209 flow_tcf_validate_vxlan_encap_ipv4(const struct rte_flow_item *item,
1210                                    struct rte_flow_error *error)
1211 {
1212         const struct rte_flow_item_ipv4 *spec = item->spec;
1213         const struct rte_flow_item_ipv4 *mask = item->mask;
1214
1215         if (!spec) {
1216                 /*
1217                  * Specification for IP addresses cannot be empty
1218                  * because it is required by tunnel_key parameter.
1219                  */
1220                 return rte_flow_error_set(error, EINVAL,
1221                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1222                                           "NULL outer ipv4 address"
1223                                           " specification for vxlan"
1224                                           " encapsulation");
1225         }
1226         if (!mask)
1227                 mask = &rte_flow_item_ipv4_mask;
1228         if (mask->hdr.dst_addr != RTE_BE32(0x00000000)) {
1229                 if (mask->hdr.dst_addr != RTE_BE32(0xffffffff))
1230                         return rte_flow_error_set
1231                                 (error, ENOTSUP,
1232                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1233                                  "no support for partial mask on"
1234                                  " \"ipv4.hdr.dst_addr\" field"
1235                                  " for vxlan encapsulation");
1236                 /* More IPv4 address validations can be put here. */
1237         } else {
1238                 /*
1239                  * Kernel uses the destination IP address to determine
1240                  * the routing path and obtain the MAC destination
1241                  * address, so IP destination address must be
1242                  * specified in the tc rule.
1243                  */
1244                 return rte_flow_error_set(error, EINVAL,
1245                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1246                                           "outer ipv4 destination address"
1247                                           " must be specified for"
1248                                           " vxlan encapsulation");
1249         }
1250         if (mask->hdr.src_addr != RTE_BE32(0x00000000)) {
1251                 if (mask->hdr.src_addr != RTE_BE32(0xffffffff))
1252                         return rte_flow_error_set
1253                                 (error, ENOTSUP,
1254                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1255                                  "no support for partial mask on"
1256                                  " \"ipv4.hdr.src_addr\" field"
1257                                  " for vxlan encapsulation");
1258                 /* More IPv4 address validations can be put here. */
1259         } else {
1260                 /*
1261                  * Kernel uses the source IP address to select the
1262                  * interface for egress encapsulated traffic, so
1263                  * it must be specified in the tc rule.
1264                  */
1265                 return rte_flow_error_set(error, EINVAL,
1266                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1267                                           "outer ipv4 source address"
1268                                           " must be specified for"
1269                                           " vxlan encapsulation");
1270         }
1271         return 0;
1272 }
1273
1274 /**
1275  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV6 item for E-Switch.
1276  * The routine checks the IPv6 fields to be used in encapsulation header.
1277  *
1278  * @param[in] item
1279  *   Pointer to the item structure.
1280  * @param[out] error
1281  *   Pointer to the error structure.
1282  *
1283  * @return
1284  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1285  **/
1286 static int
1287 flow_tcf_validate_vxlan_encap_ipv6(const struct rte_flow_item *item,
1288                                    struct rte_flow_error *error)
1289 {
1290         const struct rte_flow_item_ipv6 *spec = item->spec;
1291         const struct rte_flow_item_ipv6 *mask = item->mask;
1292
1293         if (!spec) {
1294                 /*
1295                  * Specification for IP addresses cannot be empty
1296                  * because it is required by tunnel_key parameter.
1297                  */
1298                 return rte_flow_error_set(error, EINVAL,
1299                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1300                                           "NULL outer ipv6 address"
1301                                           " specification for"
1302                                           " vxlan encapsulation");
1303         }
1304         if (!mask)
1305                 mask = &rte_flow_item_ipv6_mask;
1306         if (memcmp(&mask->hdr.dst_addr,
1307                    &flow_tcf_mask_empty.ipv6.hdr.dst_addr,
1308                    IPV6_ADDR_LEN)) {
1309                 if (memcmp(&mask->hdr.dst_addr,
1310                            &rte_flow_item_ipv6_mask.hdr.dst_addr,
1311                            IPV6_ADDR_LEN))
1312                         return rte_flow_error_set
1313                                         (error, ENOTSUP,
1314                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1315                                          "no support for partial mask on"
1316                                          " \"ipv6.hdr.dst_addr\" field"
1317                                          " for vxlan encapsulation");
1318                 /* More IPv6 address validations can be put here. */
1319         } else {
1320                 /*
1321                  * Kernel uses the destination IP address to determine
1322                  * the routing path and obtain the MAC destination
1323                  * address (heigh or gate), so IP destination address
1324                  * must be specified within the tc rule.
1325                  */
1326                 return rte_flow_error_set(error, EINVAL,
1327                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1328                                           "outer ipv6 destination address"
1329                                           " must be specified for"
1330                                           " vxlan encapsulation");
1331         }
1332         if (memcmp(&mask->hdr.src_addr,
1333                    &flow_tcf_mask_empty.ipv6.hdr.src_addr,
1334                    IPV6_ADDR_LEN)) {
1335                 if (memcmp(&mask->hdr.src_addr,
1336                            &rte_flow_item_ipv6_mask.hdr.src_addr,
1337                            IPV6_ADDR_LEN))
1338                         return rte_flow_error_set
1339                                         (error, ENOTSUP,
1340                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1341                                          "no support for partial mask on"
1342                                          " \"ipv6.hdr.src_addr\" field"
1343                                          " for vxlan encapsulation");
1344                 /* More L3 address validation can be put here. */
1345         } else {
1346                 /*
1347                  * Kernel uses the source IP address to select the
1348                  * interface for egress encapsulated traffic, so
1349                  * it must be specified in the tc rule.
1350                  */
1351                 return rte_flow_error_set(error, EINVAL,
1352                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1353                                           "outer L3 source address"
1354                                           " must be specified for"
1355                                           " vxlan encapsulation");
1356         }
1357         return 0;
1358 }
1359
1360 /**
1361  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_UDP item for E-Switch.
1362  * The routine checks the UDP fields to be used in encapsulation header.
1363  *
1364  * @param[in] item
1365  *   Pointer to the item structure.
1366  * @param[out] error
1367  *   Pointer to the error structure.
1368  *
1369  * @return
1370  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1371  **/
1372 static int
1373 flow_tcf_validate_vxlan_encap_udp(const struct rte_flow_item *item,
1374                                   struct rte_flow_error *error)
1375 {
1376         const struct rte_flow_item_udp *spec = item->spec;
1377         const struct rte_flow_item_udp *mask = item->mask;
1378
1379         if (!spec) {
1380                 /*
1381                  * Specification for UDP ports cannot be empty
1382                  * because it is required by tunnel_key parameter.
1383                  */
1384                 return rte_flow_error_set(error, EINVAL,
1385                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1386                                           "NULL UDP port specification "
1387                                           " for vxlan encapsulation");
1388         }
1389         if (!mask)
1390                 mask = &rte_flow_item_udp_mask;
1391         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1392                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1393                         return rte_flow_error_set
1394                                         (error, ENOTSUP,
1395                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1396                                          "no support for partial mask on"
1397                                          " \"udp.hdr.dst_port\" field"
1398                                          " for vxlan encapsulation");
1399                 if (!spec->hdr.dst_port)
1400                         return rte_flow_error_set
1401                                         (error, EINVAL,
1402                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
1403                                          "outer UDP remote port cannot be"
1404                                          " 0 for vxlan encapsulation");
1405         } else {
1406                 return rte_flow_error_set(error, EINVAL,
1407                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1408                                           "outer UDP remote port"
1409                                           " must be specified for"
1410                                           " vxlan encapsulation");
1411         }
1412         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1413                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1414                         return rte_flow_error_set
1415                                         (error, ENOTSUP,
1416                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1417                                          "no support for partial mask on"
1418                                          " \"udp.hdr.src_port\" field"
1419                                          " for vxlan encapsulation");
1420                 DRV_LOG(WARNING,
1421                         "outer UDP source port cannot be"
1422                         " forced for vxlan encapsulation,"
1423                         " parameter ignored");
1424         }
1425         return 0;
1426 }
1427
1428 /**
1429  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_VXLAN item for E-Switch.
1430  * The routine checks the VNIP fields to be used in encapsulation header.
1431  *
1432  * @param[in] item
1433  *   Pointer to the item structure.
1434  * @param[out] error
1435  *   Pointer to the error structure.
1436  *
1437  * @return
1438  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1439  **/
1440 static int
1441 flow_tcf_validate_vxlan_encap_vni(const struct rte_flow_item *item,
1442                                   struct rte_flow_error *error)
1443 {
1444         const struct rte_flow_item_vxlan *spec = item->spec;
1445         const struct rte_flow_item_vxlan *mask = item->mask;
1446
1447         if (!spec) {
1448                 /* Outer VNI is required by tunnel_key parameter. */
1449                 return rte_flow_error_set(error, EINVAL,
1450                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1451                                           "NULL VNI specification"
1452                                           " for vxlan encapsulation");
1453         }
1454         if (!mask)
1455                 mask = &rte_flow_item_vxlan_mask;
1456         if (!mask->vni[0] && !mask->vni[1] && !mask->vni[2])
1457                 return rte_flow_error_set(error, EINVAL,
1458                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1459                                           "outer VNI must be specified "
1460                                           "for vxlan encapsulation");
1461         if (mask->vni[0] != 0xff ||
1462             mask->vni[1] != 0xff ||
1463             mask->vni[2] != 0xff)
1464                 return rte_flow_error_set(error, ENOTSUP,
1465                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1466                                           "no support for partial mask on"
1467                                           " \"vxlan.vni\" field");
1468
1469         if (!spec->vni[0] && !spec->vni[1] && !spec->vni[2])
1470                 return rte_flow_error_set(error, EINVAL,
1471                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1472                                           "vxlan vni cannot be 0");
1473         return 0;
1474 }
1475
1476 /**
1477  * Validate VXLAN_ENCAP action item list for E-Switch.
1478  * The routine checks items to be used in encapsulation header.
1479  *
1480  * @param[in] action
1481  *   Pointer to the VXLAN_ENCAP action structure.
1482  * @param[out] error
1483  *   Pointer to the error structure.
1484  *
1485  * @return
1486  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1487  **/
1488 static int
1489 flow_tcf_validate_vxlan_encap(const struct rte_flow_action *action,
1490                               struct rte_flow_error *error)
1491 {
1492         const struct rte_flow_item *items;
1493         int ret;
1494         uint32_t item_flags = 0;
1495
1496         if (!action->conf)
1497                 return rte_flow_error_set(error, EINVAL,
1498                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1499                                           "Missing vxlan tunnel"
1500                                           " action configuration");
1501         items = ((const struct rte_flow_action_vxlan_encap *)
1502                                         action->conf)->definition;
1503         if (!items)
1504                 return rte_flow_error_set(error, EINVAL,
1505                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1506                                           "Missing vxlan tunnel"
1507                                           " encapsulation parameters");
1508         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1509                 switch (items->type) {
1510                 case RTE_FLOW_ITEM_TYPE_VOID:
1511                         break;
1512                 case RTE_FLOW_ITEM_TYPE_ETH:
1513                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1514                                                           error);
1515                         if (ret < 0)
1516                                 return ret;
1517                         ret = flow_tcf_validate_vxlan_encap_eth(items, error);
1518                         if (ret < 0)
1519                                 return ret;
1520                         item_flags |= MLX5_FLOW_LAYER_OUTER_L2;
1521                         break;
1522                 break;
1523                 case RTE_FLOW_ITEM_TYPE_IPV4:
1524                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1525                                                            error);
1526                         if (ret < 0)
1527                                 return ret;
1528                         ret = flow_tcf_validate_vxlan_encap_ipv4(items, error);
1529                         if (ret < 0)
1530                                 return ret;
1531                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1532                         break;
1533                 case RTE_FLOW_ITEM_TYPE_IPV6:
1534                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1535                                                            error);
1536                         if (ret < 0)
1537                                 return ret;
1538                         ret = flow_tcf_validate_vxlan_encap_ipv6(items, error);
1539                         if (ret < 0)
1540                                 return ret;
1541                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1542                         break;
1543                 case RTE_FLOW_ITEM_TYPE_UDP:
1544                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1545                                                            0xFF, error);
1546                         if (ret < 0)
1547                                 return ret;
1548                         ret = flow_tcf_validate_vxlan_encap_udp(items, error);
1549                         if (ret < 0)
1550                                 return ret;
1551                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
1552                         break;
1553                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1554                         ret = mlx5_flow_validate_item_vxlan(items,
1555                                                             item_flags, error);
1556                         if (ret < 0)
1557                                 return ret;
1558                         ret = flow_tcf_validate_vxlan_encap_vni(items, error);
1559                         if (ret < 0)
1560                                 return ret;
1561                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1562                         break;
1563                 default:
1564                         return rte_flow_error_set
1565                                         (error, ENOTSUP,
1566                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1567                                          "vxlan encap item not supported");
1568                 }
1569         }
1570         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1571                 return rte_flow_error_set(error, EINVAL,
1572                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1573                                           "no outer IP layer found"
1574                                           " for vxlan encapsulation");
1575         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1576                 return rte_flow_error_set(error, EINVAL,
1577                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1578                                           "no outer UDP layer found"
1579                                           " for vxlan encapsulation");
1580         if (!(item_flags & MLX5_FLOW_LAYER_VXLAN))
1581                 return rte_flow_error_set(error, EINVAL,
1582                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1583                                           "no VXLAN VNI found"
1584                                           " for vxlan encapsulation");
1585         return 0;
1586 }
1587
1588 /**
1589  * Validate outer RTE_FLOW_ITEM_TYPE_UDP item if tunnel item
1590  * RTE_FLOW_ITEM_TYPE_VXLAN is present in item list.
1591  *
1592  * @param[in] udp
1593  *   Outer UDP layer item (if any, NULL otherwise).
1594  * @param[out] error
1595  *   Pointer to the error structure.
1596  *
1597  * @return
1598  *   0 on success, a negative errno value otherwise and rte_errno is set.
1599  **/
1600 static int
1601 flow_tcf_validate_vxlan_decap_udp(const struct rte_flow_item *udp,
1602                                   struct rte_flow_error *error)
1603 {
1604         const struct rte_flow_item_udp *spec = udp->spec;
1605         const struct rte_flow_item_udp *mask = udp->mask;
1606
1607         if (!spec)
1608                 /*
1609                  * Specification for UDP ports cannot be empty
1610                  * because it is required as decap parameter.
1611                  */
1612                 return rte_flow_error_set(error, EINVAL,
1613                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1614                                           "NULL UDP port specification"
1615                                           " for VXLAN decapsulation");
1616         if (!mask)
1617                 mask = &rte_flow_item_udp_mask;
1618         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1619                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1620                         return rte_flow_error_set
1621                                         (error, ENOTSUP,
1622                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1623                                          "no support for partial mask on"
1624                                          " \"udp.hdr.dst_port\" field");
1625                 if (!spec->hdr.dst_port)
1626                         return rte_flow_error_set
1627                                         (error, EINVAL,
1628                                          RTE_FLOW_ERROR_TYPE_ITEM, udp,
1629                                          "zero decap local UDP port");
1630         } else {
1631                 return rte_flow_error_set(error, EINVAL,
1632                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1633                                           "outer UDP destination port must be "
1634                                           "specified for vxlan decapsulation");
1635         }
1636         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1637                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1638                         return rte_flow_error_set
1639                                         (error, ENOTSUP,
1640                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1641                                          "no support for partial mask on"
1642                                          " \"udp.hdr.src_port\" field");
1643                 DRV_LOG(WARNING,
1644                         "outer UDP local port cannot be "
1645                         "forced for VXLAN encapsulation, "
1646                         "parameter ignored");
1647         }
1648         return 0;
1649 }
1650
1651 /**
1652  * Validate flow for E-Switch.
1653  *
1654  * @param[in] priv
1655  *   Pointer to the priv structure.
1656  * @param[in] attr
1657  *   Pointer to the flow attributes.
1658  * @param[in] items
1659  *   Pointer to the list of items.
1660  * @param[in] actions
1661  *   Pointer to the list of actions.
1662  * @param[out] error
1663  *   Pointer to the error structure.
1664  *
1665  * @return
1666  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1667  */
1668 static int
1669 flow_tcf_validate(struct rte_eth_dev *dev,
1670                   const struct rte_flow_attr *attr,
1671                   const struct rte_flow_item items[],
1672                   const struct rte_flow_action actions[],
1673                   struct rte_flow_error *error)
1674 {
1675         union {
1676                 const struct rte_flow_item_port_id *port_id;
1677                 const struct rte_flow_item_eth *eth;
1678                 const struct rte_flow_item_vlan *vlan;
1679                 const struct rte_flow_item_ipv4 *ipv4;
1680                 const struct rte_flow_item_ipv6 *ipv6;
1681                 const struct rte_flow_item_tcp *tcp;
1682                 const struct rte_flow_item_udp *udp;
1683                 const struct rte_flow_item_vxlan *vxlan;
1684         } spec, mask;
1685         union {
1686                 const struct rte_flow_action_port_id *port_id;
1687                 const struct rte_flow_action_jump *jump;
1688                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
1689                 const struct rte_flow_action_of_set_vlan_vid *
1690                         of_set_vlan_vid;
1691                 const struct rte_flow_action_of_set_vlan_pcp *
1692                         of_set_vlan_pcp;
1693                 const struct rte_flow_action_vxlan_encap *vxlan_encap;
1694                 const struct rte_flow_action_set_ipv4 *set_ipv4;
1695                 const struct rte_flow_action_set_ipv6 *set_ipv6;
1696         } conf;
1697         const struct rte_flow_item *outer_udp = NULL;
1698         rte_be16_t inner_etype = RTE_BE16(ETH_P_ALL);
1699         rte_be16_t outer_etype = RTE_BE16(ETH_P_ALL);
1700         rte_be16_t vlan_etype = RTE_BE16(ETH_P_ALL);
1701         uint64_t item_flags = 0;
1702         uint64_t action_flags = 0;
1703         uint8_t next_protocol = 0xff;
1704         unsigned int tcm_ifindex = 0;
1705         uint8_t pedit_validated = 0;
1706         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
1707         struct rte_eth_dev *port_id_dev = NULL;
1708         bool in_port_id_set;
1709         int ret;
1710
1711         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
1712                                                 PTOI_TABLE_SZ_MAX(dev)));
1713         ret = flow_tcf_validate_attributes(attr, error);
1714         if (ret < 0)
1715                 return ret;
1716         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1717                 unsigned int i;
1718                 uint64_t current_action_flag = 0;
1719
1720                 switch (actions->type) {
1721                 case RTE_FLOW_ACTION_TYPE_VOID:
1722                         break;
1723                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
1724                         current_action_flag = MLX5_FLOW_ACTION_PORT_ID;
1725                         if (!actions->conf)
1726                                 break;
1727                         conf.port_id = actions->conf;
1728                         if (conf.port_id->original)
1729                                 i = 0;
1730                         else
1731                                 for (i = 0; ptoi[i].ifindex; ++i)
1732                                         if (ptoi[i].port_id == conf.port_id->id)
1733                                                 break;
1734                         if (!ptoi[i].ifindex)
1735                                 return rte_flow_error_set
1736                                         (error, ENODEV,
1737                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1738                                          conf.port_id,
1739                                          "missing data to convert port ID to"
1740                                          " ifindex");
1741                         port_id_dev = &rte_eth_devices[conf.port_id->id];
1742                         break;
1743                 case RTE_FLOW_ACTION_TYPE_JUMP:
1744                         current_action_flag = MLX5_FLOW_ACTION_JUMP;
1745                         if (!actions->conf)
1746                                 break;
1747                         conf.jump = actions->conf;
1748                         if (attr->group >= conf.jump->group)
1749                                 return rte_flow_error_set
1750                                         (error, ENOTSUP,
1751                                          RTE_FLOW_ERROR_TYPE_ACTION,
1752                                          actions,
1753                                          "can jump only to a group forward");
1754                         break;
1755                 case RTE_FLOW_ACTION_TYPE_DROP:
1756                         current_action_flag = MLX5_FLOW_ACTION_DROP;
1757                         break;
1758                 case RTE_FLOW_ACTION_TYPE_COUNT:
1759                         break;
1760                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
1761                         current_action_flag = MLX5_FLOW_ACTION_OF_POP_VLAN;
1762                         break;
1763                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1764                         rte_be16_t ethertype;
1765
1766                         current_action_flag = MLX5_FLOW_ACTION_OF_PUSH_VLAN;
1767                         if (!actions->conf)
1768                                 break;
1769                         conf.of_push_vlan = actions->conf;
1770                         ethertype = conf.of_push_vlan->ethertype;
1771                         if (ethertype != RTE_BE16(ETH_P_8021Q) &&
1772                             ethertype != RTE_BE16(ETH_P_8021AD))
1773                                 return rte_flow_error_set
1774                                         (error, EINVAL,
1775                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1776                                          "vlan push TPID must be "
1777                                          "802.1Q or 802.1AD");
1778                         break;
1779                 }
1780                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
1781                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1782                                 return rte_flow_error_set
1783                                         (error, ENOTSUP,
1784                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1785                                          "vlan modify is not supported,"
1786                                          " set action must follow push action");
1787                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
1788                         break;
1789                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
1790                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1791                                 return rte_flow_error_set
1792                                         (error, ENOTSUP,
1793                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1794                                          "vlan modify is not supported,"
1795                                          " set action must follow push action");
1796                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
1797                         break;
1798                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1799                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_DECAP;
1800                         break;
1801                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1802                         ret = flow_tcf_validate_vxlan_encap(actions, error);
1803                         if (ret < 0)
1804                                 return ret;
1805                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_ENCAP;
1806                         break;
1807                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
1808                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_SRC;
1809                         break;
1810                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
1811                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_DST;
1812                         break;
1813                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
1814                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_SRC;
1815                         break;
1816                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
1817                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_DST;
1818                         break;
1819                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
1820                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_SRC;
1821                         break;
1822                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
1823                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_DST;
1824                         break;
1825                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
1826                         current_action_flag = MLX5_FLOW_ACTION_SET_TTL;
1827                         break;
1828                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
1829                         current_action_flag = MLX5_FLOW_ACTION_DEC_TTL;
1830                         break;
1831                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
1832                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_SRC;
1833                         break;
1834                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
1835                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_DST;
1836                         break;
1837                 default:
1838                         return rte_flow_error_set(error, ENOTSUP,
1839                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1840                                                   actions,
1841                                                   "action not supported");
1842                 }
1843                 if (current_action_flag & MLX5_TCF_CONFIG_ACTIONS) {
1844                         if (!actions->conf)
1845                                 return rte_flow_error_set
1846                                         (error, EINVAL,
1847                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1848                                          actions,
1849                                          "action configuration not set");
1850                 }
1851                 if ((current_action_flag & MLX5_TCF_PEDIT_ACTIONS) &&
1852                     pedit_validated)
1853                         return rte_flow_error_set(error, ENOTSUP,
1854                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1855                                                   actions,
1856                                                   "set actions should be "
1857                                                   "listed successively");
1858                 if ((current_action_flag & ~MLX5_TCF_PEDIT_ACTIONS) &&
1859                     (action_flags & MLX5_TCF_PEDIT_ACTIONS))
1860                         pedit_validated = 1;
1861                 if ((current_action_flag & MLX5_TCF_FATE_ACTIONS) &&
1862                     (action_flags & MLX5_TCF_FATE_ACTIONS))
1863                         return rte_flow_error_set(error, EINVAL,
1864                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1865                                                   actions,
1866                                                   "can't have multiple fate"
1867                                                   " actions");
1868                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1869                     (action_flags & MLX5_TCF_VXLAN_ACTIONS))
1870                         return rte_flow_error_set(error, EINVAL,
1871                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1872                                                   actions,
1873                                                   "can't have multiple vxlan"
1874                                                   " actions");
1875                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1876                     (action_flags & MLX5_TCF_VLAN_ACTIONS))
1877                         return rte_flow_error_set(error, ENOTSUP,
1878                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1879                                                   actions,
1880                                                   "can't have vxlan and vlan"
1881                                                   " actions in the same rule");
1882                 action_flags |= current_action_flag;
1883         }
1884         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1885                 unsigned int i;
1886
1887                 switch (items->type) {
1888                 case RTE_FLOW_ITEM_TYPE_VOID:
1889                         break;
1890                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
1891                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1892                                 return rte_flow_error_set
1893                                         (error, ENOTSUP,
1894                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1895                                          "inner tunnel port id"
1896                                          " item is not supported");
1897                         mask.port_id = flow_tcf_item_mask
1898                                 (items, &rte_flow_item_port_id_mask,
1899                                  &flow_tcf_mask_supported.port_id,
1900                                  &flow_tcf_mask_empty.port_id,
1901                                  sizeof(flow_tcf_mask_supported.port_id),
1902                                  error);
1903                         if (!mask.port_id)
1904                                 return -rte_errno;
1905                         if (mask.port_id == &flow_tcf_mask_empty.port_id) {
1906                                 in_port_id_set = 1;
1907                                 break;
1908                         }
1909                         spec.port_id = items->spec;
1910                         if (mask.port_id->id && mask.port_id->id != 0xffffffff)
1911                                 return rte_flow_error_set
1912                                         (error, ENOTSUP,
1913                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1914                                          mask.port_id,
1915                                          "no support for partial mask on"
1916                                          " \"id\" field");
1917                         if (!mask.port_id->id)
1918                                 i = 0;
1919                         else
1920                                 for (i = 0; ptoi[i].ifindex; ++i)
1921                                         if (ptoi[i].port_id == spec.port_id->id)
1922                                                 break;
1923                         if (!ptoi[i].ifindex)
1924                                 return rte_flow_error_set
1925                                         (error, ENODEV,
1926                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1927                                          spec.port_id,
1928                                          "missing data to convert port ID to"
1929                                          " ifindex");
1930                         if (in_port_id_set && ptoi[i].ifindex != tcm_ifindex)
1931                                 return rte_flow_error_set
1932                                         (error, ENOTSUP,
1933                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1934                                          spec.port_id,
1935                                          "cannot match traffic for"
1936                                          " several port IDs through"
1937                                          " a single flow rule");
1938                         tcm_ifindex = ptoi[i].ifindex;
1939                         in_port_id_set = 1;
1940                         break;
1941                 case RTE_FLOW_ITEM_TYPE_ETH:
1942                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1943                                                           error);
1944                         if (ret < 0)
1945                                 return ret;
1946                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
1947                                       MLX5_FLOW_LAYER_INNER_L2 :
1948                                       MLX5_FLOW_LAYER_OUTER_L2;
1949                         /* TODO:
1950                          * Redundant check due to different supported mask.
1951                          * Same for the rest of items.
1952                          */
1953                         mask.eth = flow_tcf_item_mask
1954                                 (items, &rte_flow_item_eth_mask,
1955                                  &flow_tcf_mask_supported.eth,
1956                                  &flow_tcf_mask_empty.eth,
1957                                  sizeof(flow_tcf_mask_supported.eth),
1958                                  error);
1959                         if (!mask.eth)
1960                                 return -rte_errno;
1961                         if (mask.eth->type && mask.eth->type !=
1962                             RTE_BE16(0xffff))
1963                                 return rte_flow_error_set
1964                                         (error, ENOTSUP,
1965                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1966                                          mask.eth,
1967                                          "no support for partial mask on"
1968                                          " \"type\" field");
1969                         assert(items->spec);
1970                         spec.eth = items->spec;
1971                         if (mask.eth->type &&
1972                             (item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
1973                             inner_etype != RTE_BE16(ETH_P_ALL) &&
1974                             inner_etype != spec.eth->type)
1975                                 return rte_flow_error_set
1976                                         (error, EINVAL,
1977                                          RTE_FLOW_ERROR_TYPE_ITEM,
1978                                          items,
1979                                          "inner eth_type conflict");
1980                         if (mask.eth->type &&
1981                             !(item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
1982                             outer_etype != RTE_BE16(ETH_P_ALL) &&
1983                             outer_etype != spec.eth->type)
1984                                 return rte_flow_error_set
1985                                         (error, EINVAL,
1986                                          RTE_FLOW_ERROR_TYPE_ITEM,
1987                                          items,
1988                                          "outer eth_type conflict");
1989                         if (mask.eth->type) {
1990                                 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1991                                         inner_etype = spec.eth->type;
1992                                 else
1993                                         outer_etype = spec.eth->type;
1994                         }
1995                         break;
1996                 case RTE_FLOW_ITEM_TYPE_VLAN:
1997                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1998                                 return rte_flow_error_set
1999                                         (error, ENOTSUP,
2000                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
2001                                          "inner tunnel VLAN"
2002                                          " is not supported");
2003                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
2004                                                            error);
2005                         if (ret < 0)
2006                                 return ret;
2007                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
2008                         mask.vlan = flow_tcf_item_mask
2009                                 (items, &rte_flow_item_vlan_mask,
2010                                  &flow_tcf_mask_supported.vlan,
2011                                  &flow_tcf_mask_empty.vlan,
2012                                  sizeof(flow_tcf_mask_supported.vlan),
2013                                  error);
2014                         if (!mask.vlan)
2015                                 return -rte_errno;
2016                         if ((mask.vlan->tci & RTE_BE16(0xe000) &&
2017                              (mask.vlan->tci & RTE_BE16(0xe000)) !=
2018                               RTE_BE16(0xe000)) ||
2019                             (mask.vlan->tci & RTE_BE16(0x0fff) &&
2020                              (mask.vlan->tci & RTE_BE16(0x0fff)) !=
2021                               RTE_BE16(0x0fff)) ||
2022                             (mask.vlan->inner_type &&
2023                              mask.vlan->inner_type != RTE_BE16(0xffff)))
2024                                 return rte_flow_error_set
2025                                         (error, ENOTSUP,
2026                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2027                                          mask.vlan,
2028                                          "no support for partial masks on"
2029                                          " \"tci\" (PCP and VID parts) and"
2030                                          " \"inner_type\" fields");
2031                         if (outer_etype != RTE_BE16(ETH_P_ALL) &&
2032                             outer_etype != RTE_BE16(ETH_P_8021Q))
2033                                 return rte_flow_error_set
2034                                         (error, EINVAL,
2035                                          RTE_FLOW_ERROR_TYPE_ITEM,
2036                                          items,
2037                                          "outer eth_type conflict,"
2038                                          " must be 802.1Q");
2039                         outer_etype = RTE_BE16(ETH_P_8021Q);
2040                         assert(items->spec);
2041                         spec.vlan = items->spec;
2042                         if (mask.vlan->inner_type &&
2043                             vlan_etype != RTE_BE16(ETH_P_ALL) &&
2044                             vlan_etype != spec.vlan->inner_type)
2045                                 return rte_flow_error_set
2046                                         (error, EINVAL,
2047                                          RTE_FLOW_ERROR_TYPE_ITEM,
2048                                          items,
2049                                          "vlan eth_type conflict");
2050                         if (mask.vlan->inner_type)
2051                                 vlan_etype = spec.vlan->inner_type;
2052                         break;
2053                 case RTE_FLOW_ITEM_TYPE_IPV4:
2054                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2055                                                            error);
2056                         if (ret < 0)
2057                                 return ret;
2058                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2059                                       MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2060                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2061                         mask.ipv4 = flow_tcf_item_mask
2062                                 (items, &rte_flow_item_ipv4_mask,
2063                                  &flow_tcf_mask_supported.ipv4,
2064                                  &flow_tcf_mask_empty.ipv4,
2065                                  sizeof(flow_tcf_mask_supported.ipv4),
2066                                  error);
2067                         if (!mask.ipv4)
2068                                 return -rte_errno;
2069                         if (mask.ipv4->hdr.next_proto_id &&
2070                             mask.ipv4->hdr.next_proto_id != 0xff)
2071                                 return rte_flow_error_set
2072                                         (error, ENOTSUP,
2073                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2074                                          mask.ipv4,
2075                                          "no support for partial mask on"
2076                                          " \"hdr.next_proto_id\" field");
2077                         else if (mask.ipv4->hdr.next_proto_id)
2078                                 next_protocol =
2079                                         ((const struct rte_flow_item_ipv4 *)
2080                                          (items->spec))->hdr.next_proto_id;
2081                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2082                                 if (inner_etype != RTE_BE16(ETH_P_ALL) &&
2083                                     inner_etype != RTE_BE16(ETH_P_IP))
2084                                         return rte_flow_error_set
2085                                                 (error, EINVAL,
2086                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2087                                                  items,
2088                                                  "inner eth_type conflict,"
2089                                                  " IPv4 is required");
2090                                 inner_etype = RTE_BE16(ETH_P_IP);
2091                         } else if (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN) {
2092                                 if (vlan_etype != RTE_BE16(ETH_P_ALL) &&
2093                                     vlan_etype != RTE_BE16(ETH_P_IP))
2094                                         return rte_flow_error_set
2095                                                 (error, EINVAL,
2096                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2097                                                  items,
2098                                                  "vlan eth_type conflict,"
2099                                                  " IPv4 is required");
2100                                 vlan_etype = RTE_BE16(ETH_P_IP);
2101                         } else {
2102                                 if (outer_etype != RTE_BE16(ETH_P_ALL) &&
2103                                     outer_etype != RTE_BE16(ETH_P_IP))
2104                                         return rte_flow_error_set
2105                                                 (error, EINVAL,
2106                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2107                                                  items,
2108                                                  "eth_type conflict,"
2109                                                  " IPv4 is required");
2110                                 outer_etype = RTE_BE16(ETH_P_IP);
2111                         }
2112                         break;
2113                 case RTE_FLOW_ITEM_TYPE_IPV6:
2114                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2115                                                            error);
2116                         if (ret < 0)
2117                                 return ret;
2118                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2119                                       MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2120                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2121                         mask.ipv6 = flow_tcf_item_mask
2122                                 (items, &rte_flow_item_ipv6_mask,
2123                                  &flow_tcf_mask_supported.ipv6,
2124                                  &flow_tcf_mask_empty.ipv6,
2125                                  sizeof(flow_tcf_mask_supported.ipv6),
2126                                  error);
2127                         if (!mask.ipv6)
2128                                 return -rte_errno;
2129                         if (mask.ipv6->hdr.proto &&
2130                             mask.ipv6->hdr.proto != 0xff)
2131                                 return rte_flow_error_set
2132                                         (error, ENOTSUP,
2133                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2134                                          mask.ipv6,
2135                                          "no support for partial mask on"
2136                                          " \"hdr.proto\" field");
2137                         else if (mask.ipv6->hdr.proto)
2138                                 next_protocol =
2139                                         ((const struct rte_flow_item_ipv6 *)
2140                                          (items->spec))->hdr.proto;
2141                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2142                                 if (inner_etype != RTE_BE16(ETH_P_ALL) &&
2143                                     inner_etype != RTE_BE16(ETH_P_IPV6))
2144                                         return rte_flow_error_set
2145                                                 (error, EINVAL,
2146                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2147                                                  items,
2148                                                  "inner eth_type conflict,"
2149                                                  " IPv6 is required");
2150                                 inner_etype = RTE_BE16(ETH_P_IPV6);
2151                         } else if (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN) {
2152                                 if (vlan_etype != RTE_BE16(ETH_P_ALL) &&
2153                                     vlan_etype != RTE_BE16(ETH_P_IPV6))
2154                                         return rte_flow_error_set
2155                                                 (error, EINVAL,
2156                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2157                                                  items,
2158                                                  "vlan eth_type conflict,"
2159                                                  " IPv6 is required");
2160                                 vlan_etype = RTE_BE16(ETH_P_IPV6);
2161                         } else {
2162                                 if (outer_etype != RTE_BE16(ETH_P_ALL) &&
2163                                     outer_etype != RTE_BE16(ETH_P_IPV6))
2164                                         return rte_flow_error_set
2165                                                 (error, EINVAL,
2166                                                  RTE_FLOW_ERROR_TYPE_ITEM,
2167                                                  items,
2168                                                  "eth_type conflict,"
2169                                                  " IPv6 is required");
2170                                 outer_etype = RTE_BE16(ETH_P_IPV6);
2171                         }
2172                         break;
2173                 case RTE_FLOW_ITEM_TYPE_UDP:
2174                         ret = mlx5_flow_validate_item_udp(items, item_flags,
2175                                                           next_protocol, error);
2176                         if (ret < 0)
2177                                 return ret;
2178                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2179                                       MLX5_FLOW_LAYER_INNER_L4_UDP :
2180                                       MLX5_FLOW_LAYER_OUTER_L4_UDP;
2181                         mask.udp = flow_tcf_item_mask
2182                                 (items, &rte_flow_item_udp_mask,
2183                                  &flow_tcf_mask_supported.udp,
2184                                  &flow_tcf_mask_empty.udp,
2185                                  sizeof(flow_tcf_mask_supported.udp),
2186                                  error);
2187                         if (!mask.udp)
2188                                 return -rte_errno;
2189                         /*
2190                          * Save the presumed outer UDP item for extra check
2191                          * if the tunnel item will be found later in the list.
2192                          */
2193                         if (!(item_flags & MLX5_FLOW_LAYER_TUNNEL))
2194                                 outer_udp = items;
2195                         break;
2196                 case RTE_FLOW_ITEM_TYPE_TCP:
2197                         ret = mlx5_flow_validate_item_tcp
2198                                              (items, item_flags,
2199                                               next_protocol,
2200                                               &flow_tcf_mask_supported.tcp,
2201                                               error);
2202                         if (ret < 0)
2203                                 return ret;
2204                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2205                                       MLX5_FLOW_LAYER_INNER_L4_TCP :
2206                                       MLX5_FLOW_LAYER_OUTER_L4_TCP;
2207                         mask.tcp = flow_tcf_item_mask
2208                                 (items, &rte_flow_item_tcp_mask,
2209                                  &flow_tcf_mask_supported.tcp,
2210                                  &flow_tcf_mask_empty.tcp,
2211                                  sizeof(flow_tcf_mask_supported.tcp),
2212                                  error);
2213                         if (!mask.tcp)
2214                                 return -rte_errno;
2215                         break;
2216                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2217                         if (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)
2218                                 return rte_flow_error_set
2219                                         (error, ENOTSUP,
2220                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
2221                                          "vxlan tunnel over vlan"
2222                                          " is not supported");
2223                         ret = mlx5_flow_validate_item_vxlan(items,
2224                                                             item_flags, error);
2225                         if (ret < 0)
2226                                 return ret;
2227                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
2228                         mask.vxlan = flow_tcf_item_mask
2229                                 (items, &rte_flow_item_vxlan_mask,
2230                                  &flow_tcf_mask_supported.vxlan,
2231                                  &flow_tcf_mask_empty.vxlan,
2232                                  sizeof(flow_tcf_mask_supported.vxlan), error);
2233                         if (!mask.vxlan)
2234                                 return -rte_errno;
2235                         if (mask.vxlan->vni[0] != 0xff ||
2236                             mask.vxlan->vni[1] != 0xff ||
2237                             mask.vxlan->vni[2] != 0xff)
2238                                 return rte_flow_error_set
2239                                         (error, ENOTSUP,
2240                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2241                                          mask.vxlan,
2242                                          "no support for partial or "
2243                                          "empty mask on \"vxlan.vni\" field");
2244                         /*
2245                          * The VNI item assumes the VXLAN tunnel, it requires
2246                          * at least the outer destination UDP port must be
2247                          * specified without wildcards to allow kernel select
2248                          * the virtual VXLAN device by port. Also outer IPv4
2249                          * or IPv6 item must be specified (wilcards or even
2250                          * zero mask are allowed) to let driver know the tunnel
2251                          * IP version and process UDP traffic correctly.
2252                          */
2253                         if (!(item_flags &
2254                              (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2255                               MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2256                                 return rte_flow_error_set
2257                                                  (error, EINVAL,
2258                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2259                                                   NULL,
2260                                                   "no outer IP pattern found"
2261                                                   " for vxlan tunnel");
2262                         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2263                                 return rte_flow_error_set
2264                                                  (error, EINVAL,
2265                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2266                                                   NULL,
2267                                                   "no outer UDP pattern found"
2268                                                   " for vxlan tunnel");
2269                         /*
2270                          * All items preceding the tunnel item become outer
2271                          * ones and we should do extra validation for them
2272                          * due to tc limitations for tunnel outer parameters.
2273                          * Currently only outer UDP item requres extra check,
2274                          * use the saved pointer instead of item list rescan.
2275                          */
2276                         assert(outer_udp);
2277                         ret = flow_tcf_validate_vxlan_decap_udp
2278                                                 (outer_udp, error);
2279                         if (ret < 0)
2280                                 return ret;
2281                         /* Reset L4 protocol for inner parameters. */
2282                         next_protocol = 0xff;
2283                         break;
2284                 default:
2285                         return rte_flow_error_set(error, ENOTSUP,
2286                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2287                                                   items, "item not supported");
2288                 }
2289         }
2290         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2291             (action_flags & MLX5_FLOW_ACTION_DROP))
2292                 return rte_flow_error_set(error, ENOTSUP,
2293                                           RTE_FLOW_ERROR_TYPE_ACTION,
2294                                           actions,
2295                                           "set action is not compatible with "
2296                                           "drop action");
2297         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2298             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2299                 return rte_flow_error_set(error, ENOTSUP,
2300                                           RTE_FLOW_ERROR_TYPE_ACTION,
2301                                           actions,
2302                                           "set action must be followed by "
2303                                           "port_id action");
2304         if (action_flags &
2305            (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST)) {
2306                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4))
2307                         return rte_flow_error_set(error, EINVAL,
2308                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2309                                                   actions,
2310                                                   "no ipv4 item found in"
2311                                                   " pattern");
2312         }
2313         if (action_flags &
2314            (MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST)) {
2315                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6))
2316                         return rte_flow_error_set(error, EINVAL,
2317                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2318                                                   actions,
2319                                                   "no ipv6 item found in"
2320                                                   " pattern");
2321         }
2322         if (action_flags &
2323            (MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST)) {
2324                 if (!(item_flags &
2325                      (MLX5_FLOW_LAYER_OUTER_L4_UDP |
2326                       MLX5_FLOW_LAYER_OUTER_L4_TCP)))
2327                         return rte_flow_error_set(error, EINVAL,
2328                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2329                                                   actions,
2330                                                   "no TCP/UDP item found in"
2331                                                   " pattern");
2332         }
2333         /*
2334          * FW syndrome (0xA9C090):
2335          *     set_flow_table_entry: push vlan action fte in fdb can ONLY be
2336          *     forward to the uplink.
2337          */
2338         if ((action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2339             (action_flags & MLX5_FLOW_ACTION_PORT_ID) &&
2340             ((struct priv *)port_id_dev->data->dev_private)->representor)
2341                 return rte_flow_error_set(error, ENOTSUP,
2342                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2343                                           "vlan push can only be applied"
2344                                           " when forwarding to uplink port");
2345         /*
2346          * FW syndrome (0x294609):
2347          *     set_flow_table_entry: modify/pop/push actions in fdb flow table
2348          *     are supported only while forwarding to vport.
2349          */
2350         if ((action_flags & MLX5_TCF_VLAN_ACTIONS) &&
2351             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2352                 return rte_flow_error_set(error, ENOTSUP,
2353                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2354                                           "vlan actions are supported"
2355                                           " only with port_id action");
2356         if ((action_flags & MLX5_TCF_VXLAN_ACTIONS) &&
2357             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2358                 return rte_flow_error_set(error, ENOTSUP,
2359                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2360                                           "vxlan actions are supported"
2361                                           " only with port_id action");
2362         if (!(action_flags & MLX5_TCF_FATE_ACTIONS))
2363                 return rte_flow_error_set(error, EINVAL,
2364                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2365                                           "no fate action is found");
2366         if (action_flags &
2367            (MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL)) {
2368                 if (!(item_flags &
2369                      (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2370                       MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2371                         return rte_flow_error_set(error, EINVAL,
2372                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2373                                                   actions,
2374                                                   "no IP found in pattern");
2375         }
2376         if (action_flags &
2377             (MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)) {
2378                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L2))
2379                         return rte_flow_error_set(error, ENOTSUP,
2380                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2381                                                   actions,
2382                                                   "no ethernet found in"
2383                                                   " pattern");
2384         }
2385         if ((action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) &&
2386             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
2387                 return rte_flow_error_set(error, EINVAL,
2388                                           RTE_FLOW_ERROR_TYPE_ACTION,
2389                                           NULL,
2390                                           "no VNI pattern found"
2391                                           " for vxlan decap action");
2392         if ((action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) &&
2393             (item_flags & MLX5_FLOW_LAYER_TUNNEL))
2394                 return rte_flow_error_set(error, EINVAL,
2395                                           RTE_FLOW_ERROR_TYPE_ACTION,
2396                                           NULL,
2397                                           "vxlan encap not supported"
2398                                           " for tunneled traffic");
2399         return 0;
2400 }
2401
2402 /**
2403  * Calculate maximum size of memory for flow items of Linux TC flower.
2404  *
2405  * @param[in] attr
2406  *   Pointer to the flow attributes.
2407  * @param[in] items
2408  *   Pointer to the list of items.
2409  * @param[out] action_flags
2410  *   Pointer to the detected actions.
2411  *
2412  * @return
2413  *   Maximum size of memory for items.
2414  */
2415 static int
2416 flow_tcf_get_items_size(const struct rte_flow_attr *attr,
2417                         const struct rte_flow_item items[],
2418                         uint64_t *action_flags)
2419 {
2420         int size = 0;
2421
2422         size += SZ_NLATTR_STRZ_OF("flower") +
2423                 SZ_NLATTR_NEST + /* TCA_OPTIONS. */
2424                 SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CLS_FLAGS_SKIP_SW. */
2425         if (attr->group > 0)
2426                 size += SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CHAIN. */
2427         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2428                 switch (items->type) {
2429                 case RTE_FLOW_ITEM_TYPE_VOID:
2430                         break;
2431                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2432                         break;
2433                 case RTE_FLOW_ITEM_TYPE_ETH:
2434                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2435                                 SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) * 4;
2436                                 /* dst/src MAC addr and mask. */
2437                         break;
2438                 case RTE_FLOW_ITEM_TYPE_VLAN:
2439                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2440                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2441                                 /* VLAN Ether type. */
2442                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* VLAN prio. */
2443                                 SZ_NLATTR_TYPE_OF(uint16_t); /* VLAN ID. */
2444                         break;
2445                 case RTE_FLOW_ITEM_TYPE_IPV4:
2446                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2447                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2448                                 SZ_NLATTR_TYPE_OF(uint32_t) * 4;
2449                                 /* dst/src IP addr and mask. */
2450                         break;
2451                 case RTE_FLOW_ITEM_TYPE_IPV6:
2452                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2453                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2454                                 SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 4;
2455                                 /* dst/src IP addr and mask. */
2456                         break;
2457                 case RTE_FLOW_ITEM_TYPE_UDP:
2458                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2459                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2460                                 /* dst/src port and mask. */
2461                         break;
2462                 case RTE_FLOW_ITEM_TYPE_TCP:
2463                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2464                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2465                                 /* dst/src port and mask. */
2466                         break;
2467                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2468                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2469                         /*
2470                          * There might be no VXLAN decap action in the action
2471                          * list, nonetheless the VXLAN tunnel flow requires
2472                          * the decap structure to be correctly applied to
2473                          * VXLAN device, set the flag to create the structure.
2474                          * Translation routine will not put the decap action
2475                          * in tne Netlink message if there is no actual action
2476                          * in the list.
2477                          */
2478                         *action_flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2479                         break;
2480                 default:
2481                         DRV_LOG(WARNING,
2482                                 "unsupported item %p type %d,"
2483                                 " items must be validated before flow creation",
2484                                 (const void *)items, items->type);
2485                         break;
2486                 }
2487         }
2488         return size;
2489 }
2490
2491 /**
2492  * Calculate size of memory to store the VXLAN encapsultion
2493  * related items in the Netlink message buffer. Items list
2494  * is specified by RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action.
2495  * The item list should be validated.
2496  *
2497  * @param[in] action
2498  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2499  *   List of pattern items to scan data from.
2500  *
2501  * @return
2502  *   The size the part of Netlink message buffer to store the
2503  *   VXLAN encapsulation item attributes.
2504  */
2505 static int
2506 flow_tcf_vxlan_encap_size(const struct rte_flow_action *action)
2507 {
2508         const struct rte_flow_item *items;
2509         int size = 0;
2510
2511         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2512         assert(action->conf);
2513
2514         items = ((const struct rte_flow_action_vxlan_encap *)
2515                                         action->conf)->definition;
2516         assert(items);
2517         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2518                 switch (items->type) {
2519                 case RTE_FLOW_ITEM_TYPE_VOID:
2520                         break;
2521                 case RTE_FLOW_ITEM_TYPE_ETH:
2522                         /* This item does not require message buffer. */
2523                         break;
2524                 case RTE_FLOW_ITEM_TYPE_IPV4:
2525                         size += SZ_NLATTR_DATA_OF(IPV4_ADDR_LEN) * 2;
2526                         break;
2527                 case RTE_FLOW_ITEM_TYPE_IPV6:
2528                         size += SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 2;
2529                         break;
2530                 case RTE_FLOW_ITEM_TYPE_UDP: {
2531                         const struct rte_flow_item_udp *udp = items->mask;
2532
2533                         size += SZ_NLATTR_TYPE_OF(uint16_t);
2534                         if (!udp || udp->hdr.src_port != RTE_BE16(0x0000))
2535                                 size += SZ_NLATTR_TYPE_OF(uint16_t);
2536                         break;
2537                 }
2538                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2539                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2540                         break;
2541                 default:
2542                         assert(false);
2543                         DRV_LOG(WARNING,
2544                                 "unsupported item %p type %d,"
2545                                 " items must be validated"
2546                                 " before flow creation",
2547                                 (const void *)items, items->type);
2548                         return 0;
2549                 }
2550         }
2551         return size;
2552 }
2553
2554 /**
2555  * Calculate maximum size of memory for flow actions of Linux TC flower and
2556  * extract specified actions.
2557  *
2558  * @param[in] actions
2559  *   Pointer to the list of actions.
2560  * @param[out] action_flags
2561  *   Pointer to the detected actions.
2562  *
2563  * @return
2564  *   Maximum size of memory for actions.
2565  */
2566 static int
2567 flow_tcf_get_actions_and_size(const struct rte_flow_action actions[],
2568                               uint64_t *action_flags)
2569 {
2570         int size = 0;
2571         uint64_t flags = 0;
2572
2573         size += SZ_NLATTR_NEST; /* TCA_FLOWER_ACT. */
2574         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2575                 switch (actions->type) {
2576                 case RTE_FLOW_ACTION_TYPE_VOID:
2577                         break;
2578                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
2579                         size += SZ_NLATTR_NEST + /* na_act_index. */
2580                                 SZ_NLATTR_STRZ_OF("mirred") +
2581                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2582                                 SZ_NLATTR_TYPE_OF(struct tc_mirred);
2583                         flags |= MLX5_FLOW_ACTION_PORT_ID;
2584                         break;
2585                 case RTE_FLOW_ACTION_TYPE_JUMP:
2586                         size += SZ_NLATTR_NEST + /* na_act_index. */
2587                                 SZ_NLATTR_STRZ_OF("gact") +
2588                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2589                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2590                         flags |= MLX5_FLOW_ACTION_JUMP;
2591                         break;
2592                 case RTE_FLOW_ACTION_TYPE_DROP:
2593                         size += SZ_NLATTR_NEST + /* na_act_index. */
2594                                 SZ_NLATTR_STRZ_OF("gact") +
2595                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2596                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2597                         flags |= MLX5_FLOW_ACTION_DROP;
2598                         break;
2599                 case RTE_FLOW_ACTION_TYPE_COUNT:
2600                         break;
2601                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
2602                         flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
2603                         goto action_of_vlan;
2604                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
2605                         flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
2606                         goto action_of_vlan;
2607                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
2608                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
2609                         goto action_of_vlan;
2610                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
2611                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
2612                         goto action_of_vlan;
2613 action_of_vlan:
2614                         size += SZ_NLATTR_NEST + /* na_act_index. */
2615                                 SZ_NLATTR_STRZ_OF("vlan") +
2616                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2617                                 SZ_NLATTR_TYPE_OF(struct tc_vlan) +
2618                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2619                                 /* VLAN protocol. */
2620                                 SZ_NLATTR_TYPE_OF(uint16_t) + /* VLAN ID. */
2621                                 SZ_NLATTR_TYPE_OF(uint8_t); /* VLAN prio. */
2622                         break;
2623                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2624                         size += SZ_NLATTR_NEST + /* na_act_index. */
2625                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2626                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2627                                 SZ_NLATTR_TYPE_OF(uint8_t);
2628                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2629                         size += flow_tcf_vxlan_encap_size(actions) +
2630                                 RTE_ALIGN_CEIL /* preceding encap params. */
2631                                 (sizeof(struct flow_tcf_vxlan_encap),
2632                                 MNL_ALIGNTO);
2633                         flags |= MLX5_FLOW_ACTION_VXLAN_ENCAP;
2634                         break;
2635                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
2636                         size += SZ_NLATTR_NEST + /* na_act_index. */
2637                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2638                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2639                                 SZ_NLATTR_TYPE_OF(uint8_t);
2640                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2641                         size += RTE_ALIGN_CEIL /* preceding decap params. */
2642                                 (sizeof(struct flow_tcf_vxlan_decap),
2643                                 MNL_ALIGNTO);
2644                         flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2645                         break;
2646                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
2647                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
2648                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
2649                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
2650                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
2651                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
2652                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
2653                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
2654                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
2655                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
2656                         size += flow_tcf_get_pedit_actions_size(&actions,
2657                                                                 &flags);
2658                         break;
2659                 default:
2660                         DRV_LOG(WARNING,
2661                                 "unsupported action %p type %d,"
2662                                 " items must be validated before flow creation",
2663                                 (const void *)actions, actions->type);
2664                         break;
2665                 }
2666         }
2667         *action_flags = flags;
2668         return size;
2669 }
2670
2671 /**
2672  * Brand rtnetlink buffer with unique handle.
2673  *
2674  * This handle should be unique for a given network interface to avoid
2675  * collisions.
2676  *
2677  * @param nlh
2678  *   Pointer to Netlink message.
2679  * @param handle
2680  *   Unique 32-bit handle to use.
2681  */
2682 static void
2683 flow_tcf_nl_brand(struct nlmsghdr *nlh, uint32_t handle)
2684 {
2685         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
2686
2687         tcm->tcm_handle = handle;
2688         DRV_LOG(DEBUG, "Netlink msg %p is branded with handle %x",
2689                 (void *)nlh, handle);
2690 }
2691
2692 /**
2693  * Prepare a flow object for Linux TC flower. It calculates the maximum size of
2694  * memory required, allocates the memory, initializes Netlink message headers
2695  * and set unique TC message handle.
2696  *
2697  * @param[in] attr
2698  *   Pointer to the flow attributes.
2699  * @param[in] items
2700  *   Pointer to the list of items.
2701  * @param[in] actions
2702  *   Pointer to the list of actions.
2703  * @param[out] error
2704  *   Pointer to the error structure.
2705  *
2706  * @return
2707  *   Pointer to mlx5_flow object on success,
2708  *   otherwise NULL and rte_ernno is set.
2709  */
2710 static struct mlx5_flow *
2711 flow_tcf_prepare(const struct rte_flow_attr *attr,
2712                  const struct rte_flow_item items[],
2713                  const struct rte_flow_action actions[],
2714                  struct rte_flow_error *error)
2715 {
2716         size_t size = RTE_ALIGN_CEIL
2717                         (sizeof(struct mlx5_flow),
2718                          alignof(struct flow_tcf_tunnel_hdr)) +
2719                       MNL_ALIGN(sizeof(struct nlmsghdr)) +
2720                       MNL_ALIGN(sizeof(struct tcmsg));
2721         struct mlx5_flow *dev_flow;
2722         uint64_t action_flags = 0;
2723         struct nlmsghdr *nlh;
2724         struct tcmsg *tcm;
2725         uint8_t *sp, *tun = NULL;
2726
2727         size += flow_tcf_get_items_size(attr, items, &action_flags);
2728         size += flow_tcf_get_actions_and_size(actions, &action_flags);
2729         dev_flow = rte_zmalloc(__func__, size, MNL_ALIGNTO);
2730         if (!dev_flow) {
2731                 rte_flow_error_set(error, ENOMEM,
2732                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2733                                    "not enough memory to create E-Switch flow");
2734                 return NULL;
2735         }
2736         sp = (uint8_t *)(dev_flow + 1);
2737         if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) {
2738                 sp = RTE_PTR_ALIGN
2739                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2740                 tun = sp;
2741                 sp += RTE_ALIGN_CEIL
2742                         (sizeof(struct flow_tcf_vxlan_encap),
2743                         MNL_ALIGNTO);
2744 #ifndef NDEBUG
2745                 size -= RTE_ALIGN_CEIL
2746                         (sizeof(struct flow_tcf_vxlan_encap),
2747                         MNL_ALIGNTO);
2748 #endif
2749         } else if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2750                 sp = RTE_PTR_ALIGN
2751                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2752                 tun = sp;
2753                 sp += RTE_ALIGN_CEIL
2754                         (sizeof(struct flow_tcf_vxlan_decap),
2755                         MNL_ALIGNTO);
2756 #ifndef NDEBUG
2757                 size -= RTE_ALIGN_CEIL
2758                         (sizeof(struct flow_tcf_vxlan_decap),
2759                         MNL_ALIGNTO);
2760 #endif
2761         } else {
2762                 sp = RTE_PTR_ALIGN(sp, MNL_ALIGNTO);
2763         }
2764         nlh = mnl_nlmsg_put_header(sp);
2765         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
2766         *dev_flow = (struct mlx5_flow){
2767                 .tcf = (struct mlx5_flow_tcf){
2768 #ifndef NDEBUG
2769                         .nlsize = size - RTE_ALIGN_CEIL
2770                                 (sizeof(struct mlx5_flow),
2771                                  alignof(struct flow_tcf_tunnel_hdr)),
2772 #endif
2773                         .tunnel = (struct flow_tcf_tunnel_hdr *)tun,
2774                         .nlh = nlh,
2775                         .tcm = tcm,
2776                 },
2777         };
2778         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP)
2779                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_DECAP;
2780         else if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP)
2781                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_ENCAP;
2782         /*
2783          * Generate a reasonably unique handle based on the address of the
2784          * target buffer.
2785          *
2786          * This is straightforward on 32-bit systems where the flow pointer can
2787          * be used directly. Otherwise, its least significant part is taken
2788          * after shifting it by the previous power of two of the pointed buffer
2789          * size.
2790          */
2791         if (sizeof(dev_flow) <= 4)
2792                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow);
2793         else
2794                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow >>
2795                                        rte_log2_u32(rte_align32prevpow2(size)));
2796         return dev_flow;
2797 }
2798
2799 /**
2800  * Make adjustments for supporting count actions.
2801  *
2802  * @param[in] dev
2803  *   Pointer to the Ethernet device structure.
2804  * @param[in] dev_flow
2805  *   Pointer to mlx5_flow.
2806  * @param[out] error
2807  *   Pointer to error structure.
2808  *
2809  * @return
2810  *   0 On success else a negative errno value is returned and rte_errno is set.
2811  */
2812 static int
2813 flow_tcf_translate_action_count(struct rte_eth_dev *dev __rte_unused,
2814                                   struct mlx5_flow *dev_flow,
2815                                   struct rte_flow_error *error)
2816 {
2817         struct rte_flow *flow = dev_flow->flow;
2818
2819         if (!flow->counter) {
2820                 flow->counter = flow_tcf_counter_new();
2821                 if (!flow->counter)
2822                         return rte_flow_error_set(error, rte_errno,
2823                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2824                                                   NULL,
2825                                                   "cannot get counter"
2826                                                   " context.");
2827         }
2828         return 0;
2829 }
2830
2831 /**
2832  * Convert VXLAN VNI to 32-bit integer.
2833  *
2834  * @param[in] vni
2835  *   VXLAN VNI in 24-bit wire format.
2836  *
2837  * @return
2838  *   VXLAN VNI as a 32-bit integer value in network endian.
2839  */
2840 static inline rte_be32_t
2841 vxlan_vni_as_be32(const uint8_t vni[3])
2842 {
2843         union {
2844                 uint8_t vni[4];
2845                 rte_be32_t dword;
2846         } ret = {
2847                 .vni = { 0, vni[0], vni[1], vni[2] },
2848         };
2849         return ret.dword;
2850 }
2851
2852 /**
2853  * Helper function to process RTE_FLOW_ITEM_TYPE_ETH entry in configuration
2854  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the MAC address fields
2855  * in the encapsulation parameters structure. The item must be prevalidated,
2856  * no any validation checks performed by function.
2857  *
2858  * @param[in] spec
2859  *   RTE_FLOW_ITEM_TYPE_ETH entry specification.
2860  * @param[in] mask
2861  *   RTE_FLOW_ITEM_TYPE_ETH entry mask.
2862  * @param[out] encap
2863  *   Structure to fill the gathered MAC address data.
2864  */
2865 static void
2866 flow_tcf_parse_vxlan_encap_eth(const struct rte_flow_item_eth *spec,
2867                                const struct rte_flow_item_eth *mask,
2868                                struct flow_tcf_vxlan_encap *encap)
2869 {
2870         /* Item must be validated before. No redundant checks. */
2871         assert(spec);
2872         if (!mask || !memcmp(&mask->dst,
2873                              &rte_flow_item_eth_mask.dst,
2874                              sizeof(rte_flow_item_eth_mask.dst))) {
2875                 /*
2876                  * Ethernet addresses are not supported by
2877                  * tc as tunnel_key parameters. Destination
2878                  * address is needed to form encap packet
2879                  * header and retrieved by kernel from
2880                  * implicit sources (ARP table, etc),
2881                  * address masks are not supported at all.
2882                  */
2883                 encap->eth.dst = spec->dst;
2884                 encap->mask |= FLOW_TCF_ENCAP_ETH_DST;
2885         }
2886         if (!mask || !memcmp(&mask->src,
2887                              &rte_flow_item_eth_mask.src,
2888                              sizeof(rte_flow_item_eth_mask.src))) {
2889                 /*
2890                  * Ethernet addresses are not supported by
2891                  * tc as tunnel_key parameters. Source ethernet
2892                  * address is ignored anyway.
2893                  */
2894                 encap->eth.src = spec->src;
2895                 encap->mask |= FLOW_TCF_ENCAP_ETH_SRC;
2896         }
2897 }
2898
2899 /**
2900  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV4 entry in configuration
2901  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV4 address fields
2902  * in the encapsulation parameters structure. The item must be prevalidated,
2903  * no any validation checks performed by function.
2904  *
2905  * @param[in] spec
2906  *   RTE_FLOW_ITEM_TYPE_IPV4 entry specification.
2907  * @param[out] encap
2908  *   Structure to fill the gathered IPV4 address data.
2909  */
2910 static void
2911 flow_tcf_parse_vxlan_encap_ipv4(const struct rte_flow_item_ipv4 *spec,
2912                                 struct flow_tcf_vxlan_encap *encap)
2913 {
2914         /* Item must be validated before. No redundant checks. */
2915         assert(spec);
2916         encap->ipv4.dst = spec->hdr.dst_addr;
2917         encap->ipv4.src = spec->hdr.src_addr;
2918         encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC |
2919                        FLOW_TCF_ENCAP_IPV4_DST;
2920 }
2921
2922 /**
2923  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV6 entry in configuration
2924  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV6 address fields
2925  * in the encapsulation parameters structure. The item must be prevalidated,
2926  * no any validation checks performed by function.
2927  *
2928  * @param[in] spec
2929  *   RTE_FLOW_ITEM_TYPE_IPV6 entry specification.
2930  * @param[out] encap
2931  *   Structure to fill the gathered IPV6 address data.
2932  */
2933 static void
2934 flow_tcf_parse_vxlan_encap_ipv6(const struct rte_flow_item_ipv6 *spec,
2935                                 struct flow_tcf_vxlan_encap *encap)
2936 {
2937         /* Item must be validated before. No redundant checks. */
2938         assert(spec);
2939         memcpy(encap->ipv6.dst, spec->hdr.dst_addr, IPV6_ADDR_LEN);
2940         memcpy(encap->ipv6.src, spec->hdr.src_addr, IPV6_ADDR_LEN);
2941         encap->mask |= FLOW_TCF_ENCAP_IPV6_SRC |
2942                        FLOW_TCF_ENCAP_IPV6_DST;
2943 }
2944
2945 /**
2946  * Helper function to process RTE_FLOW_ITEM_TYPE_UDP entry in configuration
2947  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the UDP port fields
2948  * in the encapsulation parameters structure. The item must be prevalidated,
2949  * no any validation checks performed by function.
2950  *
2951  * @param[in] spec
2952  *   RTE_FLOW_ITEM_TYPE_UDP entry specification.
2953  * @param[in] mask
2954  *   RTE_FLOW_ITEM_TYPE_UDP entry mask.
2955  * @param[out] encap
2956  *   Structure to fill the gathered UDP port data.
2957  */
2958 static void
2959 flow_tcf_parse_vxlan_encap_udp(const struct rte_flow_item_udp *spec,
2960                                const struct rte_flow_item_udp *mask,
2961                                struct flow_tcf_vxlan_encap *encap)
2962 {
2963         assert(spec);
2964         encap->udp.dst = spec->hdr.dst_port;
2965         encap->mask |= FLOW_TCF_ENCAP_UDP_DST;
2966         if (!mask || mask->hdr.src_port != RTE_BE16(0x0000)) {
2967                 encap->udp.src = spec->hdr.src_port;
2968                 encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC;
2969         }
2970 }
2971
2972 /**
2973  * Helper function to process RTE_FLOW_ITEM_TYPE_VXLAN entry in configuration
2974  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the VNI fields
2975  * in the encapsulation parameters structure. The item must be prevalidated,
2976  * no any validation checks performed by function.
2977  *
2978  * @param[in] spec
2979  *   RTE_FLOW_ITEM_TYPE_VXLAN entry specification.
2980  * @param[out] encap
2981  *   Structure to fill the gathered VNI address data.
2982  */
2983 static void
2984 flow_tcf_parse_vxlan_encap_vni(const struct rte_flow_item_vxlan *spec,
2985                                struct flow_tcf_vxlan_encap *encap)
2986 {
2987         /* Item must be validated before. Do not redundant checks. */
2988         assert(spec);
2989         memcpy(encap->vxlan.vni, spec->vni, sizeof(encap->vxlan.vni));
2990         encap->mask |= FLOW_TCF_ENCAP_VXLAN_VNI;
2991 }
2992
2993 /**
2994  * Populate consolidated encapsulation object from list of pattern items.
2995  *
2996  * Helper function to process configuration of action such as
2997  * RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. The item list should be
2998  * validated, there is no way to return an meaningful error.
2999  *
3000  * @param[in] action
3001  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
3002  *   List of pattern items to gather data from.
3003  * @param[out] src
3004  *   Structure to fill gathered data.
3005  */
3006 static void
3007 flow_tcf_vxlan_encap_parse(const struct rte_flow_action *action,
3008                            struct flow_tcf_vxlan_encap *encap)
3009 {
3010         union {
3011                 const struct rte_flow_item_eth *eth;
3012                 const struct rte_flow_item_ipv4 *ipv4;
3013                 const struct rte_flow_item_ipv6 *ipv6;
3014                 const struct rte_flow_item_udp *udp;
3015                 const struct rte_flow_item_vxlan *vxlan;
3016         } spec, mask;
3017         const struct rte_flow_item *items;
3018
3019         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
3020         assert(action->conf);
3021
3022         items = ((const struct rte_flow_action_vxlan_encap *)
3023                                         action->conf)->definition;
3024         assert(items);
3025         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3026                 switch (items->type) {
3027                 case RTE_FLOW_ITEM_TYPE_VOID:
3028                         break;
3029                 case RTE_FLOW_ITEM_TYPE_ETH:
3030                         mask.eth = items->mask;
3031                         spec.eth = items->spec;
3032                         flow_tcf_parse_vxlan_encap_eth(spec.eth, mask.eth,
3033                                                        encap);
3034                         break;
3035                 case RTE_FLOW_ITEM_TYPE_IPV4:
3036                         spec.ipv4 = items->spec;
3037                         flow_tcf_parse_vxlan_encap_ipv4(spec.ipv4, encap);
3038                         break;
3039                 case RTE_FLOW_ITEM_TYPE_IPV6:
3040                         spec.ipv6 = items->spec;
3041                         flow_tcf_parse_vxlan_encap_ipv6(spec.ipv6, encap);
3042                         break;
3043                 case RTE_FLOW_ITEM_TYPE_UDP:
3044                         mask.udp = items->mask;
3045                         spec.udp = items->spec;
3046                         flow_tcf_parse_vxlan_encap_udp(spec.udp, mask.udp,
3047                                                        encap);
3048                         break;
3049                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3050                         spec.vxlan = items->spec;
3051                         flow_tcf_parse_vxlan_encap_vni(spec.vxlan, encap);
3052                         break;
3053                 default:
3054                         assert(false);
3055                         DRV_LOG(WARNING,
3056                                 "unsupported item %p type %d,"
3057                                 " items must be validated"
3058                                 " before flow creation",
3059                                 (const void *)items, items->type);
3060                         encap->mask = 0;
3061                         return;
3062                 }
3063         }
3064 }
3065
3066 /**
3067  * Translate flow for Linux TC flower and construct Netlink message.
3068  *
3069  * @param[in] priv
3070  *   Pointer to the priv structure.
3071  * @param[in, out] flow
3072  *   Pointer to the sub flow.
3073  * @param[in] attr
3074  *   Pointer to the flow attributes.
3075  * @param[in] items
3076  *   Pointer to the list of items.
3077  * @param[in] actions
3078  *   Pointer to the list of actions.
3079  * @param[out] error
3080  *   Pointer to the error structure.
3081  *
3082  * @return
3083  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3084  */
3085 static int
3086 flow_tcf_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3087                    const struct rte_flow_attr *attr,
3088                    const struct rte_flow_item items[],
3089                    const struct rte_flow_action actions[],
3090                    struct rte_flow_error *error)
3091 {
3092         union {
3093                 const struct rte_flow_item_port_id *port_id;
3094                 const struct rte_flow_item_eth *eth;
3095                 const struct rte_flow_item_vlan *vlan;
3096                 const struct rte_flow_item_ipv4 *ipv4;
3097                 const struct rte_flow_item_ipv6 *ipv6;
3098                 const struct rte_flow_item_tcp *tcp;
3099                 const struct rte_flow_item_udp *udp;
3100                 const struct rte_flow_item_vxlan *vxlan;
3101         } spec, mask;
3102         union {
3103                 const struct rte_flow_action_port_id *port_id;
3104                 const struct rte_flow_action_jump *jump;
3105                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
3106                 const struct rte_flow_action_of_set_vlan_vid *
3107                         of_set_vlan_vid;
3108                 const struct rte_flow_action_of_set_vlan_pcp *
3109                         of_set_vlan_pcp;
3110         } conf;
3111         union {
3112                 struct flow_tcf_tunnel_hdr *hdr;
3113                 struct flow_tcf_vxlan_decap *vxlan;
3114         } decap = {
3115                 .hdr = NULL,
3116         };
3117         union {
3118                 struct flow_tcf_tunnel_hdr *hdr;
3119                 struct flow_tcf_vxlan_encap *vxlan;
3120         } encap = {
3121                 .hdr = NULL,
3122         };
3123         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
3124         struct nlmsghdr *nlh = dev_flow->tcf.nlh;
3125         struct tcmsg *tcm = dev_flow->tcf.tcm;
3126         uint32_t na_act_index_cur;
3127         bool eth_type_set = 0;
3128         bool vlan_present = 0;
3129         bool vlan_eth_type_set = 0;
3130         bool ip_proto_set = 0;
3131         bool tunnel_outer = 0;
3132         struct nlattr *na_flower;
3133         struct nlattr *na_flower_act;
3134         struct nlattr *na_vlan_id = NULL;
3135         struct nlattr *na_vlan_priority = NULL;
3136         uint64_t item_flags = 0;
3137         int ret;
3138
3139         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
3140                                                 PTOI_TABLE_SZ_MAX(dev)));
3141         if (dev_flow->tcf.tunnel) {
3142                 switch (dev_flow->tcf.tunnel->type) {
3143                 case FLOW_TCF_TUNACT_VXLAN_DECAP:
3144                         decap.vxlan = dev_flow->tcf.vxlan_decap;
3145                         tunnel_outer = 1;
3146                         break;
3147                 case FLOW_TCF_TUNACT_VXLAN_ENCAP:
3148                         encap.vxlan = dev_flow->tcf.vxlan_encap;
3149                         break;
3150                 /* New tunnel actions can be added here. */
3151                 default:
3152                         assert(false);
3153                         break;
3154                 }
3155         }
3156         nlh = dev_flow->tcf.nlh;
3157         tcm = dev_flow->tcf.tcm;
3158         /* Prepare API must have been called beforehand. */
3159         assert(nlh != NULL && tcm != NULL);
3160         tcm->tcm_family = AF_UNSPEC;
3161         tcm->tcm_ifindex = ptoi[0].ifindex;
3162         tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
3163         /*
3164          * Priority cannot be zero to prevent the kernel from picking one
3165          * automatically.
3166          */
3167         tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
3168                                   RTE_BE16(ETH_P_ALL));
3169         if (attr->group > 0)
3170                 mnl_attr_put_u32(nlh, TCA_CHAIN, attr->group);
3171         mnl_attr_put_strz(nlh, TCA_KIND, "flower");
3172         na_flower = mnl_attr_nest_start(nlh, TCA_OPTIONS);
3173         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3174                 unsigned int i;
3175
3176                 switch (items->type) {
3177                 case RTE_FLOW_ITEM_TYPE_VOID:
3178                         break;
3179                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3180                         mask.port_id = flow_tcf_item_mask
3181                                 (items, &rte_flow_item_port_id_mask,
3182                                  &flow_tcf_mask_supported.port_id,
3183                                  &flow_tcf_mask_empty.port_id,
3184                                  sizeof(flow_tcf_mask_supported.port_id),
3185                                  error);
3186                         assert(mask.port_id);
3187                         if (mask.port_id == &flow_tcf_mask_empty.port_id)
3188                                 break;
3189                         spec.port_id = items->spec;
3190                         if (!mask.port_id->id)
3191                                 i = 0;
3192                         else
3193                                 for (i = 0; ptoi[i].ifindex; ++i)
3194                                         if (ptoi[i].port_id == spec.port_id->id)
3195                                                 break;
3196                         assert(ptoi[i].ifindex);
3197                         tcm->tcm_ifindex = ptoi[i].ifindex;
3198                         break;
3199                 case RTE_FLOW_ITEM_TYPE_ETH:
3200                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3201                                       MLX5_FLOW_LAYER_INNER_L2 :
3202                                       MLX5_FLOW_LAYER_OUTER_L2;
3203                         mask.eth = flow_tcf_item_mask
3204                                 (items, &rte_flow_item_eth_mask,
3205                                  &flow_tcf_mask_supported.eth,
3206                                  &flow_tcf_mask_empty.eth,
3207                                  sizeof(flow_tcf_mask_supported.eth),
3208                                  error);
3209                         assert(mask.eth);
3210                         if (mask.eth == &flow_tcf_mask_empty.eth)
3211                                 break;
3212                         spec.eth = items->spec;
3213                         if (tunnel_outer) {
3214                                 DRV_LOG(WARNING,
3215                                         "outer L2 addresses cannot be"
3216                                         " forced is outer ones for tunnel,"
3217                                         " parameter is ignored");
3218                                 break;
3219                         }
3220                         if (mask.eth->type) {
3221                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3222                                                  spec.eth->type);
3223                                 eth_type_set = 1;
3224                         }
3225                         if (!is_zero_ether_addr(&mask.eth->dst)) {
3226                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST,
3227                                              ETHER_ADDR_LEN,
3228                                              spec.eth->dst.addr_bytes);
3229                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST_MASK,
3230                                              ETHER_ADDR_LEN,
3231                                              mask.eth->dst.addr_bytes);
3232                         }
3233                         if (!is_zero_ether_addr(&mask.eth->src)) {
3234                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC,
3235                                              ETHER_ADDR_LEN,
3236                                              spec.eth->src.addr_bytes);
3237                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC_MASK,
3238                                              ETHER_ADDR_LEN,
3239                                              mask.eth->src.addr_bytes);
3240                         }
3241                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3242                         break;
3243                 case RTE_FLOW_ITEM_TYPE_VLAN:
3244                         assert(!encap.hdr);
3245                         assert(!decap.hdr);
3246                         assert(!tunnel_outer);
3247                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
3248                         mask.vlan = flow_tcf_item_mask
3249                                 (items, &rte_flow_item_vlan_mask,
3250                                  &flow_tcf_mask_supported.vlan,
3251                                  &flow_tcf_mask_empty.vlan,
3252                                  sizeof(flow_tcf_mask_supported.vlan),
3253                                  error);
3254                         assert(mask.vlan);
3255                         if (!eth_type_set)
3256                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3257                                                  RTE_BE16(ETH_P_8021Q));
3258                         eth_type_set = 1;
3259                         vlan_present = 1;
3260                         if (mask.vlan == &flow_tcf_mask_empty.vlan)
3261                                 break;
3262                         spec.vlan = items->spec;
3263                         if (mask.vlan->inner_type) {
3264                                 mnl_attr_put_u16(nlh,
3265                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE,
3266                                                  spec.vlan->inner_type);
3267                                 vlan_eth_type_set = 1;
3268                         }
3269                         if (mask.vlan->tci & RTE_BE16(0xe000))
3270                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_VLAN_PRIO,
3271                                                 (rte_be_to_cpu_16
3272                                                  (spec.vlan->tci) >> 13) & 0x7);
3273                         if (mask.vlan->tci & RTE_BE16(0x0fff))
3274                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_VLAN_ID,
3275                                                  rte_be_to_cpu_16
3276                                                  (spec.vlan->tci &
3277                                                   RTE_BE16(0x0fff)));
3278                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3279                         break;
3280                 case RTE_FLOW_ITEM_TYPE_IPV4:
3281                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3282                                       MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3283                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3284                         mask.ipv4 = flow_tcf_item_mask
3285                                 (items, &rte_flow_item_ipv4_mask,
3286                                  &flow_tcf_mask_supported.ipv4,
3287                                  &flow_tcf_mask_empty.ipv4,
3288                                  sizeof(flow_tcf_mask_supported.ipv4),
3289                                  error);
3290                         assert(mask.ipv4);
3291                         spec.ipv4 = items->spec;
3292                         if (!tunnel_outer) {
3293                                 if (!eth_type_set ||
3294                                     (!vlan_eth_type_set && vlan_present))
3295                                         mnl_attr_put_u16
3296                                                 (nlh,
3297                                                  vlan_present ?
3298                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3299                                                  TCA_FLOWER_KEY_ETH_TYPE,
3300                                                  RTE_BE16(ETH_P_IP));
3301                                 eth_type_set = 1;
3302                                 vlan_eth_type_set = 1;
3303                         }
3304                         if (!tunnel_outer && mask.ipv4->hdr.next_proto_id) {
3305                                 /*
3306                                  * No way to set IP protocol for outer tunnel
3307                                  * layers. Usually it is fixed, for example,
3308                                  * to UDP for VXLAN/GPE.
3309                                  */
3310                                 assert(spec.ipv4); /* Mask is not empty. */
3311                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3312                                                 spec.ipv4->hdr.next_proto_id);
3313                                 ip_proto_set = 1;
3314                         }
3315                         if (mask.ipv4 == &flow_tcf_mask_empty.ipv4 ||
3316                              (!mask.ipv4->hdr.src_addr &&
3317                               !mask.ipv4->hdr.dst_addr)) {
3318                                 if (!tunnel_outer)
3319                                         break;
3320                                 /*
3321                                  * For tunnel outer we must set outer IP key
3322                                  * anyway, even if the specification/mask is
3323                                  * empty. There is no another way to tell
3324                                  * kernel about he outer layer protocol.
3325                                  */
3326                                 mnl_attr_put_u32
3327                                         (nlh, TCA_FLOWER_KEY_ENC_IPV4_SRC,
3328                                          mask.ipv4->hdr.src_addr);
3329                                 mnl_attr_put_u32
3330                                         (nlh, TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
3331                                          mask.ipv4->hdr.src_addr);
3332                                 assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3333                                 break;
3334                         }
3335                         if (mask.ipv4->hdr.src_addr) {
3336                                 mnl_attr_put_u32
3337                                         (nlh, tunnel_outer ?
3338                                          TCA_FLOWER_KEY_ENC_IPV4_SRC :
3339                                          TCA_FLOWER_KEY_IPV4_SRC,
3340                                          spec.ipv4->hdr.src_addr);
3341                                 mnl_attr_put_u32
3342                                         (nlh, tunnel_outer ?
3343                                          TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK :
3344                                          TCA_FLOWER_KEY_IPV4_SRC_MASK,
3345                                          mask.ipv4->hdr.src_addr);
3346                         }
3347                         if (mask.ipv4->hdr.dst_addr) {
3348                                 mnl_attr_put_u32
3349                                         (nlh, tunnel_outer ?
3350                                          TCA_FLOWER_KEY_ENC_IPV4_DST :
3351                                          TCA_FLOWER_KEY_IPV4_DST,
3352                                          spec.ipv4->hdr.dst_addr);
3353                                 mnl_attr_put_u32
3354                                         (nlh, tunnel_outer ?
3355                                          TCA_FLOWER_KEY_ENC_IPV4_DST_MASK :
3356                                          TCA_FLOWER_KEY_IPV4_DST_MASK,
3357                                          mask.ipv4->hdr.dst_addr);
3358                         }
3359                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3360                         break;
3361                 case RTE_FLOW_ITEM_TYPE_IPV6: {
3362                         bool ipv6_src, ipv6_dst;
3363
3364                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3365                                       MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3366                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3367                         mask.ipv6 = flow_tcf_item_mask
3368                                 (items, &rte_flow_item_ipv6_mask,
3369                                  &flow_tcf_mask_supported.ipv6,
3370                                  &flow_tcf_mask_empty.ipv6,
3371                                  sizeof(flow_tcf_mask_supported.ipv6),
3372                                  error);
3373                         assert(mask.ipv6);
3374                         spec.ipv6 = items->spec;
3375                         if (!tunnel_outer) {
3376                                 if (!eth_type_set ||
3377                                     (!vlan_eth_type_set && vlan_present))
3378                                         mnl_attr_put_u16
3379                                                 (nlh,
3380                                                  vlan_present ?
3381                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3382                                                  TCA_FLOWER_KEY_ETH_TYPE,
3383                                                  RTE_BE16(ETH_P_IPV6));
3384                                 eth_type_set = 1;
3385                                 vlan_eth_type_set = 1;
3386                         }
3387                         if (!tunnel_outer && mask.ipv6->hdr.proto) {
3388                                 /*
3389                                  * No way to set IP protocol for outer tunnel
3390                                  * layers. Usually it is fixed, for example,
3391                                  * to UDP for VXLAN/GPE.
3392                                  */
3393                                 assert(spec.ipv6); /* Mask is not empty. */
3394                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3395                                                 spec.ipv6->hdr.proto);
3396                                 ip_proto_set = 1;
3397                         }
3398                         ipv6_dst = !IN6_IS_ADDR_UNSPECIFIED
3399                                                 (mask.ipv6->hdr.dst_addr);
3400                         ipv6_src = !IN6_IS_ADDR_UNSPECIFIED
3401                                                 (mask.ipv6->hdr.src_addr);
3402                         if (mask.ipv6 == &flow_tcf_mask_empty.ipv6 ||
3403                              (!ipv6_dst && !ipv6_src)) {
3404                                 if (!tunnel_outer)
3405                                         break;
3406                                 /*
3407                                  * For tunnel outer we must set outer IP key
3408                                  * anyway, even if the specification/mask is
3409                                  * empty. There is no another way to tell
3410                                  * kernel about he outer layer protocol.
3411                                  */
3412                                 mnl_attr_put(nlh,
3413                                              TCA_FLOWER_KEY_ENC_IPV6_SRC,
3414                                              IPV6_ADDR_LEN,
3415                                              mask.ipv6->hdr.src_addr);
3416                                 mnl_attr_put(nlh,
3417                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
3418                                              IPV6_ADDR_LEN,
3419                                              mask.ipv6->hdr.src_addr);
3420                                 assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3421                                 break;
3422                         }
3423                         if (ipv6_src) {
3424                                 mnl_attr_put(nlh, tunnel_outer ?
3425                                              TCA_FLOWER_KEY_ENC_IPV6_SRC :
3426                                              TCA_FLOWER_KEY_IPV6_SRC,
3427                                              IPV6_ADDR_LEN,
3428                                              spec.ipv6->hdr.src_addr);
3429                                 mnl_attr_put(nlh, tunnel_outer ?
3430                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK :
3431                                              TCA_FLOWER_KEY_IPV6_SRC_MASK,
3432                                              IPV6_ADDR_LEN,
3433                                              mask.ipv6->hdr.src_addr);
3434                         }
3435                         if (ipv6_dst) {
3436                                 mnl_attr_put(nlh, tunnel_outer ?
3437                                              TCA_FLOWER_KEY_ENC_IPV6_DST :
3438                                              TCA_FLOWER_KEY_IPV6_DST,
3439                                              IPV6_ADDR_LEN,
3440                                              spec.ipv6->hdr.dst_addr);
3441                                 mnl_attr_put(nlh, tunnel_outer ?
3442                                              TCA_FLOWER_KEY_ENC_IPV6_DST_MASK :
3443                                              TCA_FLOWER_KEY_IPV6_DST_MASK,
3444                                              IPV6_ADDR_LEN,
3445                                              mask.ipv6->hdr.dst_addr);
3446                         }
3447                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3448                         break;
3449                 }
3450                 case RTE_FLOW_ITEM_TYPE_UDP:
3451                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3452                                       MLX5_FLOW_LAYER_INNER_L4_UDP :
3453                                       MLX5_FLOW_LAYER_OUTER_L4_UDP;
3454                         mask.udp = flow_tcf_item_mask
3455                                 (items, &rte_flow_item_udp_mask,
3456                                  &flow_tcf_mask_supported.udp,
3457                                  &flow_tcf_mask_empty.udp,
3458                                  sizeof(flow_tcf_mask_supported.udp),
3459                                  error);
3460                         assert(mask.udp);
3461                         spec.udp = items->spec;
3462                         if (!tunnel_outer) {
3463                                 if (!ip_proto_set)
3464                                         mnl_attr_put_u8
3465                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3466                                                 IPPROTO_UDP);
3467                                 if (mask.udp == &flow_tcf_mask_empty.udp)
3468                                         break;
3469                         } else {
3470                                 assert(mask.udp != &flow_tcf_mask_empty.udp);
3471                                 decap.vxlan->udp_port =
3472                                         rte_be_to_cpu_16
3473                                                 (spec.udp->hdr.dst_port);
3474                         }
3475                         if (mask.udp->hdr.src_port) {
3476                                 mnl_attr_put_u16
3477                                         (nlh, tunnel_outer ?
3478                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT :
3479                                          TCA_FLOWER_KEY_UDP_SRC,
3480                                          spec.udp->hdr.src_port);
3481                                 mnl_attr_put_u16
3482                                         (nlh, tunnel_outer ?
3483                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK :
3484                                          TCA_FLOWER_KEY_UDP_SRC_MASK,
3485                                          mask.udp->hdr.src_port);
3486                         }
3487                         if (mask.udp->hdr.dst_port) {
3488                                 mnl_attr_put_u16
3489                                         (nlh, tunnel_outer ?
3490                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT :
3491                                          TCA_FLOWER_KEY_UDP_DST,
3492                                          spec.udp->hdr.dst_port);
3493                                 mnl_attr_put_u16
3494                                         (nlh, tunnel_outer ?
3495                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK :
3496                                          TCA_FLOWER_KEY_UDP_DST_MASK,
3497                                          mask.udp->hdr.dst_port);
3498                         }
3499                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3500                         break;
3501                 case RTE_FLOW_ITEM_TYPE_TCP:
3502                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3503                                       MLX5_FLOW_LAYER_INNER_L4_TCP :
3504                                       MLX5_FLOW_LAYER_OUTER_L4_TCP;
3505                         mask.tcp = flow_tcf_item_mask
3506                                 (items, &rte_flow_item_tcp_mask,
3507                                  &flow_tcf_mask_supported.tcp,
3508                                  &flow_tcf_mask_empty.tcp,
3509                                  sizeof(flow_tcf_mask_supported.tcp),
3510                                  error);
3511                         assert(mask.tcp);
3512                         if (!ip_proto_set)
3513                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3514                                                 IPPROTO_TCP);
3515                         if (mask.tcp == &flow_tcf_mask_empty.tcp)
3516                                 break;
3517                         spec.tcp = items->spec;
3518                         if (mask.tcp->hdr.src_port) {
3519                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_SRC,
3520                                                  spec.tcp->hdr.src_port);
3521                                 mnl_attr_put_u16(nlh,
3522                                                  TCA_FLOWER_KEY_TCP_SRC_MASK,
3523                                                  mask.tcp->hdr.src_port);
3524                         }
3525                         if (mask.tcp->hdr.dst_port) {
3526                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_DST,
3527                                                  spec.tcp->hdr.dst_port);
3528                                 mnl_attr_put_u16(nlh,
3529                                                  TCA_FLOWER_KEY_TCP_DST_MASK,
3530                                                  mask.tcp->hdr.dst_port);
3531                         }
3532                         if (mask.tcp->hdr.tcp_flags) {
3533                                 mnl_attr_put_u16
3534                                         (nlh,
3535                                          TCA_FLOWER_KEY_TCP_FLAGS,
3536                                          rte_cpu_to_be_16
3537                                                 (spec.tcp->hdr.tcp_flags));
3538                                 mnl_attr_put_u16
3539                                         (nlh,
3540                                          TCA_FLOWER_KEY_TCP_FLAGS_MASK,
3541                                          rte_cpu_to_be_16
3542                                                 (mask.tcp->hdr.tcp_flags));
3543                         }
3544                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3545                         break;
3546                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3547                         assert(decap.vxlan);
3548                         tunnel_outer = 0;
3549                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
3550                         spec.vxlan = items->spec;
3551                         mnl_attr_put_u32(nlh,
3552                                          TCA_FLOWER_KEY_ENC_KEY_ID,
3553                                          vxlan_vni_as_be32(spec.vxlan->vni));
3554                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3555                         break;
3556                 default:
3557                         return rte_flow_error_set(error, ENOTSUP,
3558                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3559                                                   NULL, "item not supported");
3560                 }
3561         }
3562         na_flower_act = mnl_attr_nest_start(nlh, TCA_FLOWER_ACT);
3563         na_act_index_cur = 1;
3564         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3565                 struct nlattr *na_act_index;
3566                 struct nlattr *na_act;
3567                 unsigned int vlan_act;
3568                 unsigned int i;
3569
3570                 switch (actions->type) {
3571                 case RTE_FLOW_ACTION_TYPE_VOID:
3572                         break;
3573                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3574                         conf.port_id = actions->conf;
3575                         if (conf.port_id->original)
3576                                 i = 0;
3577                         else
3578                                 for (i = 0; ptoi[i].ifindex; ++i)
3579                                         if (ptoi[i].port_id == conf.port_id->id)
3580                                                 break;
3581                         assert(ptoi[i].ifindex);
3582                         na_act_index =
3583                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3584                         assert(na_act_index);
3585                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "mirred");
3586                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3587                         assert(na_act);
3588                         if (encap.hdr) {
3589                                 assert(dev_flow->tcf.tunnel);
3590                                 dev_flow->tcf.tunnel->ifindex_ptr =
3591                                         &((struct tc_mirred *)
3592                                         mnl_attr_get_payload
3593                                         (mnl_nlmsg_get_payload_tail
3594                                                 (nlh)))->ifindex;
3595                         }
3596                         mnl_attr_put(nlh, TCA_MIRRED_PARMS,
3597                                      sizeof(struct tc_mirred),
3598                                      &(struct tc_mirred){
3599                                         .action = TC_ACT_STOLEN,
3600                                         .eaction = TCA_EGRESS_REDIR,
3601                                         .ifindex = ptoi[i].ifindex,
3602                                      });
3603                         mnl_attr_nest_end(nlh, na_act);
3604                         mnl_attr_nest_end(nlh, na_act_index);
3605                         break;
3606                 case RTE_FLOW_ACTION_TYPE_JUMP:
3607                         conf.jump = actions->conf;
3608                         na_act_index =
3609                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3610                         assert(na_act_index);
3611                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3612                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3613                         assert(na_act);
3614                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3615                                      sizeof(struct tc_gact),
3616                                      &(struct tc_gact){
3617                                         .action = TC_ACT_GOTO_CHAIN |
3618                                                   conf.jump->group,
3619                                      });
3620                         mnl_attr_nest_end(nlh, na_act);
3621                         mnl_attr_nest_end(nlh, na_act_index);
3622                         break;
3623                 case RTE_FLOW_ACTION_TYPE_DROP:
3624                         na_act_index =
3625                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3626                         assert(na_act_index);
3627                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3628                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3629                         assert(na_act);
3630                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3631                                      sizeof(struct tc_gact),
3632                                      &(struct tc_gact){
3633                                         .action = TC_ACT_SHOT,
3634                                      });
3635                         mnl_attr_nest_end(nlh, na_act);
3636                         mnl_attr_nest_end(nlh, na_act_index);
3637                         break;
3638                 case RTE_FLOW_ACTION_TYPE_COUNT:
3639                         /*
3640                          * Driver adds the count action implicitly for
3641                          * each rule it creates.
3642                          */
3643                         ret = flow_tcf_translate_action_count(dev,
3644                                                               dev_flow, error);
3645                         if (ret < 0)
3646                                 return ret;
3647                         break;
3648                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3649                         conf.of_push_vlan = NULL;
3650                         vlan_act = TCA_VLAN_ACT_POP;
3651                         goto action_of_vlan;
3652                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3653                         conf.of_push_vlan = actions->conf;
3654                         vlan_act = TCA_VLAN_ACT_PUSH;
3655                         goto action_of_vlan;
3656                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3657                         conf.of_set_vlan_vid = actions->conf;
3658                         if (na_vlan_id)
3659                                 goto override_na_vlan_id;
3660                         vlan_act = TCA_VLAN_ACT_MODIFY;
3661                         goto action_of_vlan;
3662                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3663                         conf.of_set_vlan_pcp = actions->conf;
3664                         if (na_vlan_priority)
3665                                 goto override_na_vlan_priority;
3666                         vlan_act = TCA_VLAN_ACT_MODIFY;
3667                         goto action_of_vlan;
3668 action_of_vlan:
3669                         na_act_index =
3670                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3671                         assert(na_act_index);
3672                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "vlan");
3673                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3674                         assert(na_act);
3675                         mnl_attr_put(nlh, TCA_VLAN_PARMS,
3676                                      sizeof(struct tc_vlan),
3677                                      &(struct tc_vlan){
3678                                         .action = TC_ACT_PIPE,
3679                                         .v_action = vlan_act,
3680                                      });
3681                         if (vlan_act == TCA_VLAN_ACT_POP) {
3682                                 mnl_attr_nest_end(nlh, na_act);
3683                                 mnl_attr_nest_end(nlh, na_act_index);
3684                                 break;
3685                         }
3686                         if (vlan_act == TCA_VLAN_ACT_PUSH)
3687                                 mnl_attr_put_u16(nlh,
3688                                                  TCA_VLAN_PUSH_VLAN_PROTOCOL,
3689                                                  conf.of_push_vlan->ethertype);
3690                         na_vlan_id = mnl_nlmsg_get_payload_tail(nlh);
3691                         mnl_attr_put_u16(nlh, TCA_VLAN_PAD, 0);
3692                         na_vlan_priority = mnl_nlmsg_get_payload_tail(nlh);
3693                         mnl_attr_put_u8(nlh, TCA_VLAN_PAD, 0);
3694                         mnl_attr_nest_end(nlh, na_act);
3695                         mnl_attr_nest_end(nlh, na_act_index);
3696                         if (actions->type ==
3697                             RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
3698 override_na_vlan_id:
3699                                 na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
3700                                 *(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
3701                                         rte_be_to_cpu_16
3702                                         (conf.of_set_vlan_vid->vlan_vid);
3703                         } else if (actions->type ==
3704                                    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
3705 override_na_vlan_priority:
3706                                 na_vlan_priority->nla_type =
3707                                         TCA_VLAN_PUSH_VLAN_PRIORITY;
3708                                 *(uint8_t *)mnl_attr_get_payload
3709                                         (na_vlan_priority) =
3710                                         conf.of_set_vlan_pcp->vlan_pcp;
3711                         }
3712                         break;
3713                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3714                         assert(decap.vxlan);
3715                         assert(dev_flow->tcf.tunnel);
3716                         dev_flow->tcf.tunnel->ifindex_ptr =
3717                                 (unsigned int *)&tcm->tcm_ifindex;
3718                         na_act_index =
3719                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3720                         assert(na_act_index);
3721                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3722                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3723                         assert(na_act);
3724                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3725                                 sizeof(struct tc_tunnel_key),
3726                                 &(struct tc_tunnel_key){
3727                                         .action = TC_ACT_PIPE,
3728                                         .t_action = TCA_TUNNEL_KEY_ACT_RELEASE,
3729                                         });
3730                         mnl_attr_nest_end(nlh, na_act);
3731                         mnl_attr_nest_end(nlh, na_act_index);
3732                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3733                         break;
3734                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3735                         assert(encap.vxlan);
3736                         flow_tcf_vxlan_encap_parse(actions, encap.vxlan);
3737                         na_act_index =
3738                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3739                         assert(na_act_index);
3740                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3741                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3742                         assert(na_act);
3743                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3744                                 sizeof(struct tc_tunnel_key),
3745                                 &(struct tc_tunnel_key){
3746                                         .action = TC_ACT_PIPE,
3747                                         .t_action = TCA_TUNNEL_KEY_ACT_SET,
3748                                         });
3749                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_UDP_DST)
3750                                 mnl_attr_put_u16(nlh,
3751                                          TCA_TUNNEL_KEY_ENC_DST_PORT,
3752                                          encap.vxlan->udp.dst);
3753                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_SRC)
3754                                 mnl_attr_put_u32(nlh,
3755                                          TCA_TUNNEL_KEY_ENC_IPV4_SRC,
3756                                          encap.vxlan->ipv4.src);
3757                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_DST)
3758                                 mnl_attr_put_u32(nlh,
3759                                          TCA_TUNNEL_KEY_ENC_IPV4_DST,
3760                                          encap.vxlan->ipv4.dst);
3761                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_SRC)
3762                                 mnl_attr_put(nlh,
3763                                          TCA_TUNNEL_KEY_ENC_IPV6_SRC,
3764                                          sizeof(encap.vxlan->ipv6.src),
3765                                          &encap.vxlan->ipv6.src);
3766                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_DST)
3767                                 mnl_attr_put(nlh,
3768                                          TCA_TUNNEL_KEY_ENC_IPV6_DST,
3769                                          sizeof(encap.vxlan->ipv6.dst),
3770                                          &encap.vxlan->ipv6.dst);
3771                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_VXLAN_VNI)
3772                                 mnl_attr_put_u32(nlh,
3773                                          TCA_TUNNEL_KEY_ENC_KEY_ID,
3774                                          vxlan_vni_as_be32
3775                                                 (encap.vxlan->vxlan.vni));
3776                         mnl_attr_put_u8(nlh, TCA_TUNNEL_KEY_NO_CSUM, 0);
3777                         mnl_attr_nest_end(nlh, na_act);
3778                         mnl_attr_nest_end(nlh, na_act_index);
3779                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3780                         break;
3781                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3782                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3783                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3784                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3785                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3786                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3787                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3788                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3789                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3790                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3791                         na_act_index =
3792                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3793                         flow_tcf_create_pedit_mnl_msg(nlh,
3794                                                       &actions, item_flags);
3795                         mnl_attr_nest_end(nlh, na_act_index);
3796                         break;
3797                 default:
3798                         return rte_flow_error_set(error, ENOTSUP,
3799                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3800                                                   actions,
3801                                                   "action not supported");
3802                 }
3803         }
3804         assert(na_flower);
3805         assert(na_flower_act);
3806         mnl_attr_nest_end(nlh, na_flower_act);
3807         dev_flow->tcf.ptc_flags = mnl_attr_get_payload
3808                                         (mnl_nlmsg_get_payload_tail(nlh));
3809         mnl_attr_put_u32(nlh, TCA_FLOWER_FLAGS, decap.vxlan ?
3810                                                 0 : TCA_CLS_FLAGS_SKIP_SW);
3811         mnl_attr_nest_end(nlh, na_flower);
3812         if (dev_flow->tcf.tunnel && dev_flow->tcf.tunnel->ifindex_ptr)
3813                 dev_flow->tcf.tunnel->ifindex_org =
3814                         *dev_flow->tcf.tunnel->ifindex_ptr;
3815         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3816         return 0;
3817 }
3818
3819 /**
3820  * Send Netlink message with acknowledgment.
3821  *
3822  * @param tcf
3823  *   Flow context to use.
3824  * @param nlh
3825  *   Message to send. This function always raises the NLM_F_ACK flag before
3826  *   sending.
3827  * @param[in] cb
3828  *   Callback handler for received message.
3829  * @param[in] arg
3830  *   Context pointer for callback handler.
3831  *
3832  * @return
3833  *   0 on success, a negative errno value otherwise and rte_errno is set.
3834  */
3835 static int
3836 flow_tcf_nl_ack(struct mlx5_flow_tcf_context *tcf,
3837                 struct nlmsghdr *nlh,
3838                 mnl_cb_t cb, void *arg)
3839 {
3840         unsigned int portid = mnl_socket_get_portid(tcf->nl);
3841         uint32_t seq = tcf->seq++;
3842         int ret, err = 0;
3843
3844         assert(tcf->nl);
3845         assert(tcf->buf);
3846         if (!seq) {
3847                 /* seq 0 is reserved for kernel event-driven notifications. */
3848                 seq = tcf->seq++;
3849         }
3850         nlh->nlmsg_seq = seq;
3851         nlh->nlmsg_flags |= NLM_F_ACK;
3852         ret = mnl_socket_sendto(tcf->nl, nlh, nlh->nlmsg_len);
3853         if (ret <= 0) {
3854                 /* Message send error occurres. */
3855                 rte_errno = errno;
3856                 return -rte_errno;
3857         }
3858         nlh = (struct nlmsghdr *)(tcf->buf);
3859         /*
3860          * The following loop postpones non-fatal errors until multipart
3861          * messages are complete.
3862          */
3863         while (true) {
3864                 ret = mnl_socket_recvfrom(tcf->nl, tcf->buf, tcf->buf_size);
3865                 if (ret < 0) {
3866                         err = errno;
3867                         /*
3868                          * In case of overflow Will receive till
3869                          * end of multipart message. We may lost part
3870                          * of reply messages but mark and return an error.
3871                          */
3872                         if (err != ENOSPC ||
3873                             !(nlh->nlmsg_flags & NLM_F_MULTI) ||
3874                             nlh->nlmsg_type == NLMSG_DONE)
3875                                 break;
3876                 } else {
3877                         ret = mnl_cb_run(nlh, ret, seq, portid, cb, arg);
3878                         if (!ret) {
3879                                 /*
3880                                  * libmnl returns 0 if DONE or
3881                                  * success ACK message found.
3882                                  */
3883                                 break;
3884                         }
3885                         if (ret < 0) {
3886                                 /*
3887                                  * ACK message with error found
3888                                  * or some error occurred.
3889                                  */
3890                                 err = errno;
3891                                 break;
3892                         }
3893                         /* We should continue receiving. */
3894                 }
3895         }
3896         if (!err)
3897                 return 0;
3898         rte_errno = err;
3899         return -err;
3900 }
3901
3902 #define MNL_BUF_EXTRA_SPACE 16
3903 #define MNL_REQUEST_SIZE_MIN 256
3904 #define MNL_REQUEST_SIZE_MAX 2048
3905 #define MNL_REQUEST_SIZE RTE_MIN(RTE_MAX(sysconf(_SC_PAGESIZE), \
3906                                  MNL_REQUEST_SIZE_MIN), MNL_REQUEST_SIZE_MAX)
3907
3908 /* Data structures used by flow_tcf_xxx_cb() routines. */
3909 struct tcf_nlcb_buf {
3910         LIST_ENTRY(tcf_nlcb_buf) next;
3911         uint32_t size;
3912         alignas(struct nlmsghdr)
3913         uint8_t msg[]; /**< Netlink message data. */
3914 };
3915
3916 struct tcf_nlcb_context {
3917         unsigned int ifindex; /**< Base interface index. */
3918         uint32_t bufsize;
3919         LIST_HEAD(, tcf_nlcb_buf) nlbuf;
3920 };
3921
3922 /**
3923  * Allocate space for netlink command in buffer list
3924  *
3925  * @param[in, out] ctx
3926  *   Pointer to callback context with command buffers list.
3927  * @param[in] size
3928  *   Required size of data buffer to be allocated.
3929  *
3930  * @return
3931  *   Pointer to allocated memory, aligned as message header.
3932  *   NULL if some error occurred.
3933  */
3934 static struct nlmsghdr *
3935 flow_tcf_alloc_nlcmd(struct tcf_nlcb_context *ctx, uint32_t size)
3936 {
3937         struct tcf_nlcb_buf *buf;
3938         struct nlmsghdr *nlh;
3939
3940         size = NLMSG_ALIGN(size);
3941         buf = LIST_FIRST(&ctx->nlbuf);
3942         if (buf && (buf->size + size) <= ctx->bufsize) {
3943                 nlh = (struct nlmsghdr *)&buf->msg[buf->size];
3944                 buf->size += size;
3945                 return nlh;
3946         }
3947         if (size > ctx->bufsize) {
3948                 DRV_LOG(WARNING, "netlink: too long command buffer requested");
3949                 return NULL;
3950         }
3951         buf = rte_malloc(__func__,
3952                         ctx->bufsize + sizeof(struct tcf_nlcb_buf),
3953                         alignof(struct tcf_nlcb_buf));
3954         if (!buf) {
3955                 DRV_LOG(WARNING, "netlink: no memory for command buffer");
3956                 return NULL;
3957         }
3958         LIST_INSERT_HEAD(&ctx->nlbuf, buf, next);
3959         buf->size = size;
3960         nlh = (struct nlmsghdr *)&buf->msg[0];
3961         return nlh;
3962 }
3963
3964 /**
3965  * Send the buffers with prepared netlink commands. Scans the list and
3966  * sends all found buffers. Buffers are sent and freed anyway in order
3967  * to prevent memory leakage if some every message in received packet.
3968  *
3969  * @param[in] tcf
3970  *   Context object initialized by mlx5_flow_tcf_context_create().
3971  * @param[in, out] ctx
3972  *   Pointer to callback context with command buffers list.
3973  *
3974  * @return
3975  *   Zero value on success, negative errno value otherwise
3976  *   and rte_errno is set.
3977  */
3978 static int
3979 flow_tcf_send_nlcmd(struct mlx5_flow_tcf_context *tcf,
3980                     struct tcf_nlcb_context *ctx)
3981 {
3982         struct tcf_nlcb_buf *bc = LIST_FIRST(&ctx->nlbuf);
3983         int ret = 0;
3984
3985         while (bc) {
3986                 struct tcf_nlcb_buf *bn = LIST_NEXT(bc, next);
3987                 struct nlmsghdr *nlh;
3988                 uint32_t msg = 0;
3989                 int rc;
3990
3991                 while (msg < bc->size) {
3992                         /*
3993                          * Send Netlink commands from buffer in one by one
3994                          * fashion. If we send multiple rule deletion commands
3995                          * in one Netlink message and some error occurs it may
3996                          * cause multiple ACK error messages and break sequence
3997                          * numbers of Netlink communication, because we expect
3998                          * the only one ACK reply.
3999                          */
4000                         assert((bc->size - msg) >= sizeof(struct nlmsghdr));
4001                         nlh = (struct nlmsghdr *)&bc->msg[msg];
4002                         assert((bc->size - msg) >= nlh->nlmsg_len);
4003                         msg += nlh->nlmsg_len;
4004                         rc = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4005                         if (rc) {
4006                                 DRV_LOG(WARNING,
4007                                         "netlink: cleanup error %d", rc);
4008                                 if (!ret)
4009                                         ret = rc;
4010                         }
4011                 }
4012                 rte_free(bc);
4013                 bc = bn;
4014         }
4015         LIST_INIT(&ctx->nlbuf);
4016         return ret;
4017 }
4018
4019 /**
4020  * Collect local IP address rules with scope link attribute  on specified
4021  * network device. This is callback routine called by libmnl mnl_cb_run()
4022  * in loop for every message in received packet.
4023  *
4024  * @param[in] nlh
4025  *   Pointer to reply header.
4026  * @param[in, out] arg
4027  *   Opaque data pointer for this callback.
4028  *
4029  * @return
4030  *   A positive, nonzero value on success, negative errno value otherwise
4031  *   and rte_errno is set.
4032  */
4033 static int
4034 flow_tcf_collect_local_cb(const struct nlmsghdr *nlh, void *arg)
4035 {
4036         struct tcf_nlcb_context *ctx = arg;
4037         struct nlmsghdr *cmd;
4038         struct ifaddrmsg *ifa;
4039         struct nlattr *na;
4040         struct nlattr *na_local = NULL;
4041         struct nlattr *na_peer = NULL;
4042         unsigned char family;
4043         uint32_t size;
4044
4045         if (nlh->nlmsg_type != RTM_NEWADDR) {
4046                 rte_errno = EINVAL;
4047                 return -rte_errno;
4048         }
4049         ifa = mnl_nlmsg_get_payload(nlh);
4050         family = ifa->ifa_family;
4051         if (ifa->ifa_index != ctx->ifindex ||
4052             ifa->ifa_scope != RT_SCOPE_LINK ||
4053             !(ifa->ifa_flags & IFA_F_PERMANENT) ||
4054             (family != AF_INET && family != AF_INET6))
4055                 return 1;
4056         mnl_attr_for_each(na, nlh, sizeof(*ifa)) {
4057                 switch (mnl_attr_get_type(na)) {
4058                 case IFA_LOCAL:
4059                         na_local = na;
4060                         break;
4061                 case IFA_ADDRESS:
4062                         na_peer = na;
4063                         break;
4064                 }
4065                 if (na_local && na_peer)
4066                         break;
4067         }
4068         if (!na_local || !na_peer)
4069                 return 1;
4070         /* Local rule found with scope link, permanent and assigned peer. */
4071         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4072                MNL_ALIGN(sizeof(struct ifaddrmsg)) +
4073                (family == AF_INET6 ? 2 * SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
4074                                    : 2 * SZ_NLATTR_TYPE_OF(uint32_t));
4075         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4076         if (!cmd) {
4077                 rte_errno = ENOMEM;
4078                 return -rte_errno;
4079         }
4080         cmd = mnl_nlmsg_put_header(cmd);
4081         cmd->nlmsg_type = RTM_DELADDR;
4082         cmd->nlmsg_flags = NLM_F_REQUEST;
4083         ifa = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifa));
4084         ifa->ifa_flags = IFA_F_PERMANENT;
4085         ifa->ifa_scope = RT_SCOPE_LINK;
4086         ifa->ifa_index = ctx->ifindex;
4087         if (family == AF_INET) {
4088                 ifa->ifa_family = AF_INET;
4089                 ifa->ifa_prefixlen = 32;
4090                 mnl_attr_put_u32(cmd, IFA_LOCAL, mnl_attr_get_u32(na_local));
4091                 mnl_attr_put_u32(cmd, IFA_ADDRESS, mnl_attr_get_u32(na_peer));
4092         } else {
4093                 ifa->ifa_family = AF_INET6;
4094                 ifa->ifa_prefixlen = 128;
4095                 mnl_attr_put(cmd, IFA_LOCAL, IPV6_ADDR_LEN,
4096                         mnl_attr_get_payload(na_local));
4097                 mnl_attr_put(cmd, IFA_ADDRESS, IPV6_ADDR_LEN,
4098                         mnl_attr_get_payload(na_peer));
4099         }
4100         assert(size == cmd->nlmsg_len);
4101         return 1;
4102 }
4103
4104 /**
4105  * Cleanup the local IP addresses on outer interface.
4106  *
4107  * @param[in] tcf
4108  *   Context object initialized by mlx5_flow_tcf_context_create().
4109  * @param[in] ifindex
4110  *   Network inferface index to perform cleanup.
4111  */
4112 static void
4113 flow_tcf_encap_local_cleanup(struct mlx5_flow_tcf_context *tcf,
4114                             unsigned int ifindex)
4115 {
4116         struct nlmsghdr *nlh;
4117         struct ifaddrmsg *ifa;
4118         struct tcf_nlcb_context ctx = {
4119                 .ifindex = ifindex,
4120                 .bufsize = MNL_REQUEST_SIZE,
4121                 .nlbuf = LIST_HEAD_INITIALIZER(),
4122         };
4123         int ret;
4124
4125         assert(ifindex);
4126         /*
4127          * Seek and destroy leftovers of local IP addresses with
4128          * matching properties "scope link".
4129          */
4130         nlh = mnl_nlmsg_put_header(tcf->buf);
4131         nlh->nlmsg_type = RTM_GETADDR;
4132         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4133         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4134         ifa->ifa_family = AF_UNSPEC;
4135         ifa->ifa_index = ifindex;
4136         ifa->ifa_scope = RT_SCOPE_LINK;
4137         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_local_cb, &ctx);
4138         if (ret)
4139                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4140         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4141         if (ret)
4142                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4143 }
4144
4145 /**
4146  * Collect neigh permament rules on specified network device.
4147  * This is callback routine called by libmnl mnl_cb_run() in loop for
4148  * every message in received packet.
4149  *
4150  * @param[in] nlh
4151  *   Pointer to reply header.
4152  * @param[in, out] arg
4153  *   Opaque data pointer for this callback.
4154  *
4155  * @return
4156  *   A positive, nonzero value on success, negative errno value otherwise
4157  *   and rte_errno is set.
4158  */
4159 static int
4160 flow_tcf_collect_neigh_cb(const struct nlmsghdr *nlh, void *arg)
4161 {
4162         struct tcf_nlcb_context *ctx = arg;
4163         struct nlmsghdr *cmd;
4164         struct ndmsg *ndm;
4165         struct nlattr *na;
4166         struct nlattr *na_ip = NULL;
4167         struct nlattr *na_mac = NULL;
4168         unsigned char family;
4169         uint32_t size;
4170
4171         if (nlh->nlmsg_type != RTM_NEWNEIGH) {
4172                 rte_errno = EINVAL;
4173                 return -rte_errno;
4174         }
4175         ndm = mnl_nlmsg_get_payload(nlh);
4176         family = ndm->ndm_family;
4177         if (ndm->ndm_ifindex != (int)ctx->ifindex ||
4178            !(ndm->ndm_state & NUD_PERMANENT) ||
4179            (family != AF_INET && family != AF_INET6))
4180                 return 1;
4181         mnl_attr_for_each(na, nlh, sizeof(*ndm)) {
4182                 switch (mnl_attr_get_type(na)) {
4183                 case NDA_DST:
4184                         na_ip = na;
4185                         break;
4186                 case NDA_LLADDR:
4187                         na_mac = na;
4188                         break;
4189                 }
4190                 if (na_mac && na_ip)
4191                         break;
4192         }
4193         if (!na_mac || !na_ip)
4194                 return 1;
4195         /* Neigh rule with permenent attribute found. */
4196         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4197                MNL_ALIGN(sizeof(struct ndmsg)) +
4198                SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) +
4199                (family == AF_INET6 ? SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
4200                                    : SZ_NLATTR_TYPE_OF(uint32_t));
4201         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4202         if (!cmd) {
4203                 rte_errno = ENOMEM;
4204                 return -rte_errno;
4205         }
4206         cmd = mnl_nlmsg_put_header(cmd);
4207         cmd->nlmsg_type = RTM_DELNEIGH;
4208         cmd->nlmsg_flags = NLM_F_REQUEST;
4209         ndm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ndm));
4210         ndm->ndm_ifindex = ctx->ifindex;
4211         ndm->ndm_state = NUD_PERMANENT;
4212         ndm->ndm_flags = 0;
4213         ndm->ndm_type = 0;
4214         if (family == AF_INET) {
4215                 ndm->ndm_family = AF_INET;
4216                 mnl_attr_put_u32(cmd, NDA_DST, mnl_attr_get_u32(na_ip));
4217         } else {
4218                 ndm->ndm_family = AF_INET6;
4219                 mnl_attr_put(cmd, NDA_DST, IPV6_ADDR_LEN,
4220                              mnl_attr_get_payload(na_ip));
4221         }
4222         mnl_attr_put(cmd, NDA_LLADDR, ETHER_ADDR_LEN,
4223                      mnl_attr_get_payload(na_mac));
4224         assert(size == cmd->nlmsg_len);
4225         return 1;
4226 }
4227
4228 /**
4229  * Cleanup the neigh rules on outer interface.
4230  *
4231  * @param[in] tcf
4232  *   Context object initialized by mlx5_flow_tcf_context_create().
4233  * @param[in] ifindex
4234  *   Network inferface index to perform cleanup.
4235  */
4236 static void
4237 flow_tcf_encap_neigh_cleanup(struct mlx5_flow_tcf_context *tcf,
4238                             unsigned int ifindex)
4239 {
4240         struct nlmsghdr *nlh;
4241         struct ndmsg *ndm;
4242         struct tcf_nlcb_context ctx = {
4243                 .ifindex = ifindex,
4244                 .bufsize = MNL_REQUEST_SIZE,
4245                 .nlbuf = LIST_HEAD_INITIALIZER(),
4246         };
4247         int ret;
4248
4249         assert(ifindex);
4250         /* Seek and destroy leftovers of neigh rules. */
4251         nlh = mnl_nlmsg_put_header(tcf->buf);
4252         nlh->nlmsg_type = RTM_GETNEIGH;
4253         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4254         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4255         ndm->ndm_family = AF_UNSPEC;
4256         ndm->ndm_ifindex = ifindex;
4257         ndm->ndm_state = NUD_PERMANENT;
4258         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_neigh_cb, &ctx);
4259         if (ret)
4260                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4261         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4262         if (ret)
4263                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4264 }
4265
4266 /**
4267  * Collect indices of VXLAN encap/decap interfaces associated with device.
4268  * This is callback routine called by libmnl mnl_cb_run() in loop for
4269  * every message in received packet.
4270  *
4271  * @param[in] nlh
4272  *   Pointer to reply header.
4273  * @param[in, out] arg
4274  *   Opaque data pointer for this callback.
4275  *
4276  * @return
4277  *   A positive, nonzero value on success, negative errno value otherwise
4278  *   and rte_errno is set.
4279  */
4280 static int
4281 flow_tcf_collect_vxlan_cb(const struct nlmsghdr *nlh, void *arg)
4282 {
4283         struct tcf_nlcb_context *ctx = arg;
4284         struct nlmsghdr *cmd;
4285         struct ifinfomsg *ifm;
4286         struct nlattr *na;
4287         struct nlattr *na_info = NULL;
4288         struct nlattr *na_vxlan = NULL;
4289         bool found = false;
4290         unsigned int vxindex;
4291         uint32_t size;
4292
4293         if (nlh->nlmsg_type != RTM_NEWLINK) {
4294                 rte_errno = EINVAL;
4295                 return -rte_errno;
4296         }
4297         ifm = mnl_nlmsg_get_payload(nlh);
4298         if (!ifm->ifi_index) {
4299                 rte_errno = EINVAL;
4300                 return -rte_errno;
4301         }
4302         mnl_attr_for_each(na, nlh, sizeof(*ifm))
4303                 if (mnl_attr_get_type(na) == IFLA_LINKINFO) {
4304                         na_info = na;
4305                         break;
4306                 }
4307         if (!na_info)
4308                 return 1;
4309         mnl_attr_for_each_nested(na, na_info) {
4310                 switch (mnl_attr_get_type(na)) {
4311                 case IFLA_INFO_KIND:
4312                         if (!strncmp("vxlan", mnl_attr_get_str(na),
4313                                      mnl_attr_get_len(na)))
4314                                 found = true;
4315                         break;
4316                 case IFLA_INFO_DATA:
4317                         na_vxlan = na;
4318                         break;
4319                 }
4320                 if (found && na_vxlan)
4321                         break;
4322         }
4323         if (!found || !na_vxlan)
4324                 return 1;
4325         found = false;
4326         mnl_attr_for_each_nested(na, na_vxlan) {
4327                 if (mnl_attr_get_type(na) == IFLA_VXLAN_LINK &&
4328                     mnl_attr_get_u32(na) == ctx->ifindex) {
4329                         found = true;
4330                         break;
4331                 }
4332         }
4333         if (!found)
4334                 return 1;
4335         /* Attached VXLAN device found, store the command to delete. */
4336         vxindex = ifm->ifi_index;
4337         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4338                MNL_ALIGN(sizeof(struct ifinfomsg));
4339         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4340         if (!cmd) {
4341                 rte_errno = ENOMEM;
4342                 return -rte_errno;
4343         }
4344         cmd = mnl_nlmsg_put_header(cmd);
4345         cmd->nlmsg_type = RTM_DELLINK;
4346         cmd->nlmsg_flags = NLM_F_REQUEST;
4347         ifm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifm));
4348         ifm->ifi_family = AF_UNSPEC;
4349         ifm->ifi_index = vxindex;
4350         assert(size == cmd->nlmsg_len);
4351         return 1;
4352 }
4353
4354 /**
4355  * Cleanup the outer interface. Removes all found vxlan devices
4356  * attached to specified index, flushes the meigh and local IP
4357  * datavase.
4358  *
4359  * @param[in] tcf
4360  *   Context object initialized by mlx5_flow_tcf_context_create().
4361  * @param[in] ifindex
4362  *   Network inferface index to perform cleanup.
4363  */
4364 static void
4365 flow_tcf_encap_iface_cleanup(struct mlx5_flow_tcf_context *tcf,
4366                             unsigned int ifindex)
4367 {
4368         struct nlmsghdr *nlh;
4369         struct ifinfomsg *ifm;
4370         struct tcf_nlcb_context ctx = {
4371                 .ifindex = ifindex,
4372                 .bufsize = MNL_REQUEST_SIZE,
4373                 .nlbuf = LIST_HEAD_INITIALIZER(),
4374         };
4375         int ret;
4376
4377         assert(ifindex);
4378         /*
4379          * Seek and destroy leftover VXLAN encap/decap interfaces with
4380          * matching properties.
4381          */
4382         nlh = mnl_nlmsg_put_header(tcf->buf);
4383         nlh->nlmsg_type = RTM_GETLINK;
4384         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4385         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4386         ifm->ifi_family = AF_UNSPEC;
4387         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_vxlan_cb, &ctx);
4388         if (ret)
4389                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4390         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4391         if (ret)
4392                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4393 }
4394
4395 /**
4396  * Emit Netlink message to add/remove local address to the outer device.
4397  * The address being added is visible within the link only (scope link).
4398  *
4399  * Note that an implicit route is maintained by the kernel due to the
4400  * presence of a peer address (IFA_ADDRESS).
4401  *
4402  * These rules are used for encapsultion only and allow to assign
4403  * the outer tunnel source IP address.
4404  *
4405  * @param[in] tcf
4406  *   Libmnl socket context object.
4407  * @param[in] encap
4408  *   Encapsulation properties (source address and its peer).
4409  * @param[in] ifindex
4410  *   Network interface to apply rule.
4411  * @param[in] enable
4412  *   Toggle between add and remove.
4413  * @param[out] error
4414  *   Perform verbose error reporting if not NULL.
4415  *
4416  * @return
4417  *   0 on success, a negative errno value otherwise and rte_errno is set.
4418  */
4419 static int
4420 flow_tcf_rule_local(struct mlx5_flow_tcf_context *tcf,
4421                     const struct flow_tcf_vxlan_encap *encap,
4422                     unsigned int ifindex,
4423                     bool enable,
4424                     struct rte_flow_error *error)
4425 {
4426         struct nlmsghdr *nlh;
4427         struct ifaddrmsg *ifa;
4428         alignas(struct nlmsghdr)
4429         uint8_t buf[mnl_nlmsg_size(sizeof(*ifa) + 128)];
4430
4431         nlh = mnl_nlmsg_put_header(buf);
4432         nlh->nlmsg_type = enable ? RTM_NEWADDR : RTM_DELADDR;
4433         nlh->nlmsg_flags =
4434                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4435         nlh->nlmsg_seq = 0;
4436         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4437         ifa->ifa_flags = IFA_F_PERMANENT;
4438         ifa->ifa_scope = RT_SCOPE_LINK;
4439         ifa->ifa_index = ifindex;
4440         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4441                 ifa->ifa_family = AF_INET;
4442                 ifa->ifa_prefixlen = 32;
4443                 mnl_attr_put_u32(nlh, IFA_LOCAL, encap->ipv4.src);
4444                 if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST)
4445                         mnl_attr_put_u32(nlh, IFA_ADDRESS,
4446                                               encap->ipv4.dst);
4447         } else {
4448                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4449                 ifa->ifa_family = AF_INET6;
4450                 ifa->ifa_prefixlen = 128;
4451                 mnl_attr_put(nlh, IFA_LOCAL,
4452                                   sizeof(encap->ipv6.src),
4453                                   &encap->ipv6.src);
4454                 if (encap->mask & FLOW_TCF_ENCAP_IPV6_DST)
4455                         mnl_attr_put(nlh, IFA_ADDRESS,
4456                                           sizeof(encap->ipv6.dst),
4457                                           &encap->ipv6.dst);
4458         }
4459         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4460                 return 0;
4461         return rte_flow_error_set(error, rte_errno,
4462                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4463                                   "netlink: cannot complete IFA request"
4464                                   " (ip addr add)");
4465 }
4466
4467 /**
4468  * Emit Netlink message to add/remove neighbor.
4469  *
4470  * @param[in] tcf
4471  *   Libmnl socket context object.
4472  * @param[in] encap
4473  *   Encapsulation properties (destination address).
4474  * @param[in] ifindex
4475  *   Network interface.
4476  * @param[in] enable
4477  *   Toggle between add and remove.
4478  * @param[out] error
4479  *   Perform verbose error reporting if not NULL.
4480  *
4481  * @return
4482  *   0 on success, a negative errno value otherwise and rte_errno is set.
4483  */
4484 static int
4485 flow_tcf_rule_neigh(struct mlx5_flow_tcf_context *tcf,
4486                      const struct flow_tcf_vxlan_encap *encap,
4487                      unsigned int ifindex,
4488                      bool enable,
4489                      struct rte_flow_error *error)
4490 {
4491         struct nlmsghdr *nlh;
4492         struct ndmsg *ndm;
4493         alignas(struct nlmsghdr)
4494         uint8_t buf[mnl_nlmsg_size(sizeof(*ndm) + 128)];
4495
4496         nlh = mnl_nlmsg_put_header(buf);
4497         nlh->nlmsg_type = enable ? RTM_NEWNEIGH : RTM_DELNEIGH;
4498         nlh->nlmsg_flags =
4499                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4500         nlh->nlmsg_seq = 0;
4501         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4502         ndm->ndm_ifindex = ifindex;
4503         ndm->ndm_state = NUD_PERMANENT;
4504         ndm->ndm_flags = 0;
4505         ndm->ndm_type = 0;
4506         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4507                 ndm->ndm_family = AF_INET;
4508                 mnl_attr_put_u32(nlh, NDA_DST, encap->ipv4.dst);
4509         } else {
4510                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4511                 ndm->ndm_family = AF_INET6;
4512                 mnl_attr_put(nlh, NDA_DST, sizeof(encap->ipv6.dst),
4513                                                  &encap->ipv6.dst);
4514         }
4515         if (encap->mask & FLOW_TCF_ENCAP_ETH_SRC && enable)
4516                 DRV_LOG(WARNING,
4517                         "outer ethernet source address cannot be "
4518                         "forced for VXLAN encapsulation");
4519         if (encap->mask & FLOW_TCF_ENCAP_ETH_DST)
4520                 mnl_attr_put(nlh, NDA_LLADDR, sizeof(encap->eth.dst),
4521                                                     &encap->eth.dst);
4522         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4523                 return 0;
4524         return rte_flow_error_set(error, rte_errno,
4525                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4526                                   "netlink: cannot complete ND request"
4527                                   " (ip neigh)");
4528 }
4529
4530 /**
4531  * Manage the local IP addresses and their peers IP addresses on the
4532  * outer interface for encapsulation purposes. The kernel searches the
4533  * appropriate device for tunnel egress traffic using the outer source
4534  * IP, this IP should be assigned to the outer network device, otherwise
4535  * kernel rejects the rule.
4536  *
4537  * Adds or removes the addresses using the Netlink command like this:
4538  *   ip addr add <src_ip> peer <dst_ip> scope link dev <ifouter>
4539  *
4540  * The addresses are local to the netdev ("scope link"), this reduces
4541  * the risk of conflicts. Note that an implicit route is maintained by
4542  * the kernel due to the presence of a peer address (IFA_ADDRESS).
4543  *
4544  * @param[in] tcf
4545  *   Libmnl socket context object.
4546  * @param[in] vtep
4547  *   VTEP object, contains rule database and ifouter index.
4548  * @param[in] dev_flow
4549  *   Flow object, contains the tunnel parameters (for encap only).
4550  * @param[in] enable
4551  *   Toggle between add and remove.
4552  * @param[out] error
4553  *   Perform verbose error reporting if not NULL.
4554  *
4555  * @return
4556  *   0 on success, a negative errno value otherwise and rte_errno is set.
4557  */
4558 static int
4559 flow_tcf_encap_local(struct mlx5_flow_tcf_context *tcf,
4560                      struct tcf_vtep *vtep,
4561                      struct mlx5_flow *dev_flow,
4562                      bool enable,
4563                      struct rte_flow_error *error)
4564 {
4565         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4566         struct tcf_local_rule *rule;
4567         bool found = false;
4568         int ret;
4569
4570         assert(encap);
4571         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4572         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4573                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_DST);
4574                 LIST_FOREACH(rule, &vtep->local, next) {
4575                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_SRC &&
4576                             encap->ipv4.src == rule->ipv4.src &&
4577                             encap->ipv4.dst == rule->ipv4.dst) {
4578                                 found = true;
4579                                 break;
4580                         }
4581                 }
4582         } else {
4583                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4584                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4585                 LIST_FOREACH(rule, &vtep->local, next) {
4586                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_SRC &&
4587                             !memcmp(&encap->ipv6.src, &rule->ipv6.src,
4588                                             sizeof(encap->ipv6.src)) &&
4589                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4590                                             sizeof(encap->ipv6.dst))) {
4591                                 found = true;
4592                                 break;
4593                         }
4594                 }
4595         }
4596         if (found) {
4597                 if (enable) {
4598                         rule->refcnt++;
4599                         return 0;
4600                 }
4601                 if (!rule->refcnt || !--rule->refcnt) {
4602                         LIST_REMOVE(rule, next);
4603                         return flow_tcf_rule_local(tcf, encap,
4604                                         vtep->ifouter, false, error);
4605                 }
4606                 return 0;
4607         }
4608         if (!enable) {
4609                 DRV_LOG(WARNING, "disabling not existing local rule");
4610                 rte_flow_error_set(error, ENOENT,
4611                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4612                                    "disabling not existing local rule");
4613                 return -ENOENT;
4614         }
4615         rule = rte_zmalloc(__func__, sizeof(struct tcf_local_rule),
4616                                 alignof(struct tcf_local_rule));
4617         if (!rule) {
4618                 rte_flow_error_set(error, ENOMEM,
4619                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4620                                    "unable to allocate memory for local rule");
4621                 return -rte_errno;
4622         }
4623         *rule = (struct tcf_local_rule){.refcnt = 0,
4624                                         .mask = 0,
4625                                         };
4626         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4627                 rule->mask = FLOW_TCF_ENCAP_IPV4_SRC
4628                            | FLOW_TCF_ENCAP_IPV4_DST;
4629                 rule->ipv4.src = encap->ipv4.src;
4630                 rule->ipv4.dst = encap->ipv4.dst;
4631         } else {
4632                 rule->mask = FLOW_TCF_ENCAP_IPV6_SRC
4633                            | FLOW_TCF_ENCAP_IPV6_DST;
4634                 memcpy(&rule->ipv6.src, &encap->ipv6.src, IPV6_ADDR_LEN);
4635                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4636         }
4637         ret = flow_tcf_rule_local(tcf, encap, vtep->ifouter, true, error);
4638         if (ret) {
4639                 rte_free(rule);
4640                 return ret;
4641         }
4642         rule->refcnt++;
4643         LIST_INSERT_HEAD(&vtep->local, rule, next);
4644         return 0;
4645 }
4646
4647 /**
4648  * Manage the destination MAC/IP addresses neigh database, kernel uses
4649  * this one to determine the destination MAC address within encapsulation
4650  * header. Adds or removes the entries using the Netlink command like this:
4651  *   ip neigh add dev <ifouter> lladdr <dst_mac> to <dst_ip> nud permanent
4652  *
4653  * @param[in] tcf
4654  *   Libmnl socket context object.
4655  * @param[in] vtep
4656  *   VTEP object, contains rule database and ifouter index.
4657  * @param[in] dev_flow
4658  *   Flow object, contains the tunnel parameters (for encap only).
4659  * @param[in] enable
4660  *   Toggle between add and remove.
4661  * @param[out] error
4662  *   Perform verbose error reporting if not NULL.
4663  *
4664  * @return
4665  *   0 on success, a negative errno value otherwise and rte_errno is set.
4666  */
4667 static int
4668 flow_tcf_encap_neigh(struct mlx5_flow_tcf_context *tcf,
4669                      struct tcf_vtep *vtep,
4670                      struct mlx5_flow *dev_flow,
4671                      bool enable,
4672                      struct rte_flow_error *error)
4673 {
4674         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4675         struct tcf_neigh_rule *rule;
4676         bool found = false;
4677         int ret;
4678
4679         assert(encap);
4680         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4681         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4682                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_SRC);
4683                 LIST_FOREACH(rule, &vtep->neigh, next) {
4684                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_DST &&
4685                             encap->ipv4.dst == rule->ipv4.dst) {
4686                                 found = true;
4687                                 break;
4688                         }
4689                 }
4690         } else {
4691                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4692                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4693                 LIST_FOREACH(rule, &vtep->neigh, next) {
4694                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_DST &&
4695                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4696                                                 sizeof(encap->ipv6.dst))) {
4697                                 found = true;
4698                                 break;
4699                         }
4700                 }
4701         }
4702         if (found) {
4703                 if (memcmp(&encap->eth.dst, &rule->eth,
4704                            sizeof(encap->eth.dst))) {
4705                         DRV_LOG(WARNING, "Destination MAC differs"
4706                                          " in neigh rule");
4707                         rte_flow_error_set(error, EEXIST,
4708                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4709                                            NULL, "Different MAC address"
4710                                            " neigh rule for the same"
4711                                            " destination IP");
4712                                         return -EEXIST;
4713                 }
4714                 if (enable) {
4715                         rule->refcnt++;
4716                         return 0;
4717                 }
4718                 if (!rule->refcnt || !--rule->refcnt) {
4719                         LIST_REMOVE(rule, next);
4720                         return flow_tcf_rule_neigh(tcf, encap,
4721                                                    vtep->ifouter,
4722                                                    false, error);
4723                 }
4724                 return 0;
4725         }
4726         if (!enable) {
4727                 DRV_LOG(WARNING, "Disabling not existing neigh rule");
4728                 rte_flow_error_set(error, ENOENT,
4729                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4730                                    "unable to allocate memory for neigh rule");
4731                 return -ENOENT;
4732         }
4733         rule = rte_zmalloc(__func__, sizeof(struct tcf_neigh_rule),
4734                                 alignof(struct tcf_neigh_rule));
4735         if (!rule) {
4736                 rte_flow_error_set(error, ENOMEM,
4737                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4738                                    "unable to allocate memory for neigh rule");
4739                 return -rte_errno;
4740         }
4741         *rule = (struct tcf_neigh_rule){.refcnt = 0,
4742                                         .mask = 0,
4743                                         };
4744         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4745                 rule->mask = FLOW_TCF_ENCAP_IPV4_DST;
4746                 rule->ipv4.dst = encap->ipv4.dst;
4747         } else {
4748                 rule->mask = FLOW_TCF_ENCAP_IPV6_DST;
4749                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4750         }
4751         memcpy(&rule->eth, &encap->eth.dst, sizeof(rule->eth));
4752         ret = flow_tcf_rule_neigh(tcf, encap, vtep->ifouter, true, error);
4753         if (ret) {
4754                 rte_free(rule);
4755                 return ret;
4756         }
4757         rule->refcnt++;
4758         LIST_INSERT_HEAD(&vtep->neigh, rule, next);
4759         return 0;
4760 }
4761
4762 /* VTEP device list is shared between PMD port instances. */
4763 static LIST_HEAD(, tcf_vtep) vtep_list_vxlan = LIST_HEAD_INITIALIZER();
4764 static pthread_mutex_t vtep_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4765
4766 /**
4767  * Deletes VTEP network device.
4768  *
4769  * @param[in] tcf
4770  *   Context object initialized by mlx5_flow_tcf_context_create().
4771  * @param[in] vtep
4772  *   Object represinting the network device to delete. Memory
4773  *   allocated for this object is freed by routine.
4774  */
4775 static void
4776 flow_tcf_vtep_delete(struct mlx5_flow_tcf_context *tcf,
4777                      struct tcf_vtep *vtep)
4778 {
4779         struct nlmsghdr *nlh;
4780         struct ifinfomsg *ifm;
4781         alignas(struct nlmsghdr)
4782         uint8_t buf[mnl_nlmsg_size(MNL_ALIGN(sizeof(*ifm))) +
4783                     MNL_BUF_EXTRA_SPACE];
4784         int ret;
4785
4786         assert(!vtep->refcnt);
4787         /* Delete only ifaces those we actually created. */
4788         if (vtep->created && vtep->ifindex) {
4789                 DRV_LOG(INFO, "VTEP delete (%d)", vtep->ifindex);
4790                 nlh = mnl_nlmsg_put_header(buf);
4791                 nlh->nlmsg_type = RTM_DELLINK;
4792                 nlh->nlmsg_flags = NLM_F_REQUEST;
4793                 ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4794                 ifm->ifi_family = AF_UNSPEC;
4795                 ifm->ifi_index = vtep->ifindex;
4796                 assert(sizeof(buf) >= nlh->nlmsg_len);
4797                 ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4798                 if (ret)
4799                         DRV_LOG(WARNING, "netlink: error deleting vxlan"
4800                                          " encap/decap ifindex %u",
4801                                          ifm->ifi_index);
4802         }
4803         rte_free(vtep);
4804 }
4805
4806 /**
4807  * Creates VTEP network device.
4808  *
4809  * @param[in] tcf
4810  *   Context object initialized by mlx5_flow_tcf_context_create().
4811  * @param[in] ifouter
4812  *   Outer interface to attach new-created VXLAN device
4813  *   If zero the VXLAN device will not be attached to any device.
4814  *   These VTEPs are used for decapsulation and can be precreated
4815  *   and shared between processes.
4816  * @param[in] port
4817  *   UDP port of created VTEP device.
4818  * @param[out] error
4819  *   Perform verbose error reporting if not NULL.
4820  *
4821  * @return
4822  * Pointer to created device structure on success,
4823  * NULL otherwise and rte_errno is set.
4824  */
4825 #ifdef HAVE_IFLA_VXLAN_COLLECT_METADATA
4826 static struct tcf_vtep*
4827 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf,
4828                      unsigned int ifouter,
4829                      uint16_t port, struct rte_flow_error *error)
4830 {
4831         struct tcf_vtep *vtep;
4832         struct nlmsghdr *nlh;
4833         struct ifinfomsg *ifm;
4834         char name[sizeof(MLX5_VXLAN_DEVICE_PFX) + 24];
4835         alignas(struct nlmsghdr)
4836         uint8_t buf[mnl_nlmsg_size(sizeof(*ifm)) +
4837                     SZ_NLATTR_DATA_OF(sizeof(name)) +
4838                     SZ_NLATTR_NEST * 2 +
4839                     SZ_NLATTR_STRZ_OF("vxlan") +
4840                     SZ_NLATTR_DATA_OF(sizeof(uint32_t)) +
4841                     SZ_NLATTR_DATA_OF(sizeof(uint16_t)) +
4842                     SZ_NLATTR_DATA_OF(sizeof(uint8_t)) * 3 +
4843                     MNL_BUF_EXTRA_SPACE];
4844         struct nlattr *na_info;
4845         struct nlattr *na_vxlan;
4846         rte_be16_t vxlan_port = rte_cpu_to_be_16(port);
4847         int ret;
4848
4849         vtep = rte_zmalloc(__func__, sizeof(*vtep), alignof(struct tcf_vtep));
4850         if (!vtep) {
4851                 rte_flow_error_set(error, ENOMEM,
4852                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4853                                    "unable to allocate memory for VTEP");
4854                 return NULL;
4855         }
4856         *vtep = (struct tcf_vtep){
4857                         .port = port,
4858                         .local = LIST_HEAD_INITIALIZER(),
4859                         .neigh = LIST_HEAD_INITIALIZER(),
4860         };
4861         memset(buf, 0, sizeof(buf));
4862         nlh = mnl_nlmsg_put_header(buf);
4863         nlh->nlmsg_type = RTM_NEWLINK;
4864         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE  | NLM_F_EXCL;
4865         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4866         ifm->ifi_family = AF_UNSPEC;
4867         ifm->ifi_type = 0;
4868         ifm->ifi_index = 0;
4869         ifm->ifi_flags = IFF_UP;
4870         ifm->ifi_change = 0xffffffff;
4871         snprintf(name, sizeof(name), "%s%u", MLX5_VXLAN_DEVICE_PFX, port);
4872         mnl_attr_put_strz(nlh, IFLA_IFNAME, name);
4873         na_info = mnl_attr_nest_start(nlh, IFLA_LINKINFO);
4874         assert(na_info);
4875         mnl_attr_put_strz(nlh, IFLA_INFO_KIND, "vxlan");
4876         na_vxlan = mnl_attr_nest_start(nlh, IFLA_INFO_DATA);
4877         if (ifouter)
4878                 mnl_attr_put_u32(nlh, IFLA_VXLAN_LINK, ifouter);
4879         assert(na_vxlan);
4880         mnl_attr_put_u8(nlh, IFLA_VXLAN_COLLECT_METADATA, 1);
4881         mnl_attr_put_u8(nlh, IFLA_VXLAN_UDP_ZERO_CSUM6_RX, 1);
4882         mnl_attr_put_u8(nlh, IFLA_VXLAN_LEARNING, 0);
4883         mnl_attr_put_u16(nlh, IFLA_VXLAN_PORT, vxlan_port);
4884         mnl_attr_nest_end(nlh, na_vxlan);
4885         mnl_attr_nest_end(nlh, na_info);
4886         assert(sizeof(buf) >= nlh->nlmsg_len);
4887         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4888         if (ret) {
4889                 DRV_LOG(WARNING,
4890                         "netlink: VTEP %s create failure (%d)",
4891                         name, rte_errno);
4892                 if (rte_errno != EEXIST || ifouter)
4893                         /*
4894                          * Some unhandled error occurred or device is
4895                          * for encapsulation and cannot be shared.
4896                          */
4897                         goto error;
4898         } else {
4899                 /*
4900                  * Mark device we actually created.
4901                  * We should explicitly delete
4902                  * when we do not need it anymore.
4903                  */
4904                 vtep->created = 1;
4905         }
4906         /* Try to get ifindex of created of pre-existing device. */
4907         ret = if_nametoindex(name);
4908         if (!ret) {
4909                 DRV_LOG(WARNING,
4910                         "VTEP %s failed to get index (%d)", name, errno);
4911                 rte_flow_error_set
4912                         (error, -errno,
4913                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4914                          "netlink: failed to retrieve VTEP ifindex");
4915                 goto error;
4916         }
4917         vtep->ifindex = ret;
4918         vtep->ifouter = ifouter;
4919         memset(buf, 0, sizeof(buf));
4920         nlh = mnl_nlmsg_put_header(buf);
4921         nlh->nlmsg_type = RTM_NEWLINK;
4922         nlh->nlmsg_flags = NLM_F_REQUEST;
4923         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4924         ifm->ifi_family = AF_UNSPEC;
4925         ifm->ifi_type = 0;
4926         ifm->ifi_index = vtep->ifindex;
4927         ifm->ifi_flags = IFF_UP;
4928         ifm->ifi_change = IFF_UP;
4929         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4930         if (ret) {
4931                 rte_flow_error_set(error, -errno,
4932                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4933                                    "netlink: failed to set VTEP link up");
4934                 DRV_LOG(WARNING, "netlink: VTEP %s set link up failure (%d)",
4935                         name, rte_errno);
4936                 goto clean;
4937         }
4938         ret = mlx5_flow_tcf_init(tcf, vtep->ifindex, error);
4939         if (ret) {
4940                 DRV_LOG(WARNING, "VTEP %s init failure (%d)", name, rte_errno);
4941                 goto clean;
4942         }
4943         DRV_LOG(INFO, "VTEP create (%d, %d)", vtep->port, vtep->ifindex);
4944         vtep->refcnt = 1;
4945         return vtep;
4946 clean:
4947         flow_tcf_vtep_delete(tcf, vtep);
4948         return NULL;
4949 error:
4950         rte_free(vtep);
4951         return NULL;
4952 }
4953 #else
4954 static struct tcf_vtep*
4955 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf __rte_unused,
4956                      unsigned int ifouter __rte_unused,
4957                      uint16_t port __rte_unused,
4958                      struct rte_flow_error *error)
4959 {
4960         rte_flow_error_set(error, ENOTSUP,
4961                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4962                            "netlink: failed to create VTEP, "
4963                            "vxlan metadata are not supported by kernel");
4964         return NULL;
4965 }
4966 #endif /* HAVE_IFLA_VXLAN_COLLECT_METADATA */
4967
4968 /**
4969  * Acquire target interface index for VXLAN tunneling decapsulation.
4970  * In order to share the UDP port within the other interfaces the
4971  * VXLAN device created as not attached to any interface (if created).
4972  *
4973  * @param[in] tcf
4974  *   Context object initialized by mlx5_flow_tcf_context_create().
4975  * @param[in] dev_flow
4976  *   Flow tcf object with tunnel structure pointer set.
4977  * @param[out] error
4978  *   Perform verbose error reporting if not NULL.
4979  * @return
4980  *   Interface descriptor pointer on success,
4981  *   NULL otherwise and rte_errno is set.
4982  */
4983 static struct tcf_vtep*
4984 flow_tcf_decap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4985                             struct mlx5_flow *dev_flow,
4986                             struct rte_flow_error *error)
4987 {
4988         struct tcf_vtep *vtep;
4989         uint16_t port = dev_flow->tcf.vxlan_decap->udp_port;
4990
4991         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4992                 if (vtep->port == port)
4993                         break;
4994         }
4995         if (vtep && vtep->ifouter) {
4996                 rte_flow_error_set(error, -errno,
4997                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4998                                    "Failed to create decap VTEP with specified"
4999                                    " UDP port, atatched device exists");
5000                 return NULL;
5001         }
5002         if (vtep) {
5003                 /* Device exists, just increment the reference counter. */
5004                 vtep->refcnt++;
5005                 assert(vtep->ifindex);
5006                 return vtep;
5007         }
5008         /* No decapsulation device exists, try to create the new one. */
5009         vtep = flow_tcf_vtep_create(tcf, 0, port, error);
5010         if (vtep)
5011                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
5012         return vtep;
5013 }
5014
5015 /**
5016  * Aqcuire target interface index for VXLAN tunneling encapsulation.
5017  *
5018  * @param[in] tcf
5019  *   Context object initialized by mlx5_flow_tcf_context_create().
5020  * @param[in] ifouter
5021  *   Network interface index to attach VXLAN encap device to.
5022  * @param[in] dev_flow
5023  *   Flow tcf object with tunnel structure pointer set.
5024  * @param[out] error
5025  *   Perform verbose error reporting if not NULL.
5026  * @return
5027  *   Interface descriptor pointer on success,
5028  *   NULL otherwise and rte_errno is set.
5029  */
5030 static struct tcf_vtep*
5031 flow_tcf_encap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
5032                             unsigned int ifouter,
5033                             struct mlx5_flow *dev_flow __rte_unused,
5034                             struct rte_flow_error *error)
5035 {
5036         static uint16_t encap_port = MLX5_VXLAN_PORT_MIN - 1;
5037         struct tcf_vtep *vtep;
5038         int ret;
5039
5040         assert(ifouter);
5041         /* Look whether the attached VTEP for encap is created. */
5042         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
5043                 if (vtep->ifouter == ifouter)
5044                         break;
5045         }
5046         if (vtep) {
5047                 /* VTEP already exists, just increment the reference. */
5048                 vtep->refcnt++;
5049         } else {
5050                 uint16_t pcnt;
5051
5052                 /* Not found, we should create the new attached VTEP. */
5053                 flow_tcf_encap_iface_cleanup(tcf, ifouter);
5054                 flow_tcf_encap_local_cleanup(tcf, ifouter);
5055                 flow_tcf_encap_neigh_cleanup(tcf, ifouter);
5056                 for (pcnt = 0; pcnt <= (MLX5_VXLAN_PORT_MAX
5057                                      - MLX5_VXLAN_PORT_MIN); pcnt++) {
5058                         encap_port++;
5059                         /* Wraparound the UDP port index. */
5060                         if (encap_port < MLX5_VXLAN_PORT_MIN ||
5061                             encap_port > MLX5_VXLAN_PORT_MAX)
5062                                 encap_port = MLX5_VXLAN_PORT_MIN;
5063                         /* Check whether UDP port is in already in use. */
5064                         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
5065                                 if (vtep->port == encap_port)
5066                                         break;
5067                         }
5068                         if (vtep) {
5069                                 /* Port is in use, try the next one. */
5070                                 vtep = NULL;
5071                                 continue;
5072                         }
5073                         vtep = flow_tcf_vtep_create(tcf, ifouter,
5074                                                     encap_port, error);
5075                         if (vtep) {
5076                                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
5077                                 break;
5078                         }
5079                         if (rte_errno != EEXIST)
5080                                 break;
5081                 }
5082                 if (!vtep)
5083                         return NULL;
5084         }
5085         assert(vtep->ifouter == ifouter);
5086         assert(vtep->ifindex);
5087         /* Create local ipaddr with peer to specify the outer IPs. */
5088         ret = flow_tcf_encap_local(tcf, vtep, dev_flow, true, error);
5089         if (!ret) {
5090                 /* Create neigh rule to specify outer destination MAC. */
5091                 ret = flow_tcf_encap_neigh(tcf, vtep, dev_flow, true, error);
5092                 if (ret)
5093                         flow_tcf_encap_local(tcf, vtep,
5094                                              dev_flow, false, error);
5095         }
5096         if (ret) {
5097                 if (--vtep->refcnt == 0)
5098                         flow_tcf_vtep_delete(tcf, vtep);
5099                 return NULL;
5100         }
5101         return vtep;
5102 }
5103
5104 /**
5105  * Acquires target interface index for tunneling of any type.
5106  * Creates the new VTEP if needed.
5107  *
5108  * @param[in] tcf
5109  *   Context object initialized by mlx5_flow_tcf_context_create().
5110  * @param[in] ifouter
5111  *   Network interface index to attach VXLAN encap device to.
5112  * @param[in] dev_flow
5113  *   Flow tcf object with tunnel structure pointer set.
5114  * @param[out] error
5115  *   Perform verbose error reporting if not NULL.
5116  * @return
5117  *   Interface descriptor pointer on success,
5118  *   NULL otherwise and rte_errno is set.
5119  */
5120 static struct tcf_vtep*
5121 flow_tcf_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
5122                       unsigned int ifouter,
5123                       struct mlx5_flow *dev_flow,
5124                       struct rte_flow_error *error)
5125 {
5126         struct tcf_vtep *vtep = NULL;
5127
5128         assert(dev_flow->tcf.tunnel);
5129         pthread_mutex_lock(&vtep_list_mutex);
5130         switch (dev_flow->tcf.tunnel->type) {
5131         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5132                 vtep = flow_tcf_encap_vtep_acquire(tcf, ifouter,
5133                                                   dev_flow, error);
5134                 break;
5135         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5136                 vtep = flow_tcf_decap_vtep_acquire(tcf, dev_flow, error);
5137                 break;
5138         default:
5139                 rte_flow_error_set(error, ENOTSUP,
5140                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5141                                    "unsupported tunnel type");
5142                 break;
5143         }
5144         pthread_mutex_unlock(&vtep_list_mutex);
5145         return vtep;
5146 }
5147
5148 /**
5149  * Release tunneling interface by ifindex. Decrements reference
5150  * counter and actually removes the device if counter is zero.
5151  *
5152  * @param[in] tcf
5153  *   Context object initialized by mlx5_flow_tcf_context_create().
5154  * @param[in] vtep
5155  *   VTEP device descriptor structure.
5156  * @param[in] dev_flow
5157  *   Flow tcf object with tunnel structure pointer set.
5158  */
5159 static void
5160 flow_tcf_vtep_release(struct mlx5_flow_tcf_context *tcf,
5161                       struct tcf_vtep *vtep,
5162                       struct mlx5_flow *dev_flow)
5163 {
5164         assert(dev_flow->tcf.tunnel);
5165         pthread_mutex_lock(&vtep_list_mutex);
5166         switch (dev_flow->tcf.tunnel->type) {
5167         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5168                 break;
5169         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5170                 /* Remove the encap ancillary rules first. */
5171                 flow_tcf_encap_neigh(tcf, vtep, dev_flow, false, NULL);
5172                 flow_tcf_encap_local(tcf, vtep, dev_flow, false, NULL);
5173                 break;
5174         default:
5175                 assert(false);
5176                 DRV_LOG(WARNING, "Unsupported tunnel type");
5177                 break;
5178         }
5179         assert(vtep->refcnt);
5180         if (--vtep->refcnt == 0) {
5181                 LIST_REMOVE(vtep, next);
5182                 flow_tcf_vtep_delete(tcf, vtep);
5183         }
5184         pthread_mutex_unlock(&vtep_list_mutex);
5185 }
5186
5187 struct tcf_nlcb_query {
5188         uint32_t handle;
5189         uint32_t tc_flags;
5190         uint32_t flags_valid:1;
5191 };
5192
5193 /**
5194  * Collect queried rule attributes. This is callback routine called by
5195  * libmnl mnl_cb_run() in loop for every message in received packet.
5196  * Current implementation collects the flower flags only.
5197  *
5198  * @param[in] nlh
5199  *   Pointer to reply header.
5200  * @param[in, out] arg
5201  *   Context pointer for this callback.
5202  *
5203  * @return
5204  *   A positive, nonzero value on success (required by libmnl
5205  *   to continue messages processing).
5206  */
5207 static int
5208 flow_tcf_collect_query_cb(const struct nlmsghdr *nlh, void *arg)
5209 {
5210         struct tcf_nlcb_query *query = arg;
5211         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
5212         struct nlattr *na, *na_opt;
5213         bool flower = false;
5214
5215         if (nlh->nlmsg_type != RTM_NEWTFILTER ||
5216             tcm->tcm_handle != query->handle)
5217                 return 1;
5218         mnl_attr_for_each(na, nlh, sizeof(*tcm)) {
5219                 switch (mnl_attr_get_type(na)) {
5220                 case TCA_KIND:
5221                         if (strcmp(mnl_attr_get_payload(na), "flower")) {
5222                                 /* Not flower filter, drop entire message. */
5223                                 return 1;
5224                         }
5225                         flower = true;
5226                         break;
5227                 case TCA_OPTIONS:
5228                         if (!flower) {
5229                                 /* Not flower options, drop entire message. */
5230                                 return 1;
5231                         }
5232                         /* Check nested flower options. */
5233                         mnl_attr_for_each_nested(na_opt, na) {
5234                                 switch (mnl_attr_get_type(na_opt)) {
5235                                 case TCA_FLOWER_FLAGS:
5236                                         query->flags_valid = 1;
5237                                         query->tc_flags =
5238                                                 mnl_attr_get_u32(na_opt);
5239                                         break;
5240                                 }
5241                         }
5242                         break;
5243                 }
5244         }
5245         return 1;
5246 }
5247
5248 /**
5249  * Query a TC flower rule flags via netlink.
5250  *
5251  * @param[in] tcf
5252  *   Context object initialized by mlx5_flow_tcf_context_create().
5253  * @param[in] dev_flow
5254  *   Pointer to the flow.
5255  * @param[out] pflags
5256  *   pointer to the data retrieved by the query.
5257  *
5258  * @return
5259  *   0 on success, a negative errno value otherwise.
5260  */
5261 static int
5262 flow_tcf_query_flags(struct mlx5_flow_tcf_context *tcf,
5263                      struct mlx5_flow *dev_flow,
5264                      uint32_t *pflags)
5265 {
5266         struct nlmsghdr *nlh;
5267         struct tcmsg *tcm;
5268         struct tcf_nlcb_query query = {
5269                 .handle = dev_flow->tcf.tcm->tcm_handle,
5270         };
5271
5272         nlh = mnl_nlmsg_put_header(tcf->buf);
5273         nlh->nlmsg_type = RTM_GETTFILTER;
5274         nlh->nlmsg_flags = NLM_F_REQUEST;
5275         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5276         memcpy(tcm, dev_flow->tcf.tcm, sizeof(*tcm));
5277         /*
5278          * Ignore Netlink error for filter query operations.
5279          * The reply length is sent by kernel as errno.
5280          * Just check we got the flags option.
5281          */
5282         flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_query_cb, &query);
5283         if (!query.flags_valid) {
5284                 *pflags = 0;
5285                 return -ENOENT;
5286         }
5287         *pflags = query.tc_flags;
5288         return 0;
5289 }
5290
5291 /**
5292  * Query and check the in_hw set for specified rule.
5293  *
5294  * @param[in] tcf
5295  *   Context object initialized by mlx5_flow_tcf_context_create().
5296  * @param[in] dev_flow
5297  *   Pointer to the flow to check.
5298  *
5299  * @return
5300  *   0 on success, a negative errno value otherwise.
5301  */
5302 static int
5303 flow_tcf_check_inhw(struct mlx5_flow_tcf_context *tcf,
5304                     struct mlx5_flow *dev_flow)
5305 {
5306         uint32_t flags;
5307         int ret;
5308
5309         ret = flow_tcf_query_flags(tcf, dev_flow, &flags);
5310         if (ret)
5311                 return ret;
5312         return  (flags & TCA_CLS_FLAGS_IN_HW) ? 0 : -ENOENT;
5313 }
5314
5315 /**
5316  * Remove flow from E-Switch by sending Netlink message.
5317  *
5318  * @param[in] dev
5319  *   Pointer to Ethernet device.
5320  * @param[in, out] flow
5321  *   Pointer to the sub flow.
5322  */
5323 static void
5324 flow_tcf_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5325 {
5326         struct priv *priv = dev->data->dev_private;
5327         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5328         struct mlx5_flow *dev_flow;
5329         struct nlmsghdr *nlh;
5330
5331         if (!flow)
5332                 return;
5333         dev_flow = LIST_FIRST(&flow->dev_flows);
5334         if (!dev_flow)
5335                 return;
5336         /* E-Switch flow can't be expanded. */
5337         assert(!LIST_NEXT(dev_flow, next));
5338         if (dev_flow->tcf.applied) {
5339                 nlh = dev_flow->tcf.nlh;
5340                 nlh->nlmsg_type = RTM_DELTFILTER;
5341                 nlh->nlmsg_flags = NLM_F_REQUEST;
5342                 flow_tcf_nl_ack(ctx, nlh, NULL, NULL);
5343                 if (dev_flow->tcf.tunnel) {
5344                         assert(dev_flow->tcf.tunnel->vtep);
5345                         flow_tcf_vtep_release(ctx,
5346                                 dev_flow->tcf.tunnel->vtep,
5347                                 dev_flow);
5348                         dev_flow->tcf.tunnel->vtep = NULL;
5349                 }
5350                 dev_flow->tcf.applied = 0;
5351         }
5352 }
5353
5354 /**
5355  * Apply flow to E-Switch by sending Netlink message.
5356  *
5357  * @param[in] dev
5358  *   Pointer to Ethernet device.
5359  * @param[in, out] flow
5360  *   Pointer to the sub flow.
5361  * @param[out] error
5362  *   Pointer to the error structure.
5363  *
5364  * @return
5365  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5366  */
5367 static int
5368 flow_tcf_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5369                struct rte_flow_error *error)
5370 {
5371         struct priv *priv = dev->data->dev_private;
5372         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5373         struct mlx5_flow *dev_flow;
5374         struct nlmsghdr *nlh;
5375
5376         dev_flow = LIST_FIRST(&flow->dev_flows);
5377         /* E-Switch flow can't be expanded. */
5378         assert(!LIST_NEXT(dev_flow, next));
5379         if (dev_flow->tcf.applied)
5380                 return 0;
5381         nlh = dev_flow->tcf.nlh;
5382         nlh->nlmsg_type = RTM_NEWTFILTER;
5383         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5384         if (dev_flow->tcf.tunnel) {
5385                 /*
5386                  * Replace the interface index, target for
5387                  * encapsulation, source for decapsulation.
5388                  */
5389                 assert(!dev_flow->tcf.tunnel->vtep);
5390                 assert(dev_flow->tcf.tunnel->ifindex_ptr);
5391                 /* Acquire actual VTEP device when rule is being applied. */
5392                 dev_flow->tcf.tunnel->vtep =
5393                         flow_tcf_vtep_acquire(ctx,
5394                                         dev_flow->tcf.tunnel->ifindex_org,
5395                                         dev_flow, error);
5396                 if (!dev_flow->tcf.tunnel->vtep)
5397                         return -rte_errno;
5398                 DRV_LOG(INFO, "Replace ifindex: %d->%d",
5399                                 dev_flow->tcf.tunnel->vtep->ifindex,
5400                                 dev_flow->tcf.tunnel->ifindex_org);
5401                 *dev_flow->tcf.tunnel->ifindex_ptr =
5402                         dev_flow->tcf.tunnel->vtep->ifindex;
5403         }
5404         if (!flow_tcf_nl_ack(ctx, nlh, NULL, NULL)) {
5405                 dev_flow->tcf.applied = 1;
5406                 if (*dev_flow->tcf.ptc_flags & TCA_CLS_FLAGS_SKIP_SW)
5407                         return 0;
5408                 /*
5409                  * Rule was applied without skip_sw flag set.
5410                  * We should check whether the rule was acctually
5411                  * accepted by hardware (have look at in_hw flag).
5412                  */
5413                 if (flow_tcf_check_inhw(ctx, dev_flow)) {
5414                         flow_tcf_remove(dev, flow);
5415                         return rte_flow_error_set
5416                                 (error, ENOENT,
5417                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5418                                  "netlink: rule has no in_hw flag set");
5419                 }
5420                 return 0;
5421         }
5422         if (dev_flow->tcf.tunnel) {
5423                 /* Rollback the VTEP configuration if rule apply failed. */
5424                 assert(dev_flow->tcf.tunnel->vtep);
5425                 flow_tcf_vtep_release(ctx, dev_flow->tcf.tunnel->vtep,
5426                                       dev_flow);
5427                 dev_flow->tcf.tunnel->vtep = NULL;
5428         }
5429         return rte_flow_error_set(error, rte_errno,
5430                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5431                                   "netlink: failed to create TC flow rule");
5432 }
5433
5434 /**
5435  * Remove flow from E-Switch and release resources of the device flow.
5436  *
5437  * @param[in] dev
5438  *   Pointer to Ethernet device.
5439  * @param[in, out] flow
5440  *   Pointer to the sub flow.
5441  */
5442 static void
5443 flow_tcf_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5444 {
5445         struct mlx5_flow *dev_flow;
5446
5447         if (!flow)
5448                 return;
5449         flow_tcf_remove(dev, flow);
5450         if (flow->counter) {
5451                 if (--flow->counter->ref_cnt == 0) {
5452                         rte_free(flow->counter);
5453                         flow->counter = NULL;
5454                 }
5455         }
5456         dev_flow = LIST_FIRST(&flow->dev_flows);
5457         if (!dev_flow)
5458                 return;
5459         /* E-Switch flow can't be expanded. */
5460         assert(!LIST_NEXT(dev_flow, next));
5461         LIST_REMOVE(dev_flow, next);
5462         rte_free(dev_flow);
5463 }
5464
5465 /**
5466  * Helper routine for figuring the space size required for a parse buffer.
5467  *
5468  * @param array
5469  *   array of values to use.
5470  * @param idx
5471  *   Current location in array.
5472  * @param value
5473  *   Value to compare with.
5474  *
5475  * @return
5476  *   The maximum between the given value and the array value on index.
5477  */
5478 static uint16_t
5479 flow_tcf_arr_val_max(uint16_t array[], int idx, uint16_t value)
5480 {
5481         return idx < 0 ? (value) : RTE_MAX((array)[idx], value);
5482 }
5483
5484 /**
5485  * Parse rtnetlink message attributes filling the attribute table with the info
5486  * retrieved.
5487  *
5488  * @param tb
5489  *   Attribute table to be filled.
5490  * @param[out] max
5491  *   Maxinum entry in the attribute table.
5492  * @param rte
5493  *   The attributes section in the message to be parsed.
5494  * @param len
5495  *   The length of the attributes section in the message.
5496  */
5497 static void
5498 flow_tcf_nl_parse_rtattr(struct rtattr *tb[], int max,
5499                          struct rtattr *rta, int len)
5500 {
5501         unsigned short type;
5502         memset(tb, 0, sizeof(struct rtattr *) * (max + 1));
5503         while (RTA_OK(rta, len)) {
5504                 type = rta->rta_type;
5505                 if (type <= max && !tb[type])
5506                         tb[type] = rta;
5507                 rta = RTA_NEXT(rta, len);
5508         }
5509 }
5510
5511 /**
5512  * Extract flow counters from flower action.
5513  *
5514  * @param rta
5515  *   flower action stats properties in the Netlink message received.
5516  * @param rta_type
5517  *   The backward sequence of rta_types, as written in the attribute table,
5518  *   we need to traverse in order to get to the requested object.
5519  * @param idx
5520  *   Current location in rta_type table.
5521  * @param[out] data
5522  *   data holding the count statistics of the rte_flow retrieved from
5523  *   the message.
5524  *
5525  * @return
5526  *   0 if data was found and retrieved, -1 otherwise.
5527  */
5528 static int
5529 flow_tcf_nl_action_stats_parse_and_get(struct rtattr *rta,
5530                                        uint16_t rta_type[], int idx,
5531                                        struct gnet_stats_basic *data)
5532 {
5533         int tca_stats_max = flow_tcf_arr_val_max(rta_type, idx,
5534                                                  TCA_STATS_BASIC);
5535         struct rtattr *tbs[tca_stats_max + 1];
5536
5537         if (rta == NULL || idx < 0)
5538                 return -1;
5539         flow_tcf_nl_parse_rtattr(tbs, tca_stats_max,
5540                                  RTA_DATA(rta), RTA_PAYLOAD(rta));
5541         switch (rta_type[idx]) {
5542         case TCA_STATS_BASIC:
5543                 if (tbs[TCA_STATS_BASIC]) {
5544                         memcpy(data, RTA_DATA(tbs[TCA_STATS_BASIC]),
5545                                RTE_MIN(RTA_PAYLOAD(tbs[TCA_STATS_BASIC]),
5546                                sizeof(*data)));
5547                         return 0;
5548                 }
5549                 break;
5550         default:
5551                 break;
5552         }
5553         return -1;
5554 }
5555
5556 /**
5557  * Parse flower single action retrieving the requested action attribute,
5558  * if found.
5559  *
5560  * @param arg
5561  *   flower action properties in the Netlink message received.
5562  * @param rta_type
5563  *   The backward sequence of rta_types, as written in the attribute table,
5564  *   we need to traverse in order to get to the requested object.
5565  * @param idx
5566  *   Current location in rta_type table.
5567  * @param[out] data
5568  *   Count statistics retrieved from the message query.
5569  *
5570  * @return
5571  *   0 if data was found and retrieved, -1 otherwise.
5572  */
5573 static int
5574 flow_tcf_nl_parse_one_action_and_get(struct rtattr *arg,
5575                                      uint16_t rta_type[], int idx, void *data)
5576 {
5577         int tca_act_max = flow_tcf_arr_val_max(rta_type, idx, TCA_ACT_STATS);
5578         struct rtattr *tb[tca_act_max + 1];
5579
5580         if (arg == NULL || idx < 0)
5581                 return -1;
5582         flow_tcf_nl_parse_rtattr(tb, tca_act_max,
5583                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5584         if (tb[TCA_ACT_KIND] == NULL)
5585                 return -1;
5586         switch (rta_type[idx]) {
5587         case TCA_ACT_STATS:
5588                 if (tb[TCA_ACT_STATS])
5589                         return flow_tcf_nl_action_stats_parse_and_get
5590                                         (tb[TCA_ACT_STATS],
5591                                          rta_type, --idx,
5592                                          (struct gnet_stats_basic *)data);
5593                 break;
5594         default:
5595                 break;
5596         }
5597         return -1;
5598 }
5599
5600 /**
5601  * Parse flower action section in the message retrieving the requested
5602  * attribute from the first action that provides it.
5603  *
5604  * @param opt
5605  *   flower section in the Netlink message received.
5606  * @param rta_type
5607  *   The backward sequence of rta_types, as written in the attribute table,
5608  *   we need to traverse in order to get to the requested object.
5609  * @param idx
5610  *   Current location in rta_type table.
5611  * @param[out] data
5612  *   data retrieved from the message query.
5613  *
5614  * @return
5615  *   0 if data was found and retrieved, -1 otherwise.
5616  */
5617 static int
5618 flow_tcf_nl_action_parse_and_get(struct rtattr *arg,
5619                                  uint16_t rta_type[], int idx, void *data)
5620 {
5621         struct rtattr *tb[TCA_ACT_MAX_PRIO + 1];
5622         int i;
5623
5624         if (arg == NULL || idx < 0)
5625                 return -1;
5626         flow_tcf_nl_parse_rtattr(tb, TCA_ACT_MAX_PRIO,
5627                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5628         switch (rta_type[idx]) {
5629         /*
5630          * flow counters are stored in the actions defined by the flow
5631          * and not in the flow itself, therefore we need to traverse the
5632          * flower chain of actions in search for them.
5633          *
5634          * Note that the index is not decremented here.
5635          */
5636         case TCA_ACT_STATS:
5637                 for (i = 0; i <= TCA_ACT_MAX_PRIO; i++) {
5638                         if (tb[i] &&
5639                         !flow_tcf_nl_parse_one_action_and_get(tb[i],
5640                                                               rta_type,
5641                                                               idx, data))
5642                                 return 0;
5643                 }
5644                 break;
5645         default:
5646                 break;
5647         }
5648         return -1;
5649 }
5650
5651 /**
5652  * Parse flower classifier options in the message, retrieving the requested
5653  * attribute if found.
5654  *
5655  * @param opt
5656  *   flower section in the Netlink message received.
5657  * @param rta_type
5658  *   The backward sequence of rta_types, as written in the attribute table,
5659  *   we need to traverse in order to get to the requested object.
5660  * @param idx
5661  *   Current location in rta_type table.
5662  * @param[out] data
5663  *   data retrieved from the message query.
5664  *
5665  * @return
5666  *   0 if data was found and retrieved, -1 otherwise.
5667  */
5668 static int
5669 flow_tcf_nl_opts_parse_and_get(struct rtattr *opt,
5670                                uint16_t rta_type[], int idx, void *data)
5671 {
5672         int tca_flower_max = flow_tcf_arr_val_max(rta_type, idx,
5673                                                   TCA_FLOWER_ACT);
5674         struct rtattr *tb[tca_flower_max + 1];
5675
5676         if (!opt || idx < 0)
5677                 return -1;
5678         flow_tcf_nl_parse_rtattr(tb, tca_flower_max,
5679                                  RTA_DATA(opt), RTA_PAYLOAD(opt));
5680         switch (rta_type[idx]) {
5681         case TCA_FLOWER_ACT:
5682                 if (tb[TCA_FLOWER_ACT])
5683                         return flow_tcf_nl_action_parse_and_get
5684                                                         (tb[TCA_FLOWER_ACT],
5685                                                          rta_type, --idx, data);
5686                 break;
5687         default:
5688                 break;
5689         }
5690         return -1;
5691 }
5692
5693 /**
5694  * Parse Netlink reply on filter query, retrieving the flow counters.
5695  *
5696  * @param nlh
5697  *   Message received from Netlink.
5698  * @param rta_type
5699  *   The backward sequence of rta_types, as written in the attribute table,
5700  *   we need to traverse in order to get to the requested object.
5701  * @param idx
5702  *   Current location in rta_type table.
5703  * @param[out] data
5704  *   data retrieved from the message query.
5705  *
5706  * @return
5707  *   0 if data was found and retrieved, -1 otherwise.
5708  */
5709 static int
5710 flow_tcf_nl_filter_parse_and_get(struct nlmsghdr *cnlh,
5711                                  uint16_t rta_type[], int idx, void *data)
5712 {
5713         struct nlmsghdr *nlh = cnlh;
5714         struct tcmsg *t = NLMSG_DATA(nlh);
5715         int len = nlh->nlmsg_len;
5716         int tca_max = flow_tcf_arr_val_max(rta_type, idx, TCA_OPTIONS);
5717         struct rtattr *tb[tca_max + 1];
5718
5719         if (idx < 0)
5720                 return -1;
5721         if (nlh->nlmsg_type != RTM_NEWTFILTER &&
5722             nlh->nlmsg_type != RTM_GETTFILTER &&
5723             nlh->nlmsg_type != RTM_DELTFILTER)
5724                 return -1;
5725         len -= NLMSG_LENGTH(sizeof(*t));
5726         if (len < 0)
5727                 return -1;
5728         flow_tcf_nl_parse_rtattr(tb, tca_max, TCA_RTA(t), len);
5729         /* Not a TC flower flow - bail out */
5730         if (!tb[TCA_KIND] ||
5731             strcmp(RTA_DATA(tb[TCA_KIND]), "flower"))
5732                 return -1;
5733         switch (rta_type[idx]) {
5734         case TCA_OPTIONS:
5735                 if (tb[TCA_OPTIONS])
5736                         return flow_tcf_nl_opts_parse_and_get(tb[TCA_OPTIONS],
5737                                                               rta_type,
5738                                                               --idx, data);
5739                 break;
5740         default:
5741                 break;
5742         }
5743         return -1;
5744 }
5745
5746 /**
5747  * A callback to parse Netlink reply on TC flower query.
5748  *
5749  * @param nlh
5750  *   Message received from Netlink.
5751  * @param[out] data
5752  *   Pointer to data area to be filled by the parsing routine.
5753  *   assumed to be a pointer to struct flow_tcf_stats_basic.
5754  *
5755  * @return
5756  *   MNL_CB_OK value.
5757  */
5758 static int
5759 flow_tcf_nl_message_get_stats_basic(const struct nlmsghdr *nlh, void *data)
5760 {
5761         /*
5762          * The backward sequence of rta_types to pass in order to get
5763          *  to the counters.
5764          */
5765         uint16_t rta_type[] = { TCA_STATS_BASIC, TCA_ACT_STATS,
5766                                 TCA_FLOWER_ACT, TCA_OPTIONS };
5767         struct flow_tcf_stats_basic *sb_data = data;
5768         union {
5769                 const struct nlmsghdr *c;
5770                 struct nlmsghdr *nc;
5771         } tnlh = { .c = nlh };
5772
5773         if (!flow_tcf_nl_filter_parse_and_get(tnlh.nc, rta_type,
5774                                               RTE_DIM(rta_type) - 1,
5775                                               (void *)&sb_data->counters))
5776                 sb_data->valid = true;
5777         return MNL_CB_OK;
5778 }
5779
5780 /**
5781  * Query a TC flower rule for its statistics via netlink.
5782  *
5783  * @param[in] dev
5784  *   Pointer to Ethernet device.
5785  * @param[in] flow
5786  *   Pointer to the sub flow.
5787  * @param[out] data
5788  *   data retrieved by the query.
5789  * @param[out] error
5790  *   Perform verbose error reporting if not NULL.
5791  *
5792  * @return
5793  *   0 on success, a negative errno value otherwise and rte_errno is set.
5794  */
5795 static int
5796 flow_tcf_query_count(struct rte_eth_dev *dev,
5797                           struct rte_flow *flow,
5798                           void *data,
5799                           struct rte_flow_error *error)
5800 {
5801         struct flow_tcf_stats_basic sb_data;
5802         struct rte_flow_query_count *qc = data;
5803         struct priv *priv = dev->data->dev_private;
5804         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5805         struct mnl_socket *nl = ctx->nl;
5806         struct mlx5_flow *dev_flow;
5807         struct nlmsghdr *nlh;
5808         uint32_t seq = priv->tcf_context->seq++;
5809         ssize_t ret;
5810         assert(qc);
5811
5812         memset(&sb_data, 0, sizeof(sb_data));
5813         dev_flow = LIST_FIRST(&flow->dev_flows);
5814         /* E-Switch flow can't be expanded. */
5815         assert(!LIST_NEXT(dev_flow, next));
5816         if (!dev_flow->flow->counter)
5817                 goto notsup_exit;
5818         nlh = dev_flow->tcf.nlh;
5819         nlh->nlmsg_type = RTM_GETTFILTER;
5820         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_ECHO;
5821         nlh->nlmsg_seq = seq;
5822         if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) == -1)
5823                 goto error_exit;
5824         do {
5825                 ret = mnl_socket_recvfrom(nl, ctx->buf, ctx->buf_size);
5826                 if (ret <= 0)
5827                         break;
5828                 ret = mnl_cb_run(ctx->buf, ret, seq,
5829                                  mnl_socket_get_portid(nl),
5830                                  flow_tcf_nl_message_get_stats_basic,
5831                                  (void *)&sb_data);
5832         } while (ret > 0);
5833         /* Return the delta from last reset. */
5834         if (sb_data.valid) {
5835                 /* Return the delta from last reset. */
5836                 qc->hits_set = 1;
5837                 qc->bytes_set = 1;
5838                 qc->hits = sb_data.counters.packets - flow->counter->hits;
5839                 qc->bytes = sb_data.counters.bytes - flow->counter->bytes;
5840                 if (qc->reset) {
5841                         flow->counter->hits = sb_data.counters.packets;
5842                         flow->counter->bytes = sb_data.counters.bytes;
5843                 }
5844                 return 0;
5845         }
5846         return rte_flow_error_set(error, EINVAL,
5847                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5848                                   NULL,
5849                                   "flow does not have counter");
5850 error_exit:
5851         return rte_flow_error_set
5852                         (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5853                          NULL, "netlink: failed to read flow rule counters");
5854 notsup_exit:
5855         return rte_flow_error_set
5856                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5857                          NULL, "counters are not available.");
5858 }
5859
5860 /**
5861  * Query a flow.
5862  *
5863  * @see rte_flow_query()
5864  * @see rte_flow_ops
5865  */
5866 static int
5867 flow_tcf_query(struct rte_eth_dev *dev,
5868                struct rte_flow *flow,
5869                const struct rte_flow_action *actions,
5870                void *data,
5871                struct rte_flow_error *error)
5872 {
5873         int ret = -EINVAL;
5874
5875         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5876                 switch (actions->type) {
5877                 case RTE_FLOW_ACTION_TYPE_VOID:
5878                         break;
5879                 case RTE_FLOW_ACTION_TYPE_COUNT:
5880                         ret = flow_tcf_query_count(dev, flow, data, error);
5881                         break;
5882                 default:
5883                         return rte_flow_error_set(error, ENOTSUP,
5884                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5885                                                   actions,
5886                                                   "action not supported");
5887                 }
5888         }
5889         return ret;
5890 }
5891
5892 const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops = {
5893         .validate = flow_tcf_validate,
5894         .prepare = flow_tcf_prepare,
5895         .translate = flow_tcf_translate,
5896         .apply = flow_tcf_apply,
5897         .remove = flow_tcf_remove,
5898         .destroy = flow_tcf_destroy,
5899         .query = flow_tcf_query,
5900 };
5901
5902 /**
5903  * Create and configure a libmnl socket for Netlink flow rules.
5904  *
5905  * @return
5906  *   A valid libmnl socket object pointer on success, NULL otherwise and
5907  *   rte_errno is set.
5908  */
5909 static struct mnl_socket *
5910 flow_tcf_mnl_socket_create(void)
5911 {
5912         struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
5913
5914         if (nl) {
5915                 mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
5916                                       sizeof(int));
5917                 if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
5918                         return nl;
5919         }
5920         rte_errno = errno;
5921         if (nl)
5922                 mnl_socket_close(nl);
5923         return NULL;
5924 }
5925
5926 /**
5927  * Destroy a libmnl socket.
5928  *
5929  * @param nl
5930  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5931  */
5932 static void
5933 flow_tcf_mnl_socket_destroy(struct mnl_socket *nl)
5934 {
5935         if (nl)
5936                 mnl_socket_close(nl);
5937 }
5938
5939 /**
5940  * Initialize ingress qdisc of a given network interface.
5941  *
5942  * @param ctx
5943  *   Pointer to tc-flower context to use.
5944  * @param ifindex
5945  *   Index of network interface to initialize.
5946  * @param[out] error
5947  *   Perform verbose error reporting if not NULL.
5948  *
5949  * @return
5950  *   0 on success, a negative errno value otherwise and rte_errno is set.
5951  */
5952 int
5953 mlx5_flow_tcf_init(struct mlx5_flow_tcf_context *ctx,
5954                    unsigned int ifindex, struct rte_flow_error *error)
5955 {
5956         struct nlmsghdr *nlh;
5957         struct tcmsg *tcm;
5958         alignas(struct nlmsghdr)
5959         uint8_t buf[mnl_nlmsg_size(sizeof(*tcm)) +
5960                     SZ_NLATTR_STRZ_OF("ingress") +
5961                     MNL_BUF_EXTRA_SPACE];
5962
5963         /* Destroy existing ingress qdisc and everything attached to it. */
5964         nlh = mnl_nlmsg_put_header(buf);
5965         nlh->nlmsg_type = RTM_DELQDISC;
5966         nlh->nlmsg_flags = NLM_F_REQUEST;
5967         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5968         tcm->tcm_family = AF_UNSPEC;
5969         tcm->tcm_ifindex = ifindex;
5970         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5971         tcm->tcm_parent = TC_H_INGRESS;
5972         assert(sizeof(buf) >= nlh->nlmsg_len);
5973         /* Ignore errors when qdisc is already absent. */
5974         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL) &&
5975             rte_errno != EINVAL && rte_errno != ENOENT)
5976                 return rte_flow_error_set(error, rte_errno,
5977                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5978                                           "netlink: failed to remove ingress"
5979                                           " qdisc");
5980         /* Create fresh ingress qdisc. */
5981         nlh = mnl_nlmsg_put_header(buf);
5982         nlh->nlmsg_type = RTM_NEWQDISC;
5983         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5984         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5985         tcm->tcm_family = AF_UNSPEC;
5986         tcm->tcm_ifindex = ifindex;
5987         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5988         tcm->tcm_parent = TC_H_INGRESS;
5989         mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
5990         assert(sizeof(buf) >= nlh->nlmsg_len);
5991         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL))
5992                 return rte_flow_error_set(error, rte_errno,
5993                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5994                                           "netlink: failed to create ingress"
5995                                           " qdisc");
5996         return 0;
5997 }
5998
5999 /**
6000  * Create libmnl context for Netlink flow rules.
6001  *
6002  * @return
6003  *   A valid libmnl socket object pointer on success, NULL otherwise and
6004  *   rte_errno is set.
6005  */
6006 struct mlx5_flow_tcf_context *
6007 mlx5_flow_tcf_context_create(void)
6008 {
6009         struct mlx5_flow_tcf_context *ctx = rte_zmalloc(__func__,
6010                                                         sizeof(*ctx),
6011                                                         sizeof(uint32_t));
6012         if (!ctx)
6013                 goto error;
6014         ctx->nl = flow_tcf_mnl_socket_create();
6015         if (!ctx->nl)
6016                 goto error;
6017         ctx->buf_size = MNL_SOCKET_BUFFER_SIZE;
6018         ctx->buf = rte_zmalloc(__func__,
6019                                ctx->buf_size, sizeof(uint32_t));
6020         if (!ctx->buf)
6021                 goto error;
6022         ctx->seq = random();
6023         return ctx;
6024 error:
6025         mlx5_flow_tcf_context_destroy(ctx);
6026         return NULL;
6027 }
6028
6029 /**
6030  * Destroy a libmnl context.
6031  *
6032  * @param ctx
6033  *   Libmnl socket of the @p NETLINK_ROUTE kind.
6034  */
6035 void
6036 mlx5_flow_tcf_context_destroy(struct mlx5_flow_tcf_context *ctx)
6037 {
6038         if (!ctx)
6039                 return;
6040         flow_tcf_mnl_socket_destroy(ctx->nl);
6041         rte_free(ctx->buf);
6042         rte_free(ctx);
6043 }