drivers/net/mlx5/mlx5_flow_tcf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <assert.h>
   7 #include <errno.h>
   8 #include <libmnl/libmnl.h>
   9 #include <linux/gen_stats.h>
  10 #include <linux/if_ether.h>
  11 #include <linux/netlink.h>
  12 #include <linux/pkt_cls.h>
  13 #include <linux/pkt_sched.h>
  14 #include <linux/rtnetlink.h>
  15 #include <linux/tc_act/tc_gact.h>
  16 #include <linux/tc_act/tc_mirred.h>
  17 #include <netinet/in.h>
  18 #include <stdalign.h>
  19 #include <stdbool.h>
  20 #include <stddef.h>
  21 #include <stdint.h>
  22 #include <stdlib.h>
  23 #include <sys/socket.h>
  24
  25 #include <rte_byteorder.h>
  26 #include <rte_errno.h>
  27 #include <rte_ether.h>
  28 #include <rte_flow.h>
  29 #include <rte_malloc.h>
  30 #include <rte_common.h>
  31
  32 #include "mlx5.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_autoconf.h"
  35
  36 #ifdef HAVE_TC_ACT_VLAN
  37
  38 #include <linux/tc_act/tc_vlan.h>
  39
  40 #else /* HAVE_TC_ACT_VLAN */
  41
  42 #define TCA_VLAN_ACT_POP 1
  43 #define TCA_VLAN_ACT_PUSH 2
  44 #define TCA_VLAN_ACT_MODIFY 3
  45 #define TCA_VLAN_PARMS 2
  46 #define TCA_VLAN_PUSH_VLAN_ID 3
  47 #define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
  48 #define TCA_VLAN_PAD 5
  49 #define TCA_VLAN_PUSH_VLAN_PRIORITY 6
  50
  51 struct tc_vlan {
  52         tc_gen;
  53         int v_action;
  54 };
  55
  56 #endif /* HAVE_TC_ACT_VLAN */
  57
  58 #ifdef HAVE_TC_ACT_PEDIT
  59
  60 #include <linux/tc_act/tc_pedit.h>
  61
  62 #else /* HAVE_TC_ACT_VLAN */
  63
  64 enum {
  65         TCA_PEDIT_UNSPEC,
  66         TCA_PEDIT_TM,
  67         TCA_PEDIT_PARMS,
  68         TCA_PEDIT_PAD,
  69         TCA_PEDIT_PARMS_EX,
  70         TCA_PEDIT_KEYS_EX,
  71         TCA_PEDIT_KEY_EX,
  72         __TCA_PEDIT_MAX
  73 };
  74
  75 enum {
  76         TCA_PEDIT_KEY_EX_HTYPE = 1,
  77         TCA_PEDIT_KEY_EX_CMD = 2,
  78         __TCA_PEDIT_KEY_EX_MAX
  79 };
  80
  81 enum pedit_header_type {
  82         TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK = 0,
  83         TCA_PEDIT_KEY_EX_HDR_TYPE_ETH = 1,
  84         TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 = 2,
  85         TCA_PEDIT_KEY_EX_HDR_TYPE_IP6 = 3,
  86         TCA_PEDIT_KEY_EX_HDR_TYPE_TCP = 4,
  87         TCA_PEDIT_KEY_EX_HDR_TYPE_UDP = 5,
  88         __PEDIT_HDR_TYPE_MAX,
  89 };
  90
  91 enum pedit_cmd {
  92         TCA_PEDIT_KEY_EX_CMD_SET = 0,
  93         TCA_PEDIT_KEY_EX_CMD_ADD = 1,
  94         __PEDIT_CMD_MAX,
  95 };
  96
  97 struct tc_pedit_key {
  98         __u32 mask; /* AND */
  99         __u32 val; /*XOR */
 100         __u32 off; /*offset */
 101         __u32 at;
 102         __u32 offmask;
 103         __u32 shift;
 104 };
 105
 106 __extension__
 107 struct tc_pedit_sel {
 108         tc_gen;
 109         unsigned char nkeys;
 110         unsigned char flags;
 111         struct tc_pedit_key keys[0];
 112 };
 113
 114 #endif /* HAVE_TC_ACT_VLAN */
 115
 116 #ifdef HAVE_TC_ACT_TUNNEL_KEY
 117
 118 #include <linux/tc_act/tc_tunnel_key.h>
 119
 120 #ifndef HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT
 121 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 122 #endif
 123
 124 #ifndef HAVE_TCA_TUNNEL_KEY_NO_CSUM
 125 #define TCA_TUNNEL_KEY_NO_CSUM 10
 126 #endif
 127
 128 #else /* HAVE_TC_ACT_TUNNEL_KEY */
 129
 130 #define TCA_ACT_TUNNEL_KEY 17
 131 #define TCA_TUNNEL_KEY_ACT_SET 1
 132 #define TCA_TUNNEL_KEY_ACT_RELEASE 2
 133 #define TCA_TUNNEL_KEY_PARMS 2
 134 #define TCA_TUNNEL_KEY_ENC_IPV4_SRC 3
 135 #define TCA_TUNNEL_KEY_ENC_IPV4_DST 4
 136 #define TCA_TUNNEL_KEY_ENC_IPV6_SRC 5
 137 #define TCA_TUNNEL_KEY_ENC_IPV6_DST 6
 138 #define TCA_TUNNEL_KEY_ENC_KEY_ID 7
 139 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 140 #define TCA_TUNNEL_KEY_NO_CSUM 10
 141
 142 struct tc_tunnel_key {
 143         tc_gen;
 144         int t_action;
 145 };
 146
 147 #endif /* HAVE_TC_ACT_TUNNEL_KEY */
 148
 149 /* Normally found in linux/netlink.h. */
 150 #ifndef NETLINK_CAP_ACK
 151 #define NETLINK_CAP_ACK 10
 152 #endif
 153
 154 /* Normally found in linux/pkt_sched.h. */
 155 #ifndef TC_H_MIN_INGRESS
 156 #define TC_H_MIN_INGRESS 0xfff2u
 157 #endif
 158
 159 /* Normally found in linux/pkt_cls.h. */
 160 #ifndef TCA_CLS_FLAGS_SKIP_SW
 161 #define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
 162 #endif
 163 #ifndef TCA_CLS_FLAGS_IN_HW
 164 #define TCA_CLS_FLAGS_IN_HW (1 << 2)
 165 #endif
 166 #ifndef HAVE_TCA_CHAIN
 167 #define TCA_CHAIN 11
 168 #endif
 169 #ifndef HAVE_TCA_FLOWER_ACT
 170 #define TCA_FLOWER_ACT 3
 171 #endif
 172 #ifndef HAVE_TCA_FLOWER_FLAGS
 173 #define TCA_FLOWER_FLAGS 22
 174 #endif
 175 #ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
 176 #define TCA_FLOWER_KEY_ETH_TYPE 8
 177 #endif
 178 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
 179 #define TCA_FLOWER_KEY_ETH_DST 4
 180 #endif
 181 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
 182 #define TCA_FLOWER_KEY_ETH_DST_MASK 5
 183 #endif
 184 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
 185 #define TCA_FLOWER_KEY_ETH_SRC 6
 186 #endif
 187 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
 188 #define TCA_FLOWER_KEY_ETH_SRC_MASK 7
 189 #endif
 190 #ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
 191 #define TCA_FLOWER_KEY_IP_PROTO 9
 192 #endif
 193 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
 194 #define TCA_FLOWER_KEY_IPV4_SRC 10
 195 #endif
 196 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
 197 #define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
 198 #endif
 199 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
 200 #define TCA_FLOWER_KEY_IPV4_DST 12
 201 #endif
 202 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
 203 #define TCA_FLOWER_KEY_IPV4_DST_MASK 13
 204 #endif
 205 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
 206 #define TCA_FLOWER_KEY_IPV6_SRC 14
 207 #endif
 208 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
 209 #define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
 210 #endif
 211 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
 212 #define TCA_FLOWER_KEY_IPV6_DST 16
 213 #endif
 214 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
 215 #define TCA_FLOWER_KEY_IPV6_DST_MASK 17
 216 #endif
 217 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
 218 #define TCA_FLOWER_KEY_TCP_SRC 18
 219 #endif
 220 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
 221 #define TCA_FLOWER_KEY_TCP_SRC_MASK 35
 222 #endif
 223 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
 224 #define TCA_FLOWER_KEY_TCP_DST 19
 225 #endif
 226 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
 227 #define TCA_FLOWER_KEY_TCP_DST_MASK 36
 228 #endif
 229 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
 230 #define TCA_FLOWER_KEY_UDP_SRC 20
 231 #endif
 232 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
 233 #define TCA_FLOWER_KEY_UDP_SRC_MASK 37
 234 #endif
 235 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
 236 #define TCA_FLOWER_KEY_UDP_DST 21
 237 #endif
 238 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
 239 #define TCA_FLOWER_KEY_UDP_DST_MASK 38
 240 #endif
 241 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
 242 #define TCA_FLOWER_KEY_VLAN_ID 23
 243 #endif
 244 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
 245 #define TCA_FLOWER_KEY_VLAN_PRIO 24
 246 #endif
 247 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
 248 #define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
 249 #endif
 250 #ifndef HAVE_TCA_FLOWER_KEY_ENC_KEY_ID
 251 #define TCA_FLOWER_KEY_ENC_KEY_ID 26
 252 #endif
 253 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC
 254 #define TCA_FLOWER_KEY_ENC_IPV4_SRC 27
 255 #endif
 256 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK
 257 #define TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK 28
 258 #endif
 259 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST
 260 #define TCA_FLOWER_KEY_ENC_IPV4_DST 29
 261 #endif
 262 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK
 263 #define TCA_FLOWER_KEY_ENC_IPV4_DST_MASK 30
 264 #endif
 265 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC
 266 #define TCA_FLOWER_KEY_ENC_IPV6_SRC 31
 267 #endif
 268 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK
 269 #define TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK 32
 270 #endif
 271 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST
 272 #define TCA_FLOWER_KEY_ENC_IPV6_DST 33
 273 #endif
 274 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK
 275 #define TCA_FLOWER_KEY_ENC_IPV6_DST_MASK 34
 276 #endif
 277 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT
 278 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT 43
 279 #endif
 280 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK
 281 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK 44
 282 #endif
 283 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT
 284 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT 45
 285 #endif
 286 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK
 287 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK 46
 288 #endif
 289 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS
 290 #define TCA_FLOWER_KEY_TCP_FLAGS 71
 291 #endif
 292 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK
 293 #define TCA_FLOWER_KEY_TCP_FLAGS_MASK 72
 294 #endif
 295 #ifndef HAVE_TC_ACT_GOTO_CHAIN
 296 #define TC_ACT_GOTO_CHAIN 0x20000000
 297 #endif
 298
 299 #ifndef IPV6_ADDR_LEN
 300 #define IPV6_ADDR_LEN 16
 301 #endif
 302
 303 #ifndef IPV4_ADDR_LEN
 304 #define IPV4_ADDR_LEN 4
 305 #endif
 306
 307 #ifndef TP_PORT_LEN
 308 #define TP_PORT_LEN 2 /* Transport Port (UDP/TCP) Length */
 309 #endif
 310
 311 #ifndef TTL_LEN
 312 #define TTL_LEN 1
 313 #endif
 314
 315 #ifndef TCA_ACT_MAX_PRIO
 316 #define TCA_ACT_MAX_PRIO 32
 317 #endif
 318
 319 /** UDP port range of VXLAN devices created by driver. */
 320 #define MLX5_VXLAN_PORT_MIN 30000
 321 #define MLX5_VXLAN_PORT_MAX 60000
 322 #define MLX5_VXLAN_DEVICE_PFX "vmlx_"
 323
 324 /** Tunnel action type, used for @p type in header structure. */
 325 enum flow_tcf_tunact_type {
 326         FLOW_TCF_TUNACT_VXLAN_DECAP,
 327         FLOW_TCF_TUNACT_VXLAN_ENCAP,
 328 };
 329
 330 /** Flags used for @p mask in tunnel action encap descriptors. */
 331 #define FLOW_TCF_ENCAP_ETH_SRC (1u << 0)
 332 #define FLOW_TCF_ENCAP_ETH_DST (1u << 1)
 333 #define FLOW_TCF_ENCAP_IPV4_SRC (1u << 2)
 334 #define FLOW_TCF_ENCAP_IPV4_DST (1u << 3)
 335 #define FLOW_TCF_ENCAP_IPV6_SRC (1u << 4)
 336 #define FLOW_TCF_ENCAP_IPV6_DST (1u << 5)
 337 #define FLOW_TCF_ENCAP_UDP_SRC (1u << 6)
 338 #define FLOW_TCF_ENCAP_UDP_DST (1u << 7)
 339 #define FLOW_TCF_ENCAP_VXLAN_VNI (1u << 8)
 340
 341 /**
 342  * Structure for holding netlink context.
 343  * Note the size of the message buffer which is MNL_SOCKET_BUFFER_SIZE.
 344  * Using this (8KB) buffer size ensures that netlink messages will never be
 345  * truncated.
 346  */
 347 struct mlx5_flow_tcf_context {
 348         struct mnl_socket *nl; /* NETLINK_ROUTE libmnl socket. */
 349         uint32_t seq; /* Message sequence number. */
 350         uint32_t buf_size; /* Message buffer size. */
 351         uint8_t *buf; /* Message buffer. */
 352 };
 353
 354 /**
 355  * Neigh rule structure. The neigh rule is applied via Netlink to
 356  * outer tunnel iface in order to provide destination MAC address
 357  * for the VXLAN encapsultion. The neigh rule is implicitly related
 358  * to the Flow itself and can be shared by multiple Flows.
 359  */
 360 struct tcf_neigh_rule {
 361         LIST_ENTRY(tcf_neigh_rule) next;
 362         uint32_t refcnt;
 363         struct ether_addr eth;
 364         uint16_t mask;
 365         union {
 366                 struct {
 367                         rte_be32_t dst;
 368                 } ipv4;
 369                 struct {
 370                         uint8_t dst[IPV6_ADDR_LEN];
 371                 } ipv6;
 372         };
 373 };
 374
 375 /**
 376  * Local rule structure. The local rule is applied via Netlink to
 377  * outer tunnel iface in order to provide local and peer IP addresses
 378  * of the VXLAN tunnel for encapsulation. The local rule is implicitly
 379  * related to the Flow itself and can be shared by multiple Flows.
 380  */
 381 struct tcf_local_rule {
 382         LIST_ENTRY(tcf_local_rule) next;
 383         uint32_t refcnt;
 384         uint16_t mask;
 385         union {
 386                 struct {
 387                         rte_be32_t dst;
 388                         rte_be32_t src;
 389                 } ipv4;
 390                 struct {
 391                         uint8_t dst[IPV6_ADDR_LEN];
 392                         uint8_t src[IPV6_ADDR_LEN];
 393                 } ipv6;
 394         };
 395 };
 396
 397 /** VXLAN virtual netdev. */
 398 struct tcf_vtep {
 399         LIST_ENTRY(tcf_vtep) next;
 400         LIST_HEAD(, tcf_neigh_rule) neigh;
 401         LIST_HEAD(, tcf_local_rule) local;
 402         uint32_t refcnt;
 403         unsigned int ifindex; /**< Own interface index. */
 404         unsigned int ifouter; /**< Index of device attached to. */
 405         uint16_t port;
 406         uint8_t created;
 407 };
 408
 409 /** Tunnel descriptor header, common for all tunnel types. */
 410 struct flow_tcf_tunnel_hdr {
 411         uint32_t type; /**< Tunnel action type. */
 412         struct tcf_vtep *vtep; /**< Virtual tunnel endpoint device. */
 413         unsigned int ifindex_org; /**< Original dst/src interface */
 414         unsigned int *ifindex_ptr; /**< Interface ptr in message. */
 415 };
 416
 417 struct flow_tcf_vxlan_decap {
 418         struct flow_tcf_tunnel_hdr hdr;
 419         uint16_t udp_port;
 420 };
 421
 422 struct flow_tcf_vxlan_encap {
 423         struct flow_tcf_tunnel_hdr hdr;
 424         uint32_t mask;
 425         struct {
 426                 struct ether_addr dst;
 427                 struct ether_addr src;
 428         } eth;
 429         union {
 430                 struct {
 431                         rte_be32_t dst;
 432                         rte_be32_t src;
 433                 } ipv4;
 434                 struct {
 435                         uint8_t dst[IPV6_ADDR_LEN];
 436                         uint8_t src[IPV6_ADDR_LEN];
 437                 } ipv6;
 438         };
 439 struct {
 440                 rte_be16_t src;
 441                 rte_be16_t dst;
 442         } udp;
 443         struct {
 444                 uint8_t vni[3];
 445         } vxlan;
 446 };
 447
 448 /** Structure used when extracting the values of a flow counters
 449  * from a netlink message.
 450  */
 451 struct flow_tcf_stats_basic {
 452         bool valid;
 453         struct gnet_stats_basic counters;
 454 };
 455
 456 /** Empty masks for known item types. */
 457 static const union {
 458         struct rte_flow_item_port_id port_id;
 459         struct rte_flow_item_eth eth;
 460         struct rte_flow_item_vlan vlan;
 461         struct rte_flow_item_ipv4 ipv4;
 462         struct rte_flow_item_ipv6 ipv6;
 463         struct rte_flow_item_tcp tcp;
 464         struct rte_flow_item_udp udp;
 465         struct rte_flow_item_vxlan vxlan;
 466 } flow_tcf_mask_empty;
 467
 468 /** Supported masks for known item types. */
 469 static const struct {
 470         struct rte_flow_item_port_id port_id;
 471         struct rte_flow_item_eth eth;
 472         struct rte_flow_item_vlan vlan;
 473         struct rte_flow_item_ipv4 ipv4;
 474         struct rte_flow_item_ipv6 ipv6;
 475         struct rte_flow_item_tcp tcp;
 476         struct rte_flow_item_udp udp;
 477         struct rte_flow_item_vxlan vxlan;
 478 } flow_tcf_mask_supported = {
 479         .port_id = {
 480                 .id = 0xffffffff,
 481         },
 482         .eth = {
 483                 .type = RTE_BE16(0xffff),
 484                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 485                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 486         },
 487         .vlan = {
 488                 /* PCP and VID only, no DEI. */
 489                 .tci = RTE_BE16(0xefff),
 490                 .inner_type = RTE_BE16(0xffff),
 491         },
 492         .ipv4.hdr = {
 493                 .next_proto_id = 0xff,
 494                 .src_addr = RTE_BE32(0xffffffff),
 495                 .dst_addr = RTE_BE32(0xffffffff),
 496         },
 497         .ipv6.hdr = {
 498                 .proto = 0xff,
 499                 .src_addr =
 500                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 501                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 502                 .dst_addr =
 503                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 504                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 505         },
 506         .tcp.hdr = {
 507                 .src_port = RTE_BE16(0xffff),
 508                 .dst_port = RTE_BE16(0xffff),
 509                 .tcp_flags = 0xff,
 510         },
 511         .udp.hdr = {
 512                 .src_port = RTE_BE16(0xffff),
 513                 .dst_port = RTE_BE16(0xffff),
 514         },
 515         .vxlan = {
 516                .vni = "\xff\xff\xff",
 517         },
 518 };
 519
 520 #define SZ_NLATTR_HDR MNL_ALIGN(sizeof(struct nlattr))
 521 #define SZ_NLATTR_NEST SZ_NLATTR_HDR
 522 #define SZ_NLATTR_DATA_OF(len) MNL_ALIGN(SZ_NLATTR_HDR + (len))
 523 #define SZ_NLATTR_TYPE_OF(typ) SZ_NLATTR_DATA_OF(sizeof(typ))
 524 #define SZ_NLATTR_STRZ_OF(str) SZ_NLATTR_DATA_OF(strlen(str) + 1)
 525
 526 #define PTOI_TABLE_SZ_MAX(dev) (mlx5_dev_to_port_id((dev)->device, NULL, 0) + 2)
 527
 528 /** DPDK port to network interface index (ifindex) conversion. */
 529 struct flow_tcf_ptoi {
 530         uint16_t port_id; /**< DPDK port ID. */
 531         unsigned int ifindex; /**< Network interface index. */
 532 };
 533
 534 /* Due to a limitation on driver/FW. */
 535 #define MLX5_TCF_GROUP_ID_MAX 3
 536
 537 /*
 538  * Due to a limitation on driver/FW, priority ranges from 1 to 16 in kernel.
 539  * Priority in rte_flow attribute starts from 0 and is added by 1 in
 540  * translation. This is subject to be changed to determine the max priority
 541  * based on trial-and-error like Verbs driver once the restriction is lifted or
 542  * the range is extended.
 543  */
 544 #define MLX5_TCF_GROUP_PRIORITY_MAX 15
 545
 546 #define MLX5_TCF_FATE_ACTIONS \
 547         (MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
 548          MLX5_FLOW_ACTION_JUMP)
 549
 550 #define MLX5_TCF_VLAN_ACTIONS \
 551         (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN | \
 552          MLX5_FLOW_ACTION_OF_SET_VLAN_VID | MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
 553
 554 #define MLX5_TCF_VXLAN_ACTIONS \
 555         (MLX5_FLOW_ACTION_VXLAN_ENCAP | MLX5_FLOW_ACTION_VXLAN_DECAP)
 556
 557 #define MLX5_TCF_PEDIT_ACTIONS \
 558         (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST | \
 559          MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST | \
 560          MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST | \
 561          MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL | \
 562          MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)
 563
 564 #define MLX5_TCF_CONFIG_ACTIONS \
 565         (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_JUMP | \
 566          MLX5_FLOW_ACTION_OF_PUSH_VLAN | MLX5_FLOW_ACTION_OF_SET_VLAN_VID | \
 567          MLX5_FLOW_ACTION_OF_SET_VLAN_PCP | \
 568          (MLX5_TCF_PEDIT_ACTIONS & ~MLX5_FLOW_ACTION_DEC_TTL))
 569
 570 #define MAX_PEDIT_KEYS 128
 571 #define SZ_PEDIT_KEY_VAL 4
 572
 573 #define NUM_OF_PEDIT_KEYS(sz) \
 574         (((sz) / SZ_PEDIT_KEY_VAL) + (((sz) % SZ_PEDIT_KEY_VAL) ? 1 : 0))
 575
 576 struct pedit_key_ex {
 577         enum pedit_header_type htype;
 578         enum pedit_cmd cmd;
 579 };
 580
 581 struct pedit_parser {
 582         struct tc_pedit_sel sel;
 583         struct tc_pedit_key keys[MAX_PEDIT_KEYS];
 584         struct pedit_key_ex keys_ex[MAX_PEDIT_KEYS];
 585 };
 586
 587 /**
 588  * Create space for using the implicitly created TC flow counter.
 589  *
 590  * @param[in] dev
 591  *   Pointer to the Ethernet device structure.
 592  *
 593  * @return
 594  *   A pointer to the counter data structure, NULL otherwise and
 595  *   rte_errno is set.
 596  */
 597 static struct mlx5_flow_counter *
 598 flow_tcf_counter_new(void)
 599 {
 600         struct mlx5_flow_counter *cnt;
 601
 602         /*
 603          * eswitch counter cannot be shared and its id is unknown.
 604          * currently returning all with id 0.
 605          * in the future maybe better to switch to unique numbers.
 606          */
 607         struct mlx5_flow_counter tmpl = {
 608                 .ref_cnt = 1,
 609         };
 610         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 611         if (!cnt) {
 612                 rte_errno = ENOMEM;
 613                 return NULL;
 614         }
 615         *cnt = tmpl;
 616         /* Implicit counter, do not add to list. */
 617         return cnt;
 618 }
 619
 620 /**
 621  * Set pedit key of MAC address
 622  *
 623  * @param[in] actions
 624  *   pointer to action specification
 625  * @param[in,out] p_parser
 626  *   pointer to pedit_parser
 627  */
 628 static void
 629 flow_tcf_pedit_key_set_mac(const struct rte_flow_action *actions,
 630                            struct pedit_parser *p_parser)
 631 {
 632         int idx = p_parser->sel.nkeys;
 633         uint32_t off = actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
 634                                         offsetof(struct ether_hdr, s_addr) :
 635                                         offsetof(struct ether_hdr, d_addr);
 636         const struct rte_flow_action_set_mac *conf =
 637                 (const struct rte_flow_action_set_mac *)actions->conf;
 638
 639         p_parser->keys[idx].off = off;
 640         p_parser->keys[idx].mask = ~UINT32_MAX;
 641         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 642         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 643         memcpy(&p_parser->keys[idx].val,
 644                 conf->mac_addr, SZ_PEDIT_KEY_VAL);
 645         idx++;
 646         p_parser->keys[idx].off = off + SZ_PEDIT_KEY_VAL;
 647         p_parser->keys[idx].mask = 0xFFFF0000;
 648         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 649         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 650         memcpy(&p_parser->keys[idx].val,
 651                 conf->mac_addr + SZ_PEDIT_KEY_VAL,
 652                 ETHER_ADDR_LEN - SZ_PEDIT_KEY_VAL);
 653         p_parser->sel.nkeys = (++idx);
 654 }
 655
 656 /**
 657  * Set pedit key of decrease/set ttl
 658  *
 659  * @param[in] actions
 660  *   pointer to action specification
 661  * @param[in,out] p_parser
 662  *   pointer to pedit_parser
 663  * @param[in] item_flags
 664  *   flags of all items presented
 665  */
 666 static void
 667 flow_tcf_pedit_key_set_dec_ttl(const struct rte_flow_action *actions,
 668                                 struct pedit_parser *p_parser,
 669                                 uint64_t item_flags)
 670 {
 671         int idx = p_parser->sel.nkeys;
 672
 673         p_parser->keys[idx].mask = 0xFFFFFF00;
 674         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) {
 675                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 676                 p_parser->keys[idx].off =
 677                         offsetof(struct ipv4_hdr, time_to_live);
 678         }
 679         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6) {
 680                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 681                 p_parser->keys[idx].off =
 682                         offsetof(struct ipv6_hdr, hop_limits);
 683         }
 684         if (actions->type == RTE_FLOW_ACTION_TYPE_DEC_TTL) {
 685                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_ADD;
 686                 p_parser->keys[idx].val = 0x000000FF;
 687         } else {
 688                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 689                 p_parser->keys[idx].val =
 690                         (__u32)((const struct rte_flow_action_set_ttl *)
 691                          actions->conf)->ttl_value;
 692         }
 693         p_parser->sel.nkeys = (++idx);
 694 }
 695
 696 /**
 697  * Set pedit key of transport (TCP/UDP) port value
 698  *
 699  * @param[in] actions
 700  *   pointer to action specification
 701  * @param[in,out] p_parser
 702  *   pointer to pedit_parser
 703  * @param[in] item_flags
 704  *   flags of all items presented
 705  */
 706 static void
 707 flow_tcf_pedit_key_set_tp_port(const struct rte_flow_action *actions,
 708                                 struct pedit_parser *p_parser,
 709                                 uint64_t item_flags)
 710 {
 711         int idx = p_parser->sel.nkeys;
 712
 713         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 714                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_UDP;
 715         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 716                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_TCP;
 717         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 718         /* offset of src/dst port is same for TCP and UDP */
 719         p_parser->keys[idx].off =
 720                 actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
 721                 offsetof(struct tcp_hdr, src_port) :
 722                 offsetof(struct tcp_hdr, dst_port);
 723         p_parser->keys[idx].mask = 0xFFFF0000;
 724         p_parser->keys[idx].val =
 725                 (__u32)((const struct rte_flow_action_set_tp *)
 726                                 actions->conf)->port;
 727         p_parser->sel.nkeys = (++idx);
 728 }
 729
 730 /**
 731  * Set pedit key of ipv6 address
 732  *
 733  * @param[in] actions
 734  *   pointer to action specification
 735  * @param[in,out] p_parser
 736  *   pointer to pedit_parser
 737  */
 738 static void
 739 flow_tcf_pedit_key_set_ipv6_addr(const struct rte_flow_action *actions,
 740                                  struct pedit_parser *p_parser)
 741 {
 742         int idx = p_parser->sel.nkeys;
 743         int keys = NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 744         int off_base =
 745                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
 746                 offsetof(struct ipv6_hdr, src_addr) :
 747                 offsetof(struct ipv6_hdr, dst_addr);
 748         const struct rte_flow_action_set_ipv6 *conf =
 749                 (const struct rte_flow_action_set_ipv6 *)actions->conf;
 750
 751         for (int i = 0; i < keys; i++, idx++) {
 752                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 753                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 754                 p_parser->keys[idx].off = off_base + i * SZ_PEDIT_KEY_VAL;
 755                 p_parser->keys[idx].mask = ~UINT32_MAX;
 756                 memcpy(&p_parser->keys[idx].val,
 757                         conf->ipv6_addr + i *  SZ_PEDIT_KEY_VAL,
 758                         SZ_PEDIT_KEY_VAL);
 759         }
 760         p_parser->sel.nkeys += keys;
 761 }
 762
 763 /**
 764  * Set pedit key of ipv4 address
 765  *
 766  * @param[in] actions
 767  *   pointer to action specification
 768  * @param[in,out] p_parser
 769  *   pointer to pedit_parser
 770  */
 771 static void
 772 flow_tcf_pedit_key_set_ipv4_addr(const struct rte_flow_action *actions,
 773                                  struct pedit_parser *p_parser)
 774 {
 775         int idx = p_parser->sel.nkeys;
 776
 777         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 778         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 779         p_parser->keys[idx].off =
 780                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
 781                 offsetof(struct ipv4_hdr, src_addr) :
 782                 offsetof(struct ipv4_hdr, dst_addr);
 783         p_parser->keys[idx].mask = ~UINT32_MAX;
 784         p_parser->keys[idx].val =
 785                 ((const struct rte_flow_action_set_ipv4 *)
 786                  actions->conf)->ipv4_addr;
 787         p_parser->sel.nkeys = (++idx);
 788 }
 789
 790 /**
 791  * Create the pedit's na attribute in netlink message
 792  * on pre-allocate message buffer
 793  *
 794  * @param[in,out] nl
 795  *   pointer to pre-allocated netlink message buffer
 796  * @param[in,out] actions
 797  *   pointer to pointer of actions specification.
 798  * @param[in,out] action_flags
 799  *   pointer to actions flags
 800  * @param[in] item_flags
 801  *   flags of all item presented
 802  */
 803 static void
 804 flow_tcf_create_pedit_mnl_msg(struct nlmsghdr *nl,
 805                               const struct rte_flow_action **actions,
 806                               uint64_t item_flags)
 807 {
 808         struct pedit_parser p_parser;
 809         struct nlattr *na_act_options;
 810         struct nlattr *na_pedit_keys;
 811
 812         memset(&p_parser, 0, sizeof(p_parser));
 813         mnl_attr_put_strz(nl, TCA_ACT_KIND, "pedit");
 814         na_act_options = mnl_attr_nest_start(nl, TCA_ACT_OPTIONS);
 815         /* all modify header actions should be in one tc-pedit action */
 816         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 817                 switch ((*actions)->type) {
 818                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 819                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 820                         flow_tcf_pedit_key_set_ipv4_addr(*actions, &p_parser);
 821                         break;
 822                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 823                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 824                         flow_tcf_pedit_key_set_ipv6_addr(*actions, &p_parser);
 825                         break;
 826                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 827                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 828                         flow_tcf_pedit_key_set_tp_port(*actions,
 829                                                         &p_parser, item_flags);
 830                         break;
 831                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 832                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 833                         flow_tcf_pedit_key_set_dec_ttl(*actions,
 834                                                         &p_parser, item_flags);
 835                         break;
 836                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 837                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 838                         flow_tcf_pedit_key_set_mac(*actions, &p_parser);
 839                         break;
 840                 default:
 841                         goto pedit_mnl_msg_done;
 842                 }
 843         }
 844 pedit_mnl_msg_done:
 845         p_parser.sel.action = TC_ACT_PIPE;
 846         mnl_attr_put(nl, TCA_PEDIT_PARMS_EX,
 847                      sizeof(p_parser.sel) +
 848                      p_parser.sel.nkeys * sizeof(struct tc_pedit_key),
 849                      &p_parser);
 850         na_pedit_keys =
 851                 mnl_attr_nest_start(nl, TCA_PEDIT_KEYS_EX | NLA_F_NESTED);
 852         for (int i = 0; i < p_parser.sel.nkeys; i++) {
 853                 struct nlattr *na_pedit_key =
 854                         mnl_attr_nest_start(nl,
 855                                             TCA_PEDIT_KEY_EX | NLA_F_NESTED);
 856                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_HTYPE,
 857                                  p_parser.keys_ex[i].htype);
 858                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_CMD,
 859                                  p_parser.keys_ex[i].cmd);
 860                 mnl_attr_nest_end(nl, na_pedit_key);
 861         }
 862         mnl_attr_nest_end(nl, na_pedit_keys);
 863         mnl_attr_nest_end(nl, na_act_options);
 864         (*actions)--;
 865 }
 866
 867 /**
 868  * Calculate max memory size of one TC-pedit actions.
 869  * One TC-pedit action can contain set of keys each defining
 870  * a rewrite element (rte_flow action)
 871  *
 872  * @param[in,out] actions
 873  *   actions specification.
 874  * @param[in,out] action_flags
 875  *   actions flags
 876  * @param[in,out] size
 877  *   accumulated size
 878  * @return
 879  *   Max memory size of one TC-pedit action
 880  */
 881 static int
 882 flow_tcf_get_pedit_actions_size(const struct rte_flow_action **actions,
 883                                 uint64_t *action_flags)
 884 {
 885         int pedit_size = 0;
 886         int keys = 0;
 887         uint64_t flags = 0;
 888
 889         pedit_size += SZ_NLATTR_NEST + /* na_act_index. */
 890                       SZ_NLATTR_STRZ_OF("pedit") +
 891                       SZ_NLATTR_NEST; /* TCA_ACT_OPTIONS. */
 892         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 893                 switch ((*actions)->type) {
 894                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 895                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 896                         flags |= MLX5_FLOW_ACTION_SET_IPV4_SRC;
 897                         break;
 898                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 899                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 900                         flags |= MLX5_FLOW_ACTION_SET_IPV4_DST;
 901                         break;
 902                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 903                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 904                         flags |= MLX5_FLOW_ACTION_SET_IPV6_SRC;
 905                         break;
 906                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 907                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 908                         flags |= MLX5_FLOW_ACTION_SET_IPV6_DST;
 909                         break;
 910                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 911                         /* TCP is as same as UDP */
 912                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 913                         flags |= MLX5_FLOW_ACTION_SET_TP_SRC;
 914                         break;
 915                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 916                         /* TCP is as same as UDP */
 917                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 918                         flags |= MLX5_FLOW_ACTION_SET_TP_DST;
 919                         break;
 920                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 921                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 922                         flags |= MLX5_FLOW_ACTION_SET_TTL;
 923                         break;
 924                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 925                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 926                         flags |= MLX5_FLOW_ACTION_DEC_TTL;
 927                         break;
 928                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 929                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 930                         flags |= MLX5_FLOW_ACTION_SET_MAC_SRC;
 931                         break;
 932                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 933                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 934                         flags |= MLX5_FLOW_ACTION_SET_MAC_DST;
 935                         break;
 936                 default:
 937                         goto get_pedit_action_size_done;
 938                 }
 939         }
 940 get_pedit_action_size_done:
 941         /* TCA_PEDIT_PARAMS_EX */
 942         pedit_size +=
 943                 SZ_NLATTR_DATA_OF(sizeof(struct tc_pedit_sel) +
 944                                   keys * sizeof(struct tc_pedit_key));
 945         pedit_size += SZ_NLATTR_NEST; /* TCA_PEDIT_KEYS */
 946         pedit_size += keys *
 947                       /* TCA_PEDIT_KEY_EX + HTYPE + CMD */
 948                       (SZ_NLATTR_NEST + SZ_NLATTR_DATA_OF(2) +
 949                        SZ_NLATTR_DATA_OF(2));
 950         (*action_flags) |= flags;
 951         (*actions)--;
 952         return pedit_size;
 953 }
 954
 955 /**
 956  * Retrieve mask for pattern item.
 957  *
 958  * This function does basic sanity checks on a pattern item in order to
 959  * return the most appropriate mask for it.
 960  *
 961  * @param[in] item
 962  *   Item specification.
 963  * @param[in] mask_default
 964  *   Default mask for pattern item as specified by the flow API.
 965  * @param[in] mask_supported
 966  *   Mask fields supported by the implementation.
 967  * @param[in] mask_empty
 968  *   Empty mask to return when there is no specification.
 969  * @param[out] error
 970  *   Perform verbose error reporting if not NULL.
 971  *
 972  * @return
 973  *   Either @p item->mask or one of the mask parameters on success, NULL
 974  *   otherwise and rte_errno is set.
 975  */
 976 static const void *
 977 flow_tcf_item_mask(const struct rte_flow_item *item, const void *mask_default,
 978                    const void *mask_supported, const void *mask_empty,
 979                    size_t mask_size, struct rte_flow_error *error)
 980 {
 981         const uint8_t *mask;
 982         size_t i;
 983
 984         /* item->last and item->mask cannot exist without item->spec. */
 985         if (!item->spec && (item->mask || item->last)) {
 986                 rte_flow_error_set(error, EINVAL,
 987                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 988                                    "\"mask\" or \"last\" field provided without"
 989                                    " a corresponding \"spec\"");
 990                 return NULL;
 991         }
 992         /* No spec, no mask, no problem. */
 993         if (!item->spec)
 994                 return mask_empty;
 995         mask = item->mask ? item->mask : mask_default;
 996         assert(mask);
 997         /*
 998          * Single-pass check to make sure that:
 999          * - Mask is supported, no bits are set outside mask_supported.
1000          * - Both item->spec and item->last are included in mask.
1001          */
1002         for (i = 0; i != mask_size; ++i) {
1003                 if (!mask[i])
1004                         continue;
1005                 if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
1006                     ((const uint8_t *)mask_supported)[i]) {
1007                         rte_flow_error_set(error, ENOTSUP,
1008                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1009                                            "unsupported field found"
1010                                            " in \"mask\"");
1011                         return NULL;
1012                 }
1013                 if (item->last &&
1014                     (((const uint8_t *)item->spec)[i] & mask[i]) !=
1015                     (((const uint8_t *)item->last)[i] & mask[i])) {
1016                         rte_flow_error_set(error, EINVAL,
1017                                            RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1018                                            item->last,
1019                                            "range between \"spec\" and \"last\""
1020                                            " not comprised in \"mask\"");
1021                         return NULL;
1022                 }
1023         }
1024         return mask;
1025 }
1026
1027 /**
1028  * Build a conversion table between port ID and ifindex.
1029  *
1030  * @param[in] dev
1031  *   Pointer to Ethernet device.
1032  * @param[out] ptoi
1033  *   Pointer to ptoi table.
1034  * @param[in] len
1035  *   Size of ptoi table provided.
1036  *
1037  * @return
1038  *   Size of ptoi table filled.
1039  */
1040 static unsigned int
1041 flow_tcf_build_ptoi_table(struct rte_eth_dev *dev, struct flow_tcf_ptoi *ptoi,
1042                           unsigned int len)
1043 {
1044         unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
1045         uint16_t port_id[n + 1];
1046         unsigned int i;
1047         unsigned int own = 0;
1048
1049         /* At least one port is needed when no switch domain is present. */
1050         if (!n) {
1051                 n = 1;
1052                 port_id[0] = dev->data->port_id;
1053         } else {
1054                 n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
1055         }
1056         if (n > len)
1057                 return 0;
1058         for (i = 0; i != n; ++i) {
1059                 struct rte_eth_dev_info dev_info;
1060
1061                 rte_eth_dev_info_get(port_id[i], &dev_info);
1062                 if (port_id[i] == dev->data->port_id)
1063                         own = i;
1064                 ptoi[i].port_id = port_id[i];
1065                 ptoi[i].ifindex = dev_info.if_index;
1066         }
1067         /* Ensure first entry of ptoi[] is the current device. */
1068         if (own) {
1069                 ptoi[n] = ptoi[0];
1070                 ptoi[0] = ptoi[own];
1071                 ptoi[own] = ptoi[n];
1072         }
1073         /* An entry with zero ifindex terminates ptoi[]. */
1074         ptoi[n].port_id = 0;
1075         ptoi[n].ifindex = 0;
1076         return n;
1077 }
1078
1079 /**
1080  * Verify the @p attr will be correctly understood by the E-switch.
1081  *
1082  * @param[in] attr
1083  *   Pointer to flow attributes
1084  * @param[out] error
1085  *   Pointer to error structure.
1086  *
1087  * @return
1088  *   0 on success, a negative errno value otherwise and rte_errno is set.
1089  */
1090 static int
1091 flow_tcf_validate_attributes(const struct rte_flow_attr *attr,
1092                              struct rte_flow_error *error)
1093 {
1094         /*
1095          * Supported attributes: groups, some priorities and ingress only.
1096          * group is supported only if kernel supports chain. Don't care about
1097          * transfer as it is the caller's problem.
1098          */
1099         if (attr->group > MLX5_TCF_GROUP_ID_MAX)
1100                 return rte_flow_error_set(error, ENOTSUP,
1101                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1102                                           "group ID larger than "
1103                                           RTE_STR(MLX5_TCF_GROUP_ID_MAX)
1104                                           " isn't supported");
1105         else if (attr->priority > MLX5_TCF_GROUP_PRIORITY_MAX)
1106                 return rte_flow_error_set(error, ENOTSUP,
1107                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1108                                           attr,
1109                                           "priority more than "
1110                                           RTE_STR(MLX5_TCF_GROUP_PRIORITY_MAX)
1111                                           " is not supported");
1112         if (!attr->ingress)
1113                 return rte_flow_error_set(error, EINVAL,
1114                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1115                                           attr, "only ingress is supported");
1116         if (attr->egress)
1117                 return rte_flow_error_set(error, ENOTSUP,
1118                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1119                                           attr, "egress is not supported");
1120         return 0;
1121 }
1122
1123 /**
1124  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_ETH item for E-Switch.
1125  * The routine checks the L2 fields to be used in encapsulation header.
1126  *
1127  * @param[in] item
1128  *   Pointer to the item structure.
1129  * @param[out] error
1130  *   Pointer to the error structure.
1131  *
1132  * @return
1133  *   0 on success, a negative errno value otherwise and rte_errno is set.
1134  **/
1135 static int
1136 flow_tcf_validate_vxlan_encap_eth(const struct rte_flow_item *item,
1137                                   struct rte_flow_error *error)
1138 {
1139         const struct rte_flow_item_eth *spec = item->spec;
1140         const struct rte_flow_item_eth *mask = item->mask;
1141
1142         if (!spec) {
1143                 /*
1144                  * Specification for L2 addresses can be empty
1145                  * because these ones are optional and not
1146                  * required directly by tc rule. Kernel tries
1147                  * to resolve these ones on its own
1148                  */
1149                 return 0;
1150         }
1151         if (!mask) {
1152                 /* If mask is not specified use the default one. */
1153                 mask = &rte_flow_item_eth_mask;
1154         }
1155         if (memcmp(&mask->dst,
1156                    &flow_tcf_mask_empty.eth.dst,
1157                    sizeof(flow_tcf_mask_empty.eth.dst))) {
1158                 if (memcmp(&mask->dst,
1159                            &rte_flow_item_eth_mask.dst,
1160                            sizeof(rte_flow_item_eth_mask.dst)))
1161                         return rte_flow_error_set
1162                                 (error, ENOTSUP,
1163                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1164                                  "no support for partial mask on"
1165                                  " \"eth.dst\" field");
1166         }
1167         if (memcmp(&mask->src,
1168                    &flow_tcf_mask_empty.eth.src,
1169                    sizeof(flow_tcf_mask_empty.eth.src))) {
1170                 if (memcmp(&mask->src,
1171                            &rte_flow_item_eth_mask.src,
1172                            sizeof(rte_flow_item_eth_mask.src)))
1173                         return rte_flow_error_set
1174                                 (error, ENOTSUP,
1175                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1176                                  "no support for partial mask on"
1177                                  " \"eth.src\" field");
1178         }
1179         if (mask->type != RTE_BE16(0x0000)) {
1180                 if (mask->type != RTE_BE16(0xffff))
1181                         return rte_flow_error_set
1182                                 (error, ENOTSUP,
1183                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1184                                  "no support for partial mask on"
1185                                  " \"eth.type\" field");
1186                 DRV_LOG(WARNING,
1187                         "outer ethernet type field"
1188                         " cannot be forced for vxlan"
1189                         " encapsulation, parameter ignored");
1190         }
1191         return 0;
1192 }
1193
1194 /**
1195  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV4 item for E-Switch.
1196  * The routine checks the IPv4 fields to be used in encapsulation header.
1197  *
1198  * @param[in] item
1199  *   Pointer to the item structure.
1200  * @param[out] error
1201  *   Pointer to the error structure.
1202  *
1203  * @return
1204  *   0 on success, a negative errno value otherwise and rte_errno is set.
1205  **/
1206 static int
1207 flow_tcf_validate_vxlan_encap_ipv4(const struct rte_flow_item *item,
1208                                    struct rte_flow_error *error)
1209 {
1210         const struct rte_flow_item_ipv4 *spec = item->spec;
1211         const struct rte_flow_item_ipv4 *mask = item->mask;
1212
1213         if (!spec) {
1214                 /*
1215                  * Specification for IP addresses cannot be empty
1216                  * because it is required by tunnel_key parameter.
1217                  */
1218                 return rte_flow_error_set(error, EINVAL,
1219                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1220                                           "NULL outer ipv4 address"
1221                                           " specification for vxlan"
1222                                           " encapsulation");
1223         }
1224         if (!mask)
1225                 mask = &rte_flow_item_ipv4_mask;
1226         if (mask->hdr.dst_addr != RTE_BE32(0x00000000)) {
1227                 if (mask->hdr.dst_addr != RTE_BE32(0xffffffff))
1228                         return rte_flow_error_set
1229                                 (error, ENOTSUP,
1230                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1231                                  "no support for partial mask on"
1232                                  " \"ipv4.hdr.dst_addr\" field"
1233                                  " for vxlan encapsulation");
1234                 /* More IPv4 address validations can be put here. */
1235         } else {
1236                 /*
1237                  * Kernel uses the destination IP address to determine
1238                  * the routing path and obtain the MAC destination
1239                  * address, so IP destination address must be
1240                  * specified in the tc rule.
1241                  */
1242                 return rte_flow_error_set(error, EINVAL,
1243                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1244                                           "outer ipv4 destination address"
1245                                           " must be specified for"
1246                                           " vxlan encapsulation");
1247         }
1248         if (mask->hdr.src_addr != RTE_BE32(0x00000000)) {
1249                 if (mask->hdr.src_addr != RTE_BE32(0xffffffff))
1250                         return rte_flow_error_set
1251                                 (error, ENOTSUP,
1252                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1253                                  "no support for partial mask on"
1254                                  " \"ipv4.hdr.src_addr\" field"
1255                                  " for vxlan encapsulation");
1256                 /* More IPv4 address validations can be put here. */
1257         } else {
1258                 /*
1259                  * Kernel uses the source IP address to select the
1260                  * interface for egress encapsulated traffic, so
1261                  * it must be specified in the tc rule.
1262                  */
1263                 return rte_flow_error_set(error, EINVAL,
1264                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1265                                           "outer ipv4 source address"
1266                                           " must be specified for"
1267                                           " vxlan encapsulation");
1268         }
1269         return 0;
1270 }
1271
1272 /**
1273  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV6 item for E-Switch.
1274  * The routine checks the IPv6 fields to be used in encapsulation header.
1275  *
1276  * @param[in] item
1277  *   Pointer to the item structure.
1278  * @param[out] error
1279  *   Pointer to the error structure.
1280  *
1281  * @return
1282  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1283  **/
1284 static int
1285 flow_tcf_validate_vxlan_encap_ipv6(const struct rte_flow_item *item,
1286                                    struct rte_flow_error *error)
1287 {
1288         const struct rte_flow_item_ipv6 *spec = item->spec;
1289         const struct rte_flow_item_ipv6 *mask = item->mask;
1290
1291         if (!spec) {
1292                 /*
1293                  * Specification for IP addresses cannot be empty
1294                  * because it is required by tunnel_key parameter.
1295                  */
1296                 return rte_flow_error_set(error, EINVAL,
1297                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1298                                           "NULL outer ipv6 address"
1299                                           " specification for"
1300                                           " vxlan encapsulation");
1301         }
1302         if (!mask)
1303                 mask = &rte_flow_item_ipv6_mask;
1304         if (memcmp(&mask->hdr.dst_addr,
1305                    &flow_tcf_mask_empty.ipv6.hdr.dst_addr,
1306                    IPV6_ADDR_LEN)) {
1307                 if (memcmp(&mask->hdr.dst_addr,
1308                            &rte_flow_item_ipv6_mask.hdr.dst_addr,
1309                            IPV6_ADDR_LEN))
1310                         return rte_flow_error_set
1311                                         (error, ENOTSUP,
1312                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1313                                          "no support for partial mask on"
1314                                          " \"ipv6.hdr.dst_addr\" field"
1315                                          " for vxlan encapsulation");
1316                 /* More IPv6 address validations can be put here. */
1317         } else {
1318                 /*
1319                  * Kernel uses the destination IP address to determine
1320                  * the routing path and obtain the MAC destination
1321                  * address (heigh or gate), so IP destination address
1322                  * must be specified within the tc rule.
1323                  */
1324                 return rte_flow_error_set(error, EINVAL,
1325                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1326                                           "outer ipv6 destination address"
1327                                           " must be specified for"
1328                                           " vxlan encapsulation");
1329         }
1330         if (memcmp(&mask->hdr.src_addr,
1331                    &flow_tcf_mask_empty.ipv6.hdr.src_addr,
1332                    IPV6_ADDR_LEN)) {
1333                 if (memcmp(&mask->hdr.src_addr,
1334                            &rte_flow_item_ipv6_mask.hdr.src_addr,
1335                            IPV6_ADDR_LEN))
1336                         return rte_flow_error_set
1337                                         (error, ENOTSUP,
1338                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1339                                          "no support for partial mask on"
1340                                          " \"ipv6.hdr.src_addr\" field"
1341                                          " for vxlan encapsulation");
1342                 /* More L3 address validation can be put here. */
1343         } else {
1344                 /*
1345                  * Kernel uses the source IP address to select the
1346                  * interface for egress encapsulated traffic, so
1347                  * it must be specified in the tc rule.
1348                  */
1349                 return rte_flow_error_set(error, EINVAL,
1350                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1351                                           "outer L3 source address"
1352                                           " must be specified for"
1353                                           " vxlan encapsulation");
1354         }
1355         return 0;
1356 }
1357
1358 /**
1359  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_UDP item for E-Switch.
1360  * The routine checks the UDP fields to be used in encapsulation header.
1361  *
1362  * @param[in] item
1363  *   Pointer to the item structure.
1364  * @param[out] error
1365  *   Pointer to the error structure.
1366  *
1367  * @return
1368  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1369  **/
1370 static int
1371 flow_tcf_validate_vxlan_encap_udp(const struct rte_flow_item *item,
1372                                   struct rte_flow_error *error)
1373 {
1374         const struct rte_flow_item_udp *spec = item->spec;
1375         const struct rte_flow_item_udp *mask = item->mask;
1376
1377         if (!spec) {
1378                 /*
1379                  * Specification for UDP ports cannot be empty
1380                  * because it is required by tunnel_key parameter.
1381                  */
1382                 return rte_flow_error_set(error, EINVAL,
1383                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1384                                           "NULL UDP port specification "
1385                                           " for vxlan encapsulation");
1386         }
1387         if (!mask)
1388                 mask = &rte_flow_item_udp_mask;
1389         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1390                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1391                         return rte_flow_error_set
1392                                         (error, ENOTSUP,
1393                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1394                                          "no support for partial mask on"
1395                                          " \"udp.hdr.dst_port\" field"
1396                                          " for vxlan encapsulation");
1397                 if (!spec->hdr.dst_port)
1398                         return rte_flow_error_set
1399                                         (error, EINVAL,
1400                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
1401                                          "outer UDP remote port cannot be"
1402                                          " 0 for vxlan encapsulation");
1403         } else {
1404                 return rte_flow_error_set(error, EINVAL,
1405                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1406                                           "outer UDP remote port"
1407                                           " must be specified for"
1408                                           " vxlan encapsulation");
1409         }
1410         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1411                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1412                         return rte_flow_error_set
1413                                         (error, ENOTSUP,
1414                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1415                                          "no support for partial mask on"
1416                                          " \"udp.hdr.src_port\" field"
1417                                          " for vxlan encapsulation");
1418                 DRV_LOG(WARNING,
1419                         "outer UDP source port cannot be"
1420                         " forced for vxlan encapsulation,"
1421                         " parameter ignored");
1422         }
1423         return 0;
1424 }
1425
1426 /**
1427  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_VXLAN item for E-Switch.
1428  * The routine checks the VNIP fields to be used in encapsulation header.
1429  *
1430  * @param[in] item
1431  *   Pointer to the item structure.
1432  * @param[out] error
1433  *   Pointer to the error structure.
1434  *
1435  * @return
1436  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1437  **/
1438 static int
1439 flow_tcf_validate_vxlan_encap_vni(const struct rte_flow_item *item,
1440                                   struct rte_flow_error *error)
1441 {
1442         const struct rte_flow_item_vxlan *spec = item->spec;
1443         const struct rte_flow_item_vxlan *mask = item->mask;
1444
1445         if (!spec) {
1446                 /* Outer VNI is required by tunnel_key parameter. */
1447                 return rte_flow_error_set(error, EINVAL,
1448                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1449                                           "NULL VNI specification"
1450                                           " for vxlan encapsulation");
1451         }
1452         if (!mask)
1453                 mask = &rte_flow_item_vxlan_mask;
1454         if (!mask->vni[0] && !mask->vni[1] && !mask->vni[2])
1455                 return rte_flow_error_set(error, EINVAL,
1456                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1457                                           "outer VNI must be specified "
1458                                           "for vxlan encapsulation");
1459         if (mask->vni[0] != 0xff ||
1460             mask->vni[1] != 0xff ||
1461             mask->vni[2] != 0xff)
1462                 return rte_flow_error_set(error, ENOTSUP,
1463                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1464                                           "no support for partial mask on"
1465                                           " \"vxlan.vni\" field");
1466
1467         if (!spec->vni[0] && !spec->vni[1] && !spec->vni[2])
1468                 return rte_flow_error_set(error, EINVAL,
1469                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1470                                           "vxlan vni cannot be 0");
1471         return 0;
1472 }
1473
1474 /**
1475  * Validate VXLAN_ENCAP action item list for E-Switch.
1476  * The routine checks items to be used in encapsulation header.
1477  *
1478  * @param[in] action
1479  *   Pointer to the VXLAN_ENCAP action structure.
1480  * @param[out] error
1481  *   Pointer to the error structure.
1482  *
1483  * @return
1484  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1485  **/
1486 static int
1487 flow_tcf_validate_vxlan_encap(const struct rte_flow_action *action,
1488                               struct rte_flow_error *error)
1489 {
1490         const struct rte_flow_item *items;
1491         int ret;
1492         uint32_t item_flags = 0;
1493
1494         if (!action->conf)
1495                 return rte_flow_error_set(error, EINVAL,
1496                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1497                                           "Missing vxlan tunnel"
1498                                           " action configuration");
1499         items = ((const struct rte_flow_action_vxlan_encap *)
1500                                         action->conf)->definition;
1501         if (!items)
1502                 return rte_flow_error_set(error, EINVAL,
1503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1504                                           "Missing vxlan tunnel"
1505                                           " encapsulation parameters");
1506         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1507                 switch (items->type) {
1508                 case RTE_FLOW_ITEM_TYPE_VOID:
1509                         break;
1510                 case RTE_FLOW_ITEM_TYPE_ETH:
1511                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1512                                                           error);
1513                         if (ret < 0)
1514                                 return ret;
1515                         ret = flow_tcf_validate_vxlan_encap_eth(items, error);
1516                         if (ret < 0)
1517                                 return ret;
1518                         item_flags |= MLX5_FLOW_LAYER_OUTER_L2;
1519                         break;
1520                 break;
1521                 case RTE_FLOW_ITEM_TYPE_IPV4:
1522                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1523                                                            error);
1524                         if (ret < 0)
1525                                 return ret;
1526                         ret = flow_tcf_validate_vxlan_encap_ipv4(items, error);
1527                         if (ret < 0)
1528                                 return ret;
1529                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1530                         break;
1531                 case RTE_FLOW_ITEM_TYPE_IPV6:
1532                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1533                                                            error);
1534                         if (ret < 0)
1535                                 return ret;
1536                         ret = flow_tcf_validate_vxlan_encap_ipv6(items, error);
1537                         if (ret < 0)
1538                                 return ret;
1539                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1540                         break;
1541                 case RTE_FLOW_ITEM_TYPE_UDP:
1542                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1543                                                            0xFF, error);
1544                         if (ret < 0)
1545                                 return ret;
1546                         ret = flow_tcf_validate_vxlan_encap_udp(items, error);
1547                         if (ret < 0)
1548                                 return ret;
1549                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
1550                         break;
1551                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1552                         ret = mlx5_flow_validate_item_vxlan(items,
1553                                                             item_flags, error);
1554                         if (ret < 0)
1555                                 return ret;
1556                         ret = flow_tcf_validate_vxlan_encap_vni(items, error);
1557                         if (ret < 0)
1558                                 return ret;
1559                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1560                         break;
1561                 default:
1562                         return rte_flow_error_set
1563                                         (error, ENOTSUP,
1564                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1565                                          "vxlan encap item not supported");
1566                 }
1567         }
1568         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1569                 return rte_flow_error_set(error, EINVAL,
1570                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1571                                           "no outer IP layer found"
1572                                           " for vxlan encapsulation");
1573         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1574                 return rte_flow_error_set(error, EINVAL,
1575                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1576                                           "no outer UDP layer found"
1577                                           " for vxlan encapsulation");
1578         if (!(item_flags & MLX5_FLOW_LAYER_VXLAN))
1579                 return rte_flow_error_set(error, EINVAL,
1580                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1581                                           "no VXLAN VNI found"
1582                                           " for vxlan encapsulation");
1583         return 0;
1584 }
1585
1586 /**
1587  * Validate RTE_FLOW_ITEM_TYPE_IPV4 item if VXLAN_DECAP action
1588  * is present in actions list.
1589  *
1590  * @param[in] ipv4
1591  *   Outer IPv4 address item (if any, NULL otherwise).
1592  * @param[out] error
1593  *   Pointer to the error structure.
1594  *
1595  * @return
1596  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1597  **/
1598 static int
1599 flow_tcf_validate_vxlan_decap_ipv4(const struct rte_flow_item *ipv4,
1600                                    struct rte_flow_error *error)
1601 {
1602         const struct rte_flow_item_ipv4 *spec = ipv4->spec;
1603         const struct rte_flow_item_ipv4 *mask = ipv4->mask;
1604
1605         if (!spec) {
1606                 /*
1607                  * Specification for IP addresses cannot be empty
1608                  * because it is required as decap parameter.
1609                  */
1610                 return rte_flow_error_set(error, EINVAL,
1611                                           RTE_FLOW_ERROR_TYPE_ITEM, ipv4,
1612                                           "NULL outer ipv4 address"
1613                                           " specification for vxlan"
1614                                           " for vxlan decapsulation");
1615         }
1616         if (!mask)
1617                 mask = &rte_flow_item_ipv4_mask;
1618         if (mask->hdr.dst_addr != RTE_BE32(0x00000000)) {
1619                 if (mask->hdr.dst_addr != RTE_BE32(0xffffffff))
1620                         return rte_flow_error_set
1621                                         (error, ENOTSUP,
1622                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1623                                          "no support for partial mask on"
1624                                          " \"ipv4.hdr.dst_addr\" field");
1625                 /* More IP address validations can be put here. */
1626         } else {
1627                 /*
1628                  * Kernel uses the destination IP address
1629                  * to determine the ingress network interface
1630                  * for traffic being decapsulated.
1631                  */
1632                 return rte_flow_error_set(error, EINVAL,
1633                                           RTE_FLOW_ERROR_TYPE_ITEM, ipv4,
1634                                           "outer ipv4 destination address"
1635                                           " must be specified for"
1636                                           " vxlan decapsulation");
1637         }
1638         /* Source IP address is optional for decap. */
1639         if (mask->hdr.src_addr != RTE_BE32(0x00000000) &&
1640             mask->hdr.src_addr != RTE_BE32(0xffffffff))
1641                 return rte_flow_error_set(error, ENOTSUP,
1642                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1643                                           "no support for partial mask on"
1644                                           " \"ipv4.hdr.src_addr\" field");
1645         return 0;
1646 }
1647
1648 /**
1649  * Validate RTE_FLOW_ITEM_TYPE_IPV6 item if VXLAN_DECAP action
1650  * is present in actions list.
1651  *
1652  * @param[in] ipv6
1653  *   Outer IPv6 address item (if any, NULL otherwise).
1654  * @param[out] error
1655  *   Pointer to the error structure.
1656  *
1657  * @return
1658  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1659  **/
1660 static int
1661 flow_tcf_validate_vxlan_decap_ipv6(const struct rte_flow_item *ipv6,
1662                                    struct rte_flow_error *error)
1663 {
1664         const struct rte_flow_item_ipv6 *spec = ipv6->spec;
1665         const struct rte_flow_item_ipv6 *mask = ipv6->mask;
1666
1667         if (!spec) {
1668                 /*
1669                  * Specification for IP addresses cannot be empty
1670                  * because it is required as decap parameter.
1671                  */
1672                 return rte_flow_error_set(error, EINVAL,
1673                                           RTE_FLOW_ERROR_TYPE_ITEM, ipv6,
1674                                           "NULL outer ipv6 address"
1675                                           " specification for vxlan"
1676                                           " decapsulation");
1677         }
1678         if (!mask)
1679                 mask = &rte_flow_item_ipv6_mask;
1680         if (memcmp(&mask->hdr.dst_addr,
1681                    &flow_tcf_mask_empty.ipv6.hdr.dst_addr,
1682                    IPV6_ADDR_LEN)) {
1683                 if (memcmp(&mask->hdr.dst_addr,
1684                         &rte_flow_item_ipv6_mask.hdr.dst_addr,
1685                         IPV6_ADDR_LEN))
1686                         return rte_flow_error_set
1687                                         (error, ENOTSUP,
1688                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1689                                          "no support for partial mask on"
1690                                          " \"ipv6.hdr.dst_addr\" field");
1691                 /* More IP address validations can be put here. */
1692         } else {
1693                 /*
1694                  * Kernel uses the destination IP address
1695                  * to determine the ingress network interface
1696                  * for traffic being decapsulated.
1697                  */
1698                 return rte_flow_error_set(error, EINVAL,
1699                                           RTE_FLOW_ERROR_TYPE_ITEM, ipv6,
1700                                           "outer ipv6 destination address must be "
1701                                           "specified for vxlan decapsulation");
1702         }
1703         /* Source IP address is optional for decap. */
1704         if (memcmp(&mask->hdr.src_addr,
1705                    &flow_tcf_mask_empty.ipv6.hdr.src_addr,
1706                    IPV6_ADDR_LEN)) {
1707                 if (memcmp(&mask->hdr.src_addr,
1708                            &rte_flow_item_ipv6_mask.hdr.src_addr,
1709                            IPV6_ADDR_LEN))
1710                         return rte_flow_error_set
1711                                         (error, ENOTSUP,
1712                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1713                                          "no support for partial mask on"
1714                                          " \"ipv6.hdr.src_addr\" field");
1715         }
1716         return 0;
1717 }
1718
1719 /**
1720  * Validate RTE_FLOW_ITEM_TYPE_UDP item if VXLAN_DECAP action
1721  * is present in actions list.
1722  *
1723  * @param[in] udp
1724  *   Outer UDP layer item (if any, NULL otherwise).
1725  * @param[out] error
1726  *   Pointer to the error structure.
1727  *
1728  * @return
1729  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1730  **/
1731 static int
1732 flow_tcf_validate_vxlan_decap_udp(const struct rte_flow_item *udp,
1733                                   struct rte_flow_error *error)
1734 {
1735         const struct rte_flow_item_udp *spec = udp->spec;
1736         const struct rte_flow_item_udp *mask = udp->mask;
1737
1738         if (!spec)
1739                 /*
1740                  * Specification for UDP ports cannot be empty
1741                  * because it is required as decap parameter.
1742                  */
1743                 return rte_flow_error_set(error, EINVAL,
1744                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1745                                           "NULL UDP port specification"
1746                                           " for VXLAN decapsulation");
1747         if (!mask)
1748                 mask = &rte_flow_item_udp_mask;
1749         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1750                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1751                         return rte_flow_error_set
1752                                         (error, ENOTSUP,
1753                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1754                                          "no support for partial mask on"
1755                                          " \"udp.hdr.dst_port\" field");
1756                 if (!spec->hdr.dst_port)
1757                         return rte_flow_error_set
1758                                         (error, EINVAL,
1759                                          RTE_FLOW_ERROR_TYPE_ITEM, udp,
1760                                          "zero decap local UDP port");
1761         } else {
1762                 return rte_flow_error_set(error, EINVAL,
1763                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1764                                           "outer UDP destination port must be "
1765                                           "specified for vxlan decapsulation");
1766         }
1767         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1768                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1769                         return rte_flow_error_set
1770                                         (error, ENOTSUP,
1771                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1772                                          "no support for partial mask on"
1773                                          " \"udp.hdr.src_port\" field");
1774                 DRV_LOG(WARNING,
1775                         "outer UDP local port cannot be "
1776                         "forced for VXLAN encapsulation, "
1777                         "parameter ignored");
1778         }
1779         return 0;
1780 }
1781
1782 /**
1783  * Validate flow for E-Switch.
1784  *
1785  * @param[in] priv
1786  *   Pointer to the priv structure.
1787  * @param[in] attr
1788  *   Pointer to the flow attributes.
1789  * @param[in] items
1790  *   Pointer to the list of items.
1791  * @param[in] actions
1792  *   Pointer to the list of actions.
1793  * @param[out] error
1794  *   Pointer to the error structure.
1795  *
1796  * @return
1797  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1798  */
1799 static int
1800 flow_tcf_validate(struct rte_eth_dev *dev,
1801                   const struct rte_flow_attr *attr,
1802                   const struct rte_flow_item items[],
1803                   const struct rte_flow_action actions[],
1804                   struct rte_flow_error *error)
1805 {
1806         union {
1807                 const struct rte_flow_item_port_id *port_id;
1808                 const struct rte_flow_item_eth *eth;
1809                 const struct rte_flow_item_vlan *vlan;
1810                 const struct rte_flow_item_ipv4 *ipv4;
1811                 const struct rte_flow_item_ipv6 *ipv6;
1812                 const struct rte_flow_item_tcp *tcp;
1813                 const struct rte_flow_item_udp *udp;
1814                 const struct rte_flow_item_vxlan *vxlan;
1815         } spec, mask;
1816         union {
1817                 const struct rte_flow_action_port_id *port_id;
1818                 const struct rte_flow_action_jump *jump;
1819                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
1820                 const struct rte_flow_action_of_set_vlan_vid *
1821                         of_set_vlan_vid;
1822                 const struct rte_flow_action_of_set_vlan_pcp *
1823                         of_set_vlan_pcp;
1824                 const struct rte_flow_action_vxlan_encap *vxlan_encap;
1825                 const struct rte_flow_action_set_ipv4 *set_ipv4;
1826                 const struct rte_flow_action_set_ipv6 *set_ipv6;
1827         } conf;
1828         uint64_t item_flags = 0;
1829         uint64_t action_flags = 0;
1830         uint8_t next_protocol = -1;
1831         unsigned int tcm_ifindex = 0;
1832         uint8_t pedit_validated = 0;
1833         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
1834         struct rte_eth_dev *port_id_dev = NULL;
1835         bool in_port_id_set;
1836         int ret;
1837
1838         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
1839                                                 PTOI_TABLE_SZ_MAX(dev)));
1840         ret = flow_tcf_validate_attributes(attr, error);
1841         if (ret < 0)
1842                 return ret;
1843         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1844                 unsigned int i;
1845                 uint64_t current_action_flag = 0;
1846
1847                 switch (actions->type) {
1848                 case RTE_FLOW_ACTION_TYPE_VOID:
1849                         break;
1850                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
1851                         current_action_flag = MLX5_FLOW_ACTION_PORT_ID;
1852                         if (!actions->conf)
1853                                 break;
1854                         conf.port_id = actions->conf;
1855                         if (conf.port_id->original)
1856                                 i = 0;
1857                         else
1858                                 for (i = 0; ptoi[i].ifindex; ++i)
1859                                         if (ptoi[i].port_id == conf.port_id->id)
1860                                                 break;
1861                         if (!ptoi[i].ifindex)
1862                                 return rte_flow_error_set
1863                                         (error, ENODEV,
1864                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1865                                          conf.port_id,
1866                                          "missing data to convert port ID to"
1867                                          " ifindex");
1868                         port_id_dev = &rte_eth_devices[conf.port_id->id];
1869                         break;
1870                 case RTE_FLOW_ACTION_TYPE_JUMP:
1871                         current_action_flag = MLX5_FLOW_ACTION_JUMP;
1872                         if (!actions->conf)
1873                                 break;
1874                         conf.jump = actions->conf;
1875                         if (attr->group >= conf.jump->group)
1876                                 return rte_flow_error_set
1877                                         (error, ENOTSUP,
1878                                          RTE_FLOW_ERROR_TYPE_ACTION,
1879                                          actions,
1880                                          "can jump only to a group forward");
1881                         break;
1882                 case RTE_FLOW_ACTION_TYPE_DROP:
1883                         current_action_flag = MLX5_FLOW_ACTION_DROP;
1884                         break;
1885                 case RTE_FLOW_ACTION_TYPE_COUNT:
1886                         break;
1887                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
1888                         current_action_flag = MLX5_FLOW_ACTION_OF_POP_VLAN;
1889                         break;
1890                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
1891                         current_action_flag = MLX5_FLOW_ACTION_OF_PUSH_VLAN;
1892                         break;
1893                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
1894                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1895                                 return rte_flow_error_set
1896                                         (error, ENOTSUP,
1897                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1898                                          "vlan modify is not supported,"
1899                                          " set action must follow push action");
1900                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
1901                         break;
1902                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
1903                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1904                                 return rte_flow_error_set
1905                                         (error, ENOTSUP,
1906                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1907                                          "vlan modify is not supported,"
1908                                          " set action must follow push action");
1909                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
1910                         break;
1911                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1912                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_DECAP;
1913                         break;
1914                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1915                         ret = flow_tcf_validate_vxlan_encap(actions, error);
1916                         if (ret < 0)
1917                                 return ret;
1918                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_ENCAP;
1919                         break;
1920                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
1921                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_SRC;
1922                         break;
1923                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
1924                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_DST;
1925                         break;
1926                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
1927                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_SRC;
1928                         break;
1929                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
1930                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_DST;
1931                         break;
1932                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
1933                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_SRC;
1934                         break;
1935                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
1936                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_DST;
1937                         break;
1938                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
1939                         current_action_flag = MLX5_FLOW_ACTION_SET_TTL;
1940                         break;
1941                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
1942                         current_action_flag = MLX5_FLOW_ACTION_DEC_TTL;
1943                         break;
1944                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
1945                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_SRC;
1946                         break;
1947                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
1948                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_DST;
1949                         break;
1950                 default:
1951                         return rte_flow_error_set(error, ENOTSUP,
1952                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1953                                                   actions,
1954                                                   "action not supported");
1955                 }
1956                 if (current_action_flag & MLX5_TCF_CONFIG_ACTIONS) {
1957                         if (!actions->conf)
1958                                 return rte_flow_error_set
1959                                         (error, EINVAL,
1960                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1961                                          actions,
1962                                          "action configuration not set");
1963                 }
1964                 if ((current_action_flag & MLX5_TCF_PEDIT_ACTIONS) &&
1965                     pedit_validated)
1966                         return rte_flow_error_set(error, ENOTSUP,
1967                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1968                                                   actions,
1969                                                   "set actions should be "
1970                                                   "listed successively");
1971                 if ((current_action_flag & ~MLX5_TCF_PEDIT_ACTIONS) &&
1972                     (action_flags & MLX5_TCF_PEDIT_ACTIONS))
1973                         pedit_validated = 1;
1974                 if ((current_action_flag & MLX5_TCF_FATE_ACTIONS) &&
1975                     (action_flags & MLX5_TCF_FATE_ACTIONS))
1976                         return rte_flow_error_set(error, EINVAL,
1977                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1978                                                   actions,
1979                                                   "can't have multiple fate"
1980                                                   " actions");
1981                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1982                     (action_flags & MLX5_TCF_VXLAN_ACTIONS))
1983                         return rte_flow_error_set(error, EINVAL,
1984                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1985                                                   actions,
1986                                                   "can't have multiple vxlan"
1987                                                   " actions");
1988                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1989                     (action_flags & MLX5_TCF_VLAN_ACTIONS))
1990                         return rte_flow_error_set(error, ENOTSUP,
1991                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1992                                                   actions,
1993                                                   "can't have vxlan and vlan"
1994                                                   " actions in the same rule");
1995                 action_flags |= current_action_flag;
1996         }
1997         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1998                 unsigned int i;
1999
2000                 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2001                     items->type != RTE_FLOW_ITEM_TYPE_ETH)
2002                         return rte_flow_error_set(error, ENOTSUP,
2003                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2004                                                   items,
2005                                                   "only L2 inner item"
2006                                                   " is supported");
2007                 switch (items->type) {
2008                 case RTE_FLOW_ITEM_TYPE_VOID:
2009                         break;
2010                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2011                         mask.port_id = flow_tcf_item_mask
2012                                 (items, &rte_flow_item_port_id_mask,
2013                                  &flow_tcf_mask_supported.port_id,
2014                                  &flow_tcf_mask_empty.port_id,
2015                                  sizeof(flow_tcf_mask_supported.port_id),
2016                                  error);
2017                         if (!mask.port_id)
2018                                 return -rte_errno;
2019                         if (mask.port_id == &flow_tcf_mask_empty.port_id) {
2020                                 in_port_id_set = 1;
2021                                 break;
2022                         }
2023                         spec.port_id = items->spec;
2024                         if (mask.port_id->id && mask.port_id->id != 0xffffffff)
2025                                 return rte_flow_error_set
2026                                         (error, ENOTSUP,
2027                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2028                                          mask.port_id,
2029                                          "no support for partial mask on"
2030                                          " \"id\" field");
2031                         if (!mask.port_id->id)
2032                                 i = 0;
2033                         else
2034                                 for (i = 0; ptoi[i].ifindex; ++i)
2035                                         if (ptoi[i].port_id == spec.port_id->id)
2036                                                 break;
2037                         if (!ptoi[i].ifindex)
2038                                 return rte_flow_error_set
2039                                         (error, ENODEV,
2040                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2041                                          spec.port_id,
2042                                          "missing data to convert port ID to"
2043                                          " ifindex");
2044                         if (in_port_id_set && ptoi[i].ifindex != tcm_ifindex)
2045                                 return rte_flow_error_set
2046                                         (error, ENOTSUP,
2047                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2048                                          spec.port_id,
2049                                          "cannot match traffic for"
2050                                          " several port IDs through"
2051                                          " a single flow rule");
2052                         tcm_ifindex = ptoi[i].ifindex;
2053                         in_port_id_set = 1;
2054                         break;
2055                 case RTE_FLOW_ITEM_TYPE_ETH:
2056                         ret = mlx5_flow_validate_item_eth(items, item_flags,
2057                                                           error);
2058                         if (ret < 0)
2059                                 return ret;
2060                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2061                                         MLX5_FLOW_LAYER_INNER_L2 :
2062                                         MLX5_FLOW_LAYER_OUTER_L2;
2063                         /* TODO:
2064                          * Redundant check due to different supported mask.
2065                          * Same for the rest of items.
2066                          */
2067                         mask.eth = flow_tcf_item_mask
2068                                 (items, &rte_flow_item_eth_mask,
2069                                  &flow_tcf_mask_supported.eth,
2070                                  &flow_tcf_mask_empty.eth,
2071                                  sizeof(flow_tcf_mask_supported.eth),
2072                                  error);
2073                         if (!mask.eth)
2074                                 return -rte_errno;
2075                         if (mask.eth->type && mask.eth->type !=
2076                             RTE_BE16(0xffff))
2077                                 return rte_flow_error_set
2078                                         (error, ENOTSUP,
2079                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2080                                          mask.eth,
2081                                          "no support for partial mask on"
2082                                          " \"type\" field");
2083                         break;
2084                 case RTE_FLOW_ITEM_TYPE_VLAN:
2085                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
2086                                                            error);
2087                         if (ret < 0)
2088                                 return ret;
2089                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
2090                         mask.vlan = flow_tcf_item_mask
2091                                 (items, &rte_flow_item_vlan_mask,
2092                                  &flow_tcf_mask_supported.vlan,
2093                                  &flow_tcf_mask_empty.vlan,
2094                                  sizeof(flow_tcf_mask_supported.vlan),
2095                                  error);
2096                         if (!mask.vlan)
2097                                 return -rte_errno;
2098                         if ((mask.vlan->tci & RTE_BE16(0xe000) &&
2099                              (mask.vlan->tci & RTE_BE16(0xe000)) !=
2100                               RTE_BE16(0xe000)) ||
2101                             (mask.vlan->tci & RTE_BE16(0x0fff) &&
2102                              (mask.vlan->tci & RTE_BE16(0x0fff)) !=
2103                               RTE_BE16(0x0fff)) ||
2104                             (mask.vlan->inner_type &&
2105                              mask.vlan->inner_type != RTE_BE16(0xffff)))
2106                                 return rte_flow_error_set
2107                                         (error, ENOTSUP,
2108                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2109                                          mask.vlan,
2110                                          "no support for partial masks on"
2111                                          " \"tci\" (PCP and VID parts) and"
2112                                          " \"inner_type\" fields");
2113                         break;
2114                 case RTE_FLOW_ITEM_TYPE_IPV4:
2115                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2116                                                            error);
2117                         if (ret < 0)
2118                                 return ret;
2119                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2120                         mask.ipv4 = flow_tcf_item_mask
2121                                 (items, &rte_flow_item_ipv4_mask,
2122                                  &flow_tcf_mask_supported.ipv4,
2123                                  &flow_tcf_mask_empty.ipv4,
2124                                  sizeof(flow_tcf_mask_supported.ipv4),
2125                                  error);
2126                         if (!mask.ipv4)
2127                                 return -rte_errno;
2128                         if (mask.ipv4->hdr.next_proto_id &&
2129                             mask.ipv4->hdr.next_proto_id != 0xff)
2130                                 return rte_flow_error_set
2131                                         (error, ENOTSUP,
2132                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2133                                          mask.ipv4,
2134                                          "no support for partial mask on"
2135                                          " \"hdr.next_proto_id\" field");
2136                         else if (mask.ipv4->hdr.next_proto_id)
2137                                 next_protocol =
2138                                         ((const struct rte_flow_item_ipv4 *)
2139                                          (items->spec))->hdr.next_proto_id;
2140                         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2141                                 ret = flow_tcf_validate_vxlan_decap_ipv4
2142                                                                 (items, error);
2143                                 if (ret < 0)
2144                                         return ret;
2145                         }
2146                         break;
2147                 case RTE_FLOW_ITEM_TYPE_IPV6:
2148                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2149                                                            error);
2150                         if (ret < 0)
2151                                 return ret;
2152                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2153                         mask.ipv6 = flow_tcf_item_mask
2154                                 (items, &rte_flow_item_ipv6_mask,
2155                                  &flow_tcf_mask_supported.ipv6,
2156                                  &flow_tcf_mask_empty.ipv6,
2157                                  sizeof(flow_tcf_mask_supported.ipv6),
2158                                  error);
2159                         if (!mask.ipv6)
2160                                 return -rte_errno;
2161                         if (mask.ipv6->hdr.proto &&
2162                             mask.ipv6->hdr.proto != 0xff)
2163                                 return rte_flow_error_set
2164                                         (error, ENOTSUP,
2165                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2166                                          mask.ipv6,
2167                                          "no support for partial mask on"
2168                                          " \"hdr.proto\" field");
2169                         else if (mask.ipv6->hdr.proto)
2170                                 next_protocol =
2171                                         ((const struct rte_flow_item_ipv6 *)
2172                                          (items->spec))->hdr.proto;
2173                         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2174                                 ret = flow_tcf_validate_vxlan_decap_ipv6
2175                                                                 (items, error);
2176                                 if (ret < 0)
2177                                         return ret;
2178                         }
2179                         break;
2180                 case RTE_FLOW_ITEM_TYPE_UDP:
2181                         ret = mlx5_flow_validate_item_udp(items, item_flags,
2182                                                           next_protocol, error);
2183                         if (ret < 0)
2184                                 return ret;
2185                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
2186                         mask.udp = flow_tcf_item_mask
2187                                 (items, &rte_flow_item_udp_mask,
2188                                  &flow_tcf_mask_supported.udp,
2189                                  &flow_tcf_mask_empty.udp,
2190                                  sizeof(flow_tcf_mask_supported.udp),
2191                                  error);
2192                         if (!mask.udp)
2193                                 return -rte_errno;
2194                         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2195                                 ret = flow_tcf_validate_vxlan_decap_udp
2196                                                                 (items, error);
2197                                 if (ret < 0)
2198                                         return ret;
2199                         }
2200                         break;
2201                 case RTE_FLOW_ITEM_TYPE_TCP:
2202                         ret = mlx5_flow_validate_item_tcp
2203                                              (items, item_flags,
2204                                               next_protocol,
2205                                               &flow_tcf_mask_supported.tcp,
2206                                               error);
2207                         if (ret < 0)
2208                                 return ret;
2209                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
2210                         mask.tcp = flow_tcf_item_mask
2211                                 (items, &rte_flow_item_tcp_mask,
2212                                  &flow_tcf_mask_supported.tcp,
2213                                  &flow_tcf_mask_empty.tcp,
2214                                  sizeof(flow_tcf_mask_supported.tcp),
2215                                  error);
2216                         if (!mask.tcp)
2217                                 return -rte_errno;
2218                         break;
2219                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2220                         if (!(action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP))
2221                                 return rte_flow_error_set
2222                                         (error, ENOTSUP,
2223                                          RTE_FLOW_ERROR_TYPE_ITEM,
2224                                          items,
2225                                          "vni pattern should be followed by"
2226                                          " vxlan decapsulation action");
2227                         ret = mlx5_flow_validate_item_vxlan(items,
2228                                                             item_flags, error);
2229                         if (ret < 0)
2230                                 return ret;
2231                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
2232                         mask.vxlan = flow_tcf_item_mask
2233                                 (items, &rte_flow_item_vxlan_mask,
2234                                  &flow_tcf_mask_supported.vxlan,
2235                                  &flow_tcf_mask_empty.vxlan,
2236                                  sizeof(flow_tcf_mask_supported.vxlan), error);
2237                         if (!mask.vxlan)
2238                                 return -rte_errno;
2239                         if (mask.vxlan->vni[0] != 0xff ||
2240                             mask.vxlan->vni[1] != 0xff ||
2241                             mask.vxlan->vni[2] != 0xff)
2242                                 return rte_flow_error_set
2243                                         (error, ENOTSUP,
2244                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2245                                          mask.vxlan,
2246                                          "no support for partial or "
2247                                          "empty mask on \"vxlan.vni\" field");
2248                         break;
2249                 default:
2250                         return rte_flow_error_set(error, ENOTSUP,
2251                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2252                                                   items, "item not supported");
2253                 }
2254         }
2255         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2256             (action_flags & MLX5_FLOW_ACTION_DROP))
2257                 return rte_flow_error_set(error, ENOTSUP,
2258                                           RTE_FLOW_ERROR_TYPE_ACTION,
2259                                           actions,
2260                                           "set action is not compatible with "
2261                                           "drop action");
2262         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2263             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2264                 return rte_flow_error_set(error, ENOTSUP,
2265                                           RTE_FLOW_ERROR_TYPE_ACTION,
2266                                           actions,
2267                                           "set action must be followed by "
2268                                           "port_id action");
2269         if (action_flags &
2270            (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST)) {
2271                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4))
2272                         return rte_flow_error_set(error, EINVAL,
2273                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2274                                                   actions,
2275                                                   "no ipv4 item found in"
2276                                                   " pattern");
2277         }
2278         if (action_flags &
2279            (MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST)) {
2280                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6))
2281                         return rte_flow_error_set(error, EINVAL,
2282                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2283                                                   actions,
2284                                                   "no ipv6 item found in"
2285                                                   " pattern");
2286         }
2287         if (action_flags &
2288            (MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST)) {
2289                 if (!(item_flags &
2290                      (MLX5_FLOW_LAYER_OUTER_L4_UDP |
2291                       MLX5_FLOW_LAYER_OUTER_L4_TCP)))
2292                         return rte_flow_error_set(error, EINVAL,
2293                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2294                                                   actions,
2295                                                   "no TCP/UDP item found in"
2296                                                   " pattern");
2297         }
2298         /*
2299          * FW syndrome (0xA9C090):
2300          *     set_flow_table_entry: push vlan action fte in fdb can ONLY be
2301          *     forward to the uplink.
2302          */
2303         if ((action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2304             (action_flags & MLX5_FLOW_ACTION_PORT_ID) &&
2305             ((struct priv *)port_id_dev->data->dev_private)->representor)
2306                 return rte_flow_error_set(error, ENOTSUP,
2307                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2308                                           "vlan push can only be applied"
2309                                           " when forwarding to uplink port");
2310         /*
2311          * FW syndrome (0x294609):
2312          *     set_flow_table_entry: modify/pop/push actions in fdb flow table
2313          *     are supported only while forwarding to vport.
2314          */
2315         if ((action_flags & MLX5_TCF_VLAN_ACTIONS) &&
2316             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2317                 return rte_flow_error_set(error, ENOTSUP,
2318                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2319                                           "vlan actions are supported"
2320                                           " only with port_id action");
2321         if ((action_flags & MLX5_TCF_VXLAN_ACTIONS) &&
2322             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2323                 return rte_flow_error_set(error, ENOTSUP,
2324                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2325                                           "vxlan actions are supported"
2326                                           " only with port_id action");
2327         if (!(action_flags & MLX5_TCF_FATE_ACTIONS))
2328                 return rte_flow_error_set(error, EINVAL,
2329                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2330                                           "no fate action is found");
2331         if (action_flags &
2332            (MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL)) {
2333                 if (!(item_flags &
2334                      (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2335                       MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2336                         return rte_flow_error_set(error, EINVAL,
2337                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2338                                                   actions,
2339                                                   "no IP found in pattern");
2340         }
2341         if (action_flags &
2342             (MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)) {
2343                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L2))
2344                         return rte_flow_error_set(error, ENOTSUP,
2345                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2346                                                   actions,
2347                                                   "no ethernet found in"
2348                                                   " pattern");
2349         }
2350         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2351                 if (!(item_flags &
2352                      (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2353                       MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2354                         return rte_flow_error_set(error, EINVAL,
2355                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2356                                                   NULL,
2357                                                   "no outer IP pattern found"
2358                                                   " for vxlan decap action");
2359                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2360                         return rte_flow_error_set(error, EINVAL,
2361                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2362                                                   NULL,
2363                                                   "no outer UDP pattern found"
2364                                                   " for vxlan decap action");
2365                 if (!(item_flags & MLX5_FLOW_LAYER_VXLAN))
2366                         return rte_flow_error_set(error, EINVAL,
2367                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2368                                                   NULL,
2369                                                   "no VNI pattern found"
2370                                                   " for vxlan decap action");
2371         }
2372         return 0;
2373 }
2374
2375 /**
2376  * Calculate maximum size of memory for flow items of Linux TC flower.
2377  *
2378  * @param[in] attr
2379  *   Pointer to the flow attributes.
2380  * @param[in] items
2381  *   Pointer to the list of items.
2382  *
2383  * @return
2384  *   Maximum size of memory for items.
2385  */
2386 static int
2387 flow_tcf_get_items_size(const struct rte_flow_attr *attr,
2388                         const struct rte_flow_item items[])
2389 {
2390         int size = 0;
2391
2392         size += SZ_NLATTR_STRZ_OF("flower") +
2393                 SZ_NLATTR_NEST + /* TCA_OPTIONS. */
2394                 SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CLS_FLAGS_SKIP_SW. */
2395         if (attr->group > 0)
2396                 size += SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CHAIN. */
2397         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2398                 switch (items->type) {
2399                 case RTE_FLOW_ITEM_TYPE_VOID:
2400                         break;
2401                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2402                         break;
2403                 case RTE_FLOW_ITEM_TYPE_ETH:
2404                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2405                                 SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) * 4;
2406                                 /* dst/src MAC addr and mask. */
2407                         break;
2408                 case RTE_FLOW_ITEM_TYPE_VLAN:
2409                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2410                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2411                                 /* VLAN Ether type. */
2412                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* VLAN prio. */
2413                                 SZ_NLATTR_TYPE_OF(uint16_t); /* VLAN ID. */
2414                         break;
2415                 case RTE_FLOW_ITEM_TYPE_IPV4:
2416                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2417                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2418                                 SZ_NLATTR_TYPE_OF(uint32_t) * 4;
2419                                 /* dst/src IP addr and mask. */
2420                         break;
2421                 case RTE_FLOW_ITEM_TYPE_IPV6:
2422                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2423                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2424                                 SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 4;
2425                                 /* dst/src IP addr and mask. */
2426                         break;
2427                 case RTE_FLOW_ITEM_TYPE_UDP:
2428                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2429                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2430                                 /* dst/src port and mask. */
2431                         break;
2432                 case RTE_FLOW_ITEM_TYPE_TCP:
2433                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2434                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2435                                 /* dst/src port and mask. */
2436                         break;
2437                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2438                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2439                         break;
2440                 default:
2441                         DRV_LOG(WARNING,
2442                                 "unsupported item %p type %d,"
2443                                 " items must be validated before flow creation",
2444                                 (const void *)items, items->type);
2445                         break;
2446                 }
2447         }
2448         return size;
2449 }
2450
2451 /**
2452  * Calculate size of memory to store the VXLAN encapsultion
2453  * related items in the Netlink message buffer. Items list
2454  * is specified by RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action.
2455  * The item list should be validated.
2456  *
2457  * @param[in] action
2458  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2459  *   List of pattern items to scan data from.
2460  *
2461  * @return
2462  *   The size the part of Netlink message buffer to store the
2463  *   VXLAN encapsulation item attributes.
2464  */
2465 static int
2466 flow_tcf_vxlan_encap_size(const struct rte_flow_action *action)
2467 {
2468         const struct rte_flow_item *items;
2469         int size = 0;
2470
2471         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2472         assert(action->conf);
2473
2474         items = ((const struct rte_flow_action_vxlan_encap *)
2475                                         action->conf)->definition;
2476         assert(items);
2477         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2478                 switch (items->type) {
2479                 case RTE_FLOW_ITEM_TYPE_VOID:
2480                         break;
2481                 case RTE_FLOW_ITEM_TYPE_ETH:
2482                         /* This item does not require message buffer. */
2483                         break;
2484                 case RTE_FLOW_ITEM_TYPE_IPV4:
2485                         size += SZ_NLATTR_DATA_OF(IPV4_ADDR_LEN) * 2;
2486                         break;
2487                 case RTE_FLOW_ITEM_TYPE_IPV6:
2488                         size += SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 2;
2489                         break;
2490                 case RTE_FLOW_ITEM_TYPE_UDP: {
2491                         const struct rte_flow_item_udp *udp = items->mask;
2492
2493                         size += SZ_NLATTR_TYPE_OF(uint16_t);
2494                         if (!udp || udp->hdr.src_port != RTE_BE16(0x0000))
2495                                 size += SZ_NLATTR_TYPE_OF(uint16_t);
2496                         break;
2497                 }
2498                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2499                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2500                         break;
2501                 default:
2502                         assert(false);
2503                         DRV_LOG(WARNING,
2504                                 "unsupported item %p type %d,"
2505                                 " items must be validated"
2506                                 " before flow creation",
2507                                 (const void *)items, items->type);
2508                         return 0;
2509                 }
2510         }
2511         return size;
2512 }
2513
2514 /**
2515  * Calculate maximum size of memory for flow actions of Linux TC flower and
2516  * extract specified actions.
2517  *
2518  * @param[in] actions
2519  *   Pointer to the list of actions.
2520  * @param[out] action_flags
2521  *   Pointer to the detected actions.
2522  *
2523  * @return
2524  *   Maximum size of memory for actions.
2525  */
2526 static int
2527 flow_tcf_get_actions_and_size(const struct rte_flow_action actions[],
2528                               uint64_t *action_flags)
2529 {
2530         int size = 0;
2531         uint64_t flags = 0;
2532
2533         size += SZ_NLATTR_NEST; /* TCA_FLOWER_ACT. */
2534         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2535                 switch (actions->type) {
2536                 case RTE_FLOW_ACTION_TYPE_VOID:
2537                         break;
2538                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
2539                         size += SZ_NLATTR_NEST + /* na_act_index. */
2540                                 SZ_NLATTR_STRZ_OF("mirred") +
2541                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2542                                 SZ_NLATTR_TYPE_OF(struct tc_mirred);
2543                         flags |= MLX5_FLOW_ACTION_PORT_ID;
2544                         break;
2545                 case RTE_FLOW_ACTION_TYPE_JUMP:
2546                         size += SZ_NLATTR_NEST + /* na_act_index. */
2547                                 SZ_NLATTR_STRZ_OF("gact") +
2548                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2549                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2550                         flags |= MLX5_FLOW_ACTION_JUMP;
2551                         break;
2552                 case RTE_FLOW_ACTION_TYPE_DROP:
2553                         size += SZ_NLATTR_NEST + /* na_act_index. */
2554                                 SZ_NLATTR_STRZ_OF("gact") +
2555                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2556                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2557                         flags |= MLX5_FLOW_ACTION_DROP;
2558                         break;
2559                 case RTE_FLOW_ACTION_TYPE_COUNT:
2560                         break;
2561                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
2562                         flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
2563                         goto action_of_vlan;
2564                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
2565                         flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
2566                         goto action_of_vlan;
2567                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
2568                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
2569                         goto action_of_vlan;
2570                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
2571                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
2572                         goto action_of_vlan;
2573 action_of_vlan:
2574                         size += SZ_NLATTR_NEST + /* na_act_index. */
2575                                 SZ_NLATTR_STRZ_OF("vlan") +
2576                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2577                                 SZ_NLATTR_TYPE_OF(struct tc_vlan) +
2578                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2579                                 /* VLAN protocol. */
2580                                 SZ_NLATTR_TYPE_OF(uint16_t) + /* VLAN ID. */
2581                                 SZ_NLATTR_TYPE_OF(uint8_t); /* VLAN prio. */
2582                         break;
2583                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2584                         size += SZ_NLATTR_NEST + /* na_act_index. */
2585                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2586                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2587                                 SZ_NLATTR_TYPE_OF(uint8_t);
2588                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2589                         size += flow_tcf_vxlan_encap_size(actions) +
2590                                 RTE_ALIGN_CEIL /* preceding encap params. */
2591                                 (sizeof(struct flow_tcf_vxlan_encap),
2592                                 MNL_ALIGNTO);
2593                         flags |= MLX5_FLOW_ACTION_VXLAN_ENCAP;
2594                         break;
2595                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
2596                         size += SZ_NLATTR_NEST + /* na_act_index. */
2597                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2598                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2599                                 SZ_NLATTR_TYPE_OF(uint8_t);
2600                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2601                         size += RTE_ALIGN_CEIL /* preceding decap params. */
2602                                 (sizeof(struct flow_tcf_vxlan_decap),
2603                                 MNL_ALIGNTO);
2604                         flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2605                         break;
2606                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
2607                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
2608                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
2609                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
2610                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
2611                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
2612                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
2613                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
2614                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
2615                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
2616                         size += flow_tcf_get_pedit_actions_size(&actions,
2617                                                                 &flags);
2618                         break;
2619                 default:
2620                         DRV_LOG(WARNING,
2621                                 "unsupported action %p type %d,"
2622                                 " items must be validated before flow creation",
2623                                 (const void *)actions, actions->type);
2624                         break;
2625                 }
2626         }
2627         *action_flags = flags;
2628         return size;
2629 }
2630
2631 /**
2632  * Brand rtnetlink buffer with unique handle.
2633  *
2634  * This handle should be unique for a given network interface to avoid
2635  * collisions.
2636  *
2637  * @param nlh
2638  *   Pointer to Netlink message.
2639  * @param handle
2640  *   Unique 32-bit handle to use.
2641  */
2642 static void
2643 flow_tcf_nl_brand(struct nlmsghdr *nlh, uint32_t handle)
2644 {
2645         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
2646
2647         tcm->tcm_handle = handle;
2648         DRV_LOG(DEBUG, "Netlink msg %p is branded with handle %x",
2649                 (void *)nlh, handle);
2650 }
2651
2652 /**
2653  * Prepare a flow object for Linux TC flower. It calculates the maximum size of
2654  * memory required, allocates the memory, initializes Netlink message headers
2655  * and set unique TC message handle.
2656  *
2657  * @param[in] attr
2658  *   Pointer to the flow attributes.
2659  * @param[in] items
2660  *   Pointer to the list of items.
2661  * @param[in] actions
2662  *   Pointer to the list of actions.
2663  * @param[out] error
2664  *   Pointer to the error structure.
2665  *
2666  * @return
2667  *   Pointer to mlx5_flow object on success,
2668  *   otherwise NULL and rte_ernno is set.
2669  */
2670 static struct mlx5_flow *
2671 flow_tcf_prepare(const struct rte_flow_attr *attr,
2672                  const struct rte_flow_item items[],
2673                  const struct rte_flow_action actions[],
2674                  struct rte_flow_error *error)
2675 {
2676         size_t size = RTE_ALIGN_CEIL
2677                         (sizeof(struct mlx5_flow),
2678                          alignof(struct flow_tcf_tunnel_hdr)) +
2679                       MNL_ALIGN(sizeof(struct nlmsghdr)) +
2680                       MNL_ALIGN(sizeof(struct tcmsg));
2681         struct mlx5_flow *dev_flow;
2682         uint64_t action_flags = 0;
2683         struct nlmsghdr *nlh;
2684         struct tcmsg *tcm;
2685         uint8_t *sp, *tun = NULL;
2686
2687         size += flow_tcf_get_items_size(attr, items);
2688         size += flow_tcf_get_actions_and_size(actions, &action_flags);
2689         dev_flow = rte_zmalloc(__func__, size, MNL_ALIGNTO);
2690         if (!dev_flow) {
2691                 rte_flow_error_set(error, ENOMEM,
2692                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2693                                    "not enough memory to create E-Switch flow");
2694                 return NULL;
2695         }
2696         sp = (uint8_t *)(dev_flow + 1);
2697         if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) {
2698                 sp = RTE_PTR_ALIGN
2699                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2700                 tun = sp;
2701                 sp += RTE_ALIGN_CEIL
2702                         (sizeof(struct flow_tcf_vxlan_encap),
2703                         MNL_ALIGNTO);
2704 #ifndef NDEBUG
2705                 size -= RTE_ALIGN_CEIL
2706                         (sizeof(struct flow_tcf_vxlan_encap),
2707                         MNL_ALIGNTO);
2708 #endif
2709         } else if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2710                 sp = RTE_PTR_ALIGN
2711                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2712                 tun = sp;
2713                 sp += RTE_ALIGN_CEIL
2714                         (sizeof(struct flow_tcf_vxlan_decap),
2715                         MNL_ALIGNTO);
2716 #ifndef NDEBUG
2717                 size -= RTE_ALIGN_CEIL
2718                         (sizeof(struct flow_tcf_vxlan_decap),
2719                         MNL_ALIGNTO);
2720 #endif
2721         } else {
2722                 sp = RTE_PTR_ALIGN(sp, MNL_ALIGNTO);
2723         }
2724         nlh = mnl_nlmsg_put_header(sp);
2725         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
2726         *dev_flow = (struct mlx5_flow){
2727                 .tcf = (struct mlx5_flow_tcf){
2728 #ifndef NDEBUG
2729                         .nlsize = size - RTE_ALIGN_CEIL
2730                                 (sizeof(struct mlx5_flow),
2731                                  alignof(struct flow_tcf_tunnel_hdr)),
2732 #endif
2733                         .tunnel = (struct flow_tcf_tunnel_hdr *)tun,
2734                         .nlh = nlh,
2735                         .tcm = tcm,
2736                 },
2737         };
2738         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP)
2739                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_DECAP;
2740         else if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP)
2741                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_ENCAP;
2742         /*
2743          * Generate a reasonably unique handle based on the address of the
2744          * target buffer.
2745          *
2746          * This is straightforward on 32-bit systems where the flow pointer can
2747          * be used directly. Otherwise, its least significant part is taken
2748          * after shifting it by the previous power of two of the pointed buffer
2749          * size.
2750          */
2751         if (sizeof(dev_flow) <= 4)
2752                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow);
2753         else
2754                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow >>
2755                                        rte_log2_u32(rte_align32prevpow2(size)));
2756         return dev_flow;
2757 }
2758
2759 /**
2760  * Make adjustments for supporting count actions.
2761  *
2762  * @param[in] dev
2763  *   Pointer to the Ethernet device structure.
2764  * @param[in] dev_flow
2765  *   Pointer to mlx5_flow.
2766  * @param[out] error
2767  *   Pointer to error structure.
2768  *
2769  * @return
2770  *   0 On success else a negative errno value is returned and rte_errno is set.
2771  */
2772 static int
2773 flow_tcf_translate_action_count(struct rte_eth_dev *dev __rte_unused,
2774                                   struct mlx5_flow *dev_flow,
2775                                   struct rte_flow_error *error)
2776 {
2777         struct rte_flow *flow = dev_flow->flow;
2778
2779         if (!flow->counter) {
2780                 flow->counter = flow_tcf_counter_new();
2781                 if (!flow->counter)
2782                         return rte_flow_error_set(error, rte_errno,
2783                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2784                                                   NULL,
2785                                                   "cannot get counter"
2786                                                   " context.");
2787         }
2788         return 0;
2789 }
2790
2791 /**
2792  * Convert VXLAN VNI to 32-bit integer.
2793  *
2794  * @param[in] vni
2795  *   VXLAN VNI in 24-bit wire format.
2796  *
2797  * @return
2798  *   VXLAN VNI as a 32-bit integer value in network endian.
2799  */
2800 static inline rte_be32_t
2801 vxlan_vni_as_be32(const uint8_t vni[3])
2802 {
2803         union {
2804                 uint8_t vni[4];
2805                 rte_be32_t dword;
2806         } ret = {
2807                 .vni = { 0, vni[0], vni[1], vni[2] },
2808         };
2809         return ret.dword;
2810 }
2811
2812 /**
2813  * Helper function to process RTE_FLOW_ITEM_TYPE_ETH entry in configuration
2814  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the MAC address fields
2815  * in the encapsulation parameters structure. The item must be prevalidated,
2816  * no any validation checks performed by function.
2817  *
2818  * @param[in] spec
2819  *   RTE_FLOW_ITEM_TYPE_ETH entry specification.
2820  * @param[in] mask
2821  *   RTE_FLOW_ITEM_TYPE_ETH entry mask.
2822  * @param[out] encap
2823  *   Structure to fill the gathered MAC address data.
2824  */
2825 static void
2826 flow_tcf_parse_vxlan_encap_eth(const struct rte_flow_item_eth *spec,
2827                                const struct rte_flow_item_eth *mask,
2828                                struct flow_tcf_vxlan_encap *encap)
2829 {
2830         /* Item must be validated before. No redundant checks. */
2831         assert(spec);
2832         if (!mask || !memcmp(&mask->dst,
2833                              &rte_flow_item_eth_mask.dst,
2834                              sizeof(rte_flow_item_eth_mask.dst))) {
2835                 /*
2836                  * Ethernet addresses are not supported by
2837                  * tc as tunnel_key parameters. Destination
2838                  * address is needed to form encap packet
2839                  * header and retrieved by kernel from
2840                  * implicit sources (ARP table, etc),
2841                  * address masks are not supported at all.
2842                  */
2843                 encap->eth.dst = spec->dst;
2844                 encap->mask |= FLOW_TCF_ENCAP_ETH_DST;
2845         }
2846         if (!mask || !memcmp(&mask->src,
2847                              &rte_flow_item_eth_mask.src,
2848                              sizeof(rte_flow_item_eth_mask.src))) {
2849                 /*
2850                  * Ethernet addresses are not supported by
2851                  * tc as tunnel_key parameters. Source ethernet
2852                  * address is ignored anyway.
2853                  */
2854                 encap->eth.src = spec->src;
2855                 encap->mask |= FLOW_TCF_ENCAP_ETH_SRC;
2856         }
2857 }
2858
2859 /**
2860  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV4 entry in configuration
2861  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV4 address fields
2862  * in the encapsulation parameters structure. The item must be prevalidated,
2863  * no any validation checks performed by function.
2864  *
2865  * @param[in] spec
2866  *   RTE_FLOW_ITEM_TYPE_IPV4 entry specification.
2867  * @param[out] encap
2868  *   Structure to fill the gathered IPV4 address data.
2869  */
2870 static void
2871 flow_tcf_parse_vxlan_encap_ipv4(const struct rte_flow_item_ipv4 *spec,
2872                                 struct flow_tcf_vxlan_encap *encap)
2873 {
2874         /* Item must be validated before. No redundant checks. */
2875         assert(spec);
2876         encap->ipv4.dst = spec->hdr.dst_addr;
2877         encap->ipv4.src = spec->hdr.src_addr;
2878         encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC |
2879                        FLOW_TCF_ENCAP_IPV4_DST;
2880 }
2881
2882 /**
2883  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV6 entry in configuration
2884  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV6 address fields
2885  * in the encapsulation parameters structure. The item must be prevalidated,
2886  * no any validation checks performed by function.
2887  *
2888  * @param[in] spec
2889  *   RTE_FLOW_ITEM_TYPE_IPV6 entry specification.
2890  * @param[out] encap
2891  *   Structure to fill the gathered IPV6 address data.
2892  */
2893 static void
2894 flow_tcf_parse_vxlan_encap_ipv6(const struct rte_flow_item_ipv6 *spec,
2895                                 struct flow_tcf_vxlan_encap *encap)
2896 {
2897         /* Item must be validated before. No redundant checks. */
2898         assert(spec);
2899         memcpy(encap->ipv6.dst, spec->hdr.dst_addr, IPV6_ADDR_LEN);
2900         memcpy(encap->ipv6.src, spec->hdr.src_addr, IPV6_ADDR_LEN);
2901         encap->mask |= FLOW_TCF_ENCAP_IPV6_SRC |
2902                        FLOW_TCF_ENCAP_IPV6_DST;
2903 }
2904
2905 /**
2906  * Helper function to process RTE_FLOW_ITEM_TYPE_UDP entry in configuration
2907  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the UDP port fields
2908  * in the encapsulation parameters structure. The item must be prevalidated,
2909  * no any validation checks performed by function.
2910  *
2911  * @param[in] spec
2912  *   RTE_FLOW_ITEM_TYPE_UDP entry specification.
2913  * @param[in] mask
2914  *   RTE_FLOW_ITEM_TYPE_UDP entry mask.
2915  * @param[out] encap
2916  *   Structure to fill the gathered UDP port data.
2917  */
2918 static void
2919 flow_tcf_parse_vxlan_encap_udp(const struct rte_flow_item_udp *spec,
2920                                const struct rte_flow_item_udp *mask,
2921                                struct flow_tcf_vxlan_encap *encap)
2922 {
2923         assert(spec);
2924         encap->udp.dst = spec->hdr.dst_port;
2925         encap->mask |= FLOW_TCF_ENCAP_UDP_DST;
2926         if (!mask || mask->hdr.src_port != RTE_BE16(0x0000)) {
2927                 encap->udp.src = spec->hdr.src_port;
2928                 encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC;
2929         }
2930 }
2931
2932 /**
2933  * Helper function to process RTE_FLOW_ITEM_TYPE_VXLAN entry in configuration
2934  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the VNI fields
2935  * in the encapsulation parameters structure. The item must be prevalidated,
2936  * no any validation checks performed by function.
2937  *
2938  * @param[in] spec
2939  *   RTE_FLOW_ITEM_TYPE_VXLAN entry specification.
2940  * @param[out] encap
2941  *   Structure to fill the gathered VNI address data.
2942  */
2943 static void
2944 flow_tcf_parse_vxlan_encap_vni(const struct rte_flow_item_vxlan *spec,
2945                                struct flow_tcf_vxlan_encap *encap)
2946 {
2947         /* Item must be validated before. Do not redundant checks. */
2948         assert(spec);
2949         memcpy(encap->vxlan.vni, spec->vni, sizeof(encap->vxlan.vni));
2950         encap->mask |= FLOW_TCF_ENCAP_VXLAN_VNI;
2951 }
2952
2953 /**
2954  * Populate consolidated encapsulation object from list of pattern items.
2955  *
2956  * Helper function to process configuration of action such as
2957  * RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. The item list should be
2958  * validated, there is no way to return an meaningful error.
2959  *
2960  * @param[in] action
2961  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2962  *   List of pattern items to gather data from.
2963  * @param[out] src
2964  *   Structure to fill gathered data.
2965  */
2966 static void
2967 flow_tcf_vxlan_encap_parse(const struct rte_flow_action *action,
2968                            struct flow_tcf_vxlan_encap *encap)
2969 {
2970         union {
2971                 const struct rte_flow_item_eth *eth;
2972                 const struct rte_flow_item_ipv4 *ipv4;
2973                 const struct rte_flow_item_ipv6 *ipv6;
2974                 const struct rte_flow_item_udp *udp;
2975                 const struct rte_flow_item_vxlan *vxlan;
2976         } spec, mask;
2977         const struct rte_flow_item *items;
2978
2979         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2980         assert(action->conf);
2981
2982         items = ((const struct rte_flow_action_vxlan_encap *)
2983                                         action->conf)->definition;
2984         assert(items);
2985         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2986                 switch (items->type) {
2987                 case RTE_FLOW_ITEM_TYPE_VOID:
2988                         break;
2989                 case RTE_FLOW_ITEM_TYPE_ETH:
2990                         mask.eth = items->mask;
2991                         spec.eth = items->spec;
2992                         flow_tcf_parse_vxlan_encap_eth(spec.eth, mask.eth,
2993                                                        encap);
2994                         break;
2995                 case RTE_FLOW_ITEM_TYPE_IPV4:
2996                         spec.ipv4 = items->spec;
2997                         flow_tcf_parse_vxlan_encap_ipv4(spec.ipv4, encap);
2998                         break;
2999                 case RTE_FLOW_ITEM_TYPE_IPV6:
3000                         spec.ipv6 = items->spec;
3001                         flow_tcf_parse_vxlan_encap_ipv6(spec.ipv6, encap);
3002                         break;
3003                 case RTE_FLOW_ITEM_TYPE_UDP:
3004                         mask.udp = items->mask;
3005                         spec.udp = items->spec;
3006                         flow_tcf_parse_vxlan_encap_udp(spec.udp, mask.udp,
3007                                                        encap);
3008                         break;
3009                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3010                         spec.vxlan = items->spec;
3011                         flow_tcf_parse_vxlan_encap_vni(spec.vxlan, encap);
3012                         break;
3013                 default:
3014                         assert(false);
3015                         DRV_LOG(WARNING,
3016                                 "unsupported item %p type %d,"
3017                                 " items must be validated"
3018                                 " before flow creation",
3019                                 (const void *)items, items->type);
3020                         encap->mask = 0;
3021                         return;
3022                 }
3023         }
3024 }
3025
3026 /**
3027  * Translate flow for Linux TC flower and construct Netlink message.
3028  *
3029  * @param[in] priv
3030  *   Pointer to the priv structure.
3031  * @param[in, out] flow
3032  *   Pointer to the sub flow.
3033  * @param[in] attr
3034  *   Pointer to the flow attributes.
3035  * @param[in] items
3036  *   Pointer to the list of items.
3037  * @param[in] actions
3038  *   Pointer to the list of actions.
3039  * @param[out] error
3040  *   Pointer to the error structure.
3041  *
3042  * @return
3043  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3044  */
3045 static int
3046 flow_tcf_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3047                    const struct rte_flow_attr *attr,
3048                    const struct rte_flow_item items[],
3049                    const struct rte_flow_action actions[],
3050                    struct rte_flow_error *error)
3051 {
3052         union {
3053                 const struct rte_flow_item_port_id *port_id;
3054                 const struct rte_flow_item_eth *eth;
3055                 const struct rte_flow_item_vlan *vlan;
3056                 const struct rte_flow_item_ipv4 *ipv4;
3057                 const struct rte_flow_item_ipv6 *ipv6;
3058                 const struct rte_flow_item_tcp *tcp;
3059                 const struct rte_flow_item_udp *udp;
3060                 const struct rte_flow_item_vxlan *vxlan;
3061         } spec, mask;
3062         union {
3063                 const struct rte_flow_action_port_id *port_id;
3064                 const struct rte_flow_action_jump *jump;
3065                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
3066                 const struct rte_flow_action_of_set_vlan_vid *
3067                         of_set_vlan_vid;
3068                 const struct rte_flow_action_of_set_vlan_pcp *
3069                         of_set_vlan_pcp;
3070         } conf;
3071         union {
3072                 struct flow_tcf_tunnel_hdr *hdr;
3073                 struct flow_tcf_vxlan_decap *vxlan;
3074         } decap = {
3075                 .hdr = NULL,
3076         };
3077         union {
3078                 struct flow_tcf_tunnel_hdr *hdr;
3079                 struct flow_tcf_vxlan_encap *vxlan;
3080         } encap = {
3081                 .hdr = NULL,
3082         };
3083         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
3084         struct nlmsghdr *nlh = dev_flow->tcf.nlh;
3085         struct tcmsg *tcm = dev_flow->tcf.tcm;
3086         uint32_t na_act_index_cur;
3087         bool eth_type_set = 0;
3088         bool vlan_present = 0;
3089         bool vlan_eth_type_set = 0;
3090         bool ip_proto_set = 0;
3091         struct nlattr *na_flower;
3092         struct nlattr *na_flower_act;
3093         struct nlattr *na_vlan_id = NULL;
3094         struct nlattr *na_vlan_priority = NULL;
3095         uint64_t item_flags = 0;
3096         int ret;
3097
3098         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
3099                                                 PTOI_TABLE_SZ_MAX(dev)));
3100         if (dev_flow->tcf.tunnel) {
3101                 switch (dev_flow->tcf.tunnel->type) {
3102                 case FLOW_TCF_TUNACT_VXLAN_DECAP:
3103                         decap.vxlan = dev_flow->tcf.vxlan_decap;
3104                         break;
3105                 case FLOW_TCF_TUNACT_VXLAN_ENCAP:
3106                         encap.vxlan = dev_flow->tcf.vxlan_encap;
3107                         break;
3108                 /* New tunnel actions can be added here. */
3109                 default:
3110                         assert(false);
3111                         break;
3112                 }
3113         }
3114         nlh = dev_flow->tcf.nlh;
3115         tcm = dev_flow->tcf.tcm;
3116         /* Prepare API must have been called beforehand. */
3117         assert(nlh != NULL && tcm != NULL);
3118         tcm->tcm_family = AF_UNSPEC;
3119         tcm->tcm_ifindex = ptoi[0].ifindex;
3120         tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
3121         /*
3122          * Priority cannot be zero to prevent the kernel from picking one
3123          * automatically.
3124          */
3125         tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
3126                                   RTE_BE16(ETH_P_ALL));
3127         if (attr->group > 0)
3128                 mnl_attr_put_u32(nlh, TCA_CHAIN, attr->group);
3129         mnl_attr_put_strz(nlh, TCA_KIND, "flower");
3130         na_flower = mnl_attr_nest_start(nlh, TCA_OPTIONS);
3131         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3132                 unsigned int i;
3133
3134                 switch (items->type) {
3135                 case RTE_FLOW_ITEM_TYPE_VOID:
3136                         break;
3137                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3138                         mask.port_id = flow_tcf_item_mask
3139                                 (items, &rte_flow_item_port_id_mask,
3140                                  &flow_tcf_mask_supported.port_id,
3141                                  &flow_tcf_mask_empty.port_id,
3142                                  sizeof(flow_tcf_mask_supported.port_id),
3143                                  error);
3144                         assert(mask.port_id);
3145                         if (mask.port_id == &flow_tcf_mask_empty.port_id)
3146                                 break;
3147                         spec.port_id = items->spec;
3148                         if (!mask.port_id->id)
3149                                 i = 0;
3150                         else
3151                                 for (i = 0; ptoi[i].ifindex; ++i)
3152                                         if (ptoi[i].port_id == spec.port_id->id)
3153                                                 break;
3154                         assert(ptoi[i].ifindex);
3155                         tcm->tcm_ifindex = ptoi[i].ifindex;
3156                         break;
3157                 case RTE_FLOW_ITEM_TYPE_ETH:
3158                         item_flags |= (item_flags & MLX5_FLOW_LAYER_VXLAN) ?
3159                                       MLX5_FLOW_LAYER_INNER_L2 :
3160                                       MLX5_FLOW_LAYER_OUTER_L2;
3161                         mask.eth = flow_tcf_item_mask
3162                                 (items, &rte_flow_item_eth_mask,
3163                                  &flow_tcf_mask_supported.eth,
3164                                  &flow_tcf_mask_empty.eth,
3165                                  sizeof(flow_tcf_mask_supported.eth),
3166                                  error);
3167                         assert(mask.eth);
3168                         if (mask.eth == &flow_tcf_mask_empty.eth)
3169                                 break;
3170                         spec.eth = items->spec;
3171                         if (decap.vxlan &&
3172                             !(item_flags & MLX5_FLOW_LAYER_VXLAN)) {
3173                                 DRV_LOG(WARNING,
3174                                         "outer L2 addresses cannot be forced"
3175                                         " for vxlan decapsulation, parameter"
3176                                         " ignored");
3177                                 break;
3178                         }
3179                         if (mask.eth->type) {
3180                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3181                                                  spec.eth->type);
3182                                 eth_type_set = 1;
3183                         }
3184                         if (!is_zero_ether_addr(&mask.eth->dst)) {
3185                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST,
3186                                              ETHER_ADDR_LEN,
3187                                              spec.eth->dst.addr_bytes);
3188                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST_MASK,
3189                                              ETHER_ADDR_LEN,
3190                                              mask.eth->dst.addr_bytes);
3191                         }
3192                         if (!is_zero_ether_addr(&mask.eth->src)) {
3193                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC,
3194                                              ETHER_ADDR_LEN,
3195                                              spec.eth->src.addr_bytes);
3196                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC_MASK,
3197                                              ETHER_ADDR_LEN,
3198                                              mask.eth->src.addr_bytes);
3199                         }
3200                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3201                         break;
3202                 case RTE_FLOW_ITEM_TYPE_VLAN:
3203                         assert(!encap.hdr);
3204                         assert(!decap.hdr);
3205                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
3206                         mask.vlan = flow_tcf_item_mask
3207                                 (items, &rte_flow_item_vlan_mask,
3208                                  &flow_tcf_mask_supported.vlan,
3209                                  &flow_tcf_mask_empty.vlan,
3210                                  sizeof(flow_tcf_mask_supported.vlan),
3211                                  error);
3212                         assert(mask.vlan);
3213                         if (!eth_type_set)
3214                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3215                                                  RTE_BE16(ETH_P_8021Q));
3216                         eth_type_set = 1;
3217                         vlan_present = 1;
3218                         if (mask.vlan == &flow_tcf_mask_empty.vlan)
3219                                 break;
3220                         spec.vlan = items->spec;
3221                         if (mask.vlan->inner_type) {
3222                                 mnl_attr_put_u16(nlh,
3223                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE,
3224                                                  spec.vlan->inner_type);
3225                                 vlan_eth_type_set = 1;
3226                         }
3227                         if (mask.vlan->tci & RTE_BE16(0xe000))
3228                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_VLAN_PRIO,
3229                                                 (rte_be_to_cpu_16
3230                                                  (spec.vlan->tci) >> 13) & 0x7);
3231                         if (mask.vlan->tci & RTE_BE16(0x0fff))
3232                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_VLAN_ID,
3233                                                  rte_be_to_cpu_16
3234                                                  (spec.vlan->tci &
3235                                                   RTE_BE16(0x0fff)));
3236                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3237                         break;
3238                 case RTE_FLOW_ITEM_TYPE_IPV4:
3239                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3240                         mask.ipv4 = flow_tcf_item_mask
3241                                 (items, &rte_flow_item_ipv4_mask,
3242                                  &flow_tcf_mask_supported.ipv4,
3243                                  &flow_tcf_mask_empty.ipv4,
3244                                  sizeof(flow_tcf_mask_supported.ipv4),
3245                                  error);
3246                         assert(mask.ipv4);
3247                         spec.ipv4 = items->spec;
3248                         if (!decap.vxlan) {
3249                                 if (!eth_type_set ||
3250                                     (!vlan_eth_type_set && vlan_present))
3251                                         mnl_attr_put_u16
3252                                                 (nlh,
3253                                                  vlan_present ?
3254                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3255                                                  TCA_FLOWER_KEY_ETH_TYPE,
3256                                                  RTE_BE16(ETH_P_IP));
3257                                 eth_type_set = 1;
3258                                 vlan_eth_type_set = 1;
3259                                 if (mask.ipv4 == &flow_tcf_mask_empty.ipv4)
3260                                         break;
3261                                 if (mask.ipv4->hdr.next_proto_id) {
3262                                         mnl_attr_put_u8
3263                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3264                                                  spec.ipv4->hdr.next_proto_id);
3265                                         ip_proto_set = 1;
3266                                 }
3267                         } else {
3268                                 assert(mask.ipv4 != &flow_tcf_mask_empty.ipv4);
3269                         }
3270                         if (mask.ipv4->hdr.src_addr) {
3271                                 mnl_attr_put_u32
3272                                         (nlh, decap.vxlan ?
3273                                          TCA_FLOWER_KEY_ENC_IPV4_SRC :
3274                                          TCA_FLOWER_KEY_IPV4_SRC,
3275                                          spec.ipv4->hdr.src_addr);
3276                                 mnl_attr_put_u32
3277                                         (nlh, decap.vxlan ?
3278                                          TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK :
3279                                          TCA_FLOWER_KEY_IPV4_SRC_MASK,
3280                                          mask.ipv4->hdr.src_addr);
3281                         }
3282                         if (mask.ipv4->hdr.dst_addr) {
3283                                 mnl_attr_put_u32
3284                                         (nlh, decap.vxlan ?
3285                                          TCA_FLOWER_KEY_ENC_IPV4_DST :
3286                                          TCA_FLOWER_KEY_IPV4_DST,
3287                                          spec.ipv4->hdr.dst_addr);
3288                                 mnl_attr_put_u32
3289                                         (nlh, decap.vxlan ?
3290                                          TCA_FLOWER_KEY_ENC_IPV4_DST_MASK :
3291                                          TCA_FLOWER_KEY_IPV4_DST_MASK,
3292                                          mask.ipv4->hdr.dst_addr);
3293                         }
3294                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3295                         break;
3296                 case RTE_FLOW_ITEM_TYPE_IPV6:
3297                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3298                         mask.ipv6 = flow_tcf_item_mask
3299                                 (items, &rte_flow_item_ipv6_mask,
3300                                  &flow_tcf_mask_supported.ipv6,
3301                                  &flow_tcf_mask_empty.ipv6,
3302                                  sizeof(flow_tcf_mask_supported.ipv6),
3303                                  error);
3304                         assert(mask.ipv6);
3305                         spec.ipv6 = items->spec;
3306                         if (!decap.vxlan) {
3307                                 if (!eth_type_set ||
3308                                     (!vlan_eth_type_set && vlan_present))
3309                                         mnl_attr_put_u16
3310                                                 (nlh,
3311                                                  vlan_present ?
3312                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3313                                                  TCA_FLOWER_KEY_ETH_TYPE,
3314                                                  RTE_BE16(ETH_P_IPV6));
3315                                 eth_type_set = 1;
3316                                 vlan_eth_type_set = 1;
3317                                 if (mask.ipv6 == &flow_tcf_mask_empty.ipv6)
3318                                         break;
3319                                 if (mask.ipv6->hdr.proto) {
3320                                         mnl_attr_put_u8
3321                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3322                                                  spec.ipv6->hdr.proto);
3323                                         ip_proto_set = 1;
3324                                 }
3325                         } else {
3326                                 assert(mask.ipv6 != &flow_tcf_mask_empty.ipv6);
3327                         }
3328                         if (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.src_addr)) {
3329                                 mnl_attr_put(nlh, decap.vxlan ?
3330                                              TCA_FLOWER_KEY_ENC_IPV6_SRC :
3331                                              TCA_FLOWER_KEY_IPV6_SRC,
3332                                              IPV6_ADDR_LEN,
3333                                              spec.ipv6->hdr.src_addr);
3334                                 mnl_attr_put(nlh, decap.vxlan ?
3335                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK :
3336                                              TCA_FLOWER_KEY_IPV6_SRC_MASK,
3337                                              IPV6_ADDR_LEN,
3338                                              mask.ipv6->hdr.src_addr);
3339                         }
3340                         if (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.dst_addr)) {
3341                                 mnl_attr_put(nlh, decap.vxlan ?
3342                                              TCA_FLOWER_KEY_ENC_IPV6_DST :
3343                                              TCA_FLOWER_KEY_IPV6_DST,
3344                                              IPV6_ADDR_LEN,
3345                                              spec.ipv6->hdr.dst_addr);
3346                                 mnl_attr_put(nlh, decap.vxlan ?
3347                                              TCA_FLOWER_KEY_ENC_IPV6_DST_MASK :
3348                                              TCA_FLOWER_KEY_IPV6_DST_MASK,
3349                                              IPV6_ADDR_LEN,
3350                                              mask.ipv6->hdr.dst_addr);
3351                         }
3352                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3353                         break;
3354                 case RTE_FLOW_ITEM_TYPE_UDP:
3355                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3356                         mask.udp = flow_tcf_item_mask
3357                                 (items, &rte_flow_item_udp_mask,
3358                                  &flow_tcf_mask_supported.udp,
3359                                  &flow_tcf_mask_empty.udp,
3360                                  sizeof(flow_tcf_mask_supported.udp),
3361                                  error);
3362                         assert(mask.udp);
3363                         spec.udp = items->spec;
3364                         if (!decap.vxlan) {
3365                                 if (!ip_proto_set)
3366                                         mnl_attr_put_u8
3367                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3368                                                 IPPROTO_UDP);
3369                                 if (mask.udp == &flow_tcf_mask_empty.udp)
3370                                         break;
3371                         } else {
3372                                 assert(mask.udp != &flow_tcf_mask_empty.udp);
3373                                 decap.vxlan->udp_port =
3374                                         rte_be_to_cpu_16
3375                                                 (spec.udp->hdr.dst_port);
3376                         }
3377                         if (mask.udp->hdr.src_port) {
3378                                 mnl_attr_put_u16
3379                                         (nlh, decap.vxlan ?
3380                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT :
3381                                          TCA_FLOWER_KEY_UDP_SRC,
3382                                          spec.udp->hdr.src_port);
3383                                 mnl_attr_put_u16
3384                                         (nlh, decap.vxlan ?
3385                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK :
3386                                          TCA_FLOWER_KEY_UDP_SRC_MASK,
3387                                          mask.udp->hdr.src_port);
3388                         }
3389                         if (mask.udp->hdr.dst_port) {
3390                                 mnl_attr_put_u16
3391                                         (nlh, decap.vxlan ?
3392                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT :
3393                                          TCA_FLOWER_KEY_UDP_DST,
3394                                          spec.udp->hdr.dst_port);
3395                                 mnl_attr_put_u16
3396                                         (nlh, decap.vxlan ?
3397                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK :
3398                                          TCA_FLOWER_KEY_UDP_DST_MASK,
3399                                          mask.udp->hdr.dst_port);
3400                         }
3401                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3402                         break;
3403                 case RTE_FLOW_ITEM_TYPE_TCP:
3404                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3405                         mask.tcp = flow_tcf_item_mask
3406                                 (items, &rte_flow_item_tcp_mask,
3407                                  &flow_tcf_mask_supported.tcp,
3408                                  &flow_tcf_mask_empty.tcp,
3409                                  sizeof(flow_tcf_mask_supported.tcp),
3410                                  error);
3411                         assert(mask.tcp);
3412                         if (!ip_proto_set)
3413                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3414                                                 IPPROTO_TCP);
3415                         if (mask.tcp == &flow_tcf_mask_empty.tcp)
3416                                 break;
3417                         spec.tcp = items->spec;
3418                         if (mask.tcp->hdr.src_port) {
3419                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_SRC,
3420                                                  spec.tcp->hdr.src_port);
3421                                 mnl_attr_put_u16(nlh,
3422                                                  TCA_FLOWER_KEY_TCP_SRC_MASK,
3423                                                  mask.tcp->hdr.src_port);
3424                         }
3425                         if (mask.tcp->hdr.dst_port) {
3426                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_DST,
3427                                                  spec.tcp->hdr.dst_port);
3428                                 mnl_attr_put_u16(nlh,
3429                                                  TCA_FLOWER_KEY_TCP_DST_MASK,
3430                                                  mask.tcp->hdr.dst_port);
3431                         }
3432                         if (mask.tcp->hdr.tcp_flags) {
3433                                 mnl_attr_put_u16
3434                                         (nlh,
3435                                          TCA_FLOWER_KEY_TCP_FLAGS,
3436                                          rte_cpu_to_be_16
3437                                                 (spec.tcp->hdr.tcp_flags));
3438                                 mnl_attr_put_u16
3439                                         (nlh,
3440                                          TCA_FLOWER_KEY_TCP_FLAGS_MASK,
3441                                          rte_cpu_to_be_16
3442                                                 (mask.tcp->hdr.tcp_flags));
3443                         }
3444                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3445                         break;
3446                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3447                         assert(decap.vxlan);
3448                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
3449                         spec.vxlan = items->spec;
3450                         mnl_attr_put_u32(nlh,
3451                                          TCA_FLOWER_KEY_ENC_KEY_ID,
3452                                          vxlan_vni_as_be32(spec.vxlan->vni));
3453                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3454                         break;
3455                 default:
3456                         return rte_flow_error_set(error, ENOTSUP,
3457                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3458                                                   NULL, "item not supported");
3459                 }
3460         }
3461         na_flower_act = mnl_attr_nest_start(nlh, TCA_FLOWER_ACT);
3462         na_act_index_cur = 1;
3463         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3464                 struct nlattr *na_act_index;
3465                 struct nlattr *na_act;
3466                 unsigned int vlan_act;
3467                 unsigned int i;
3468
3469                 switch (actions->type) {
3470                 case RTE_FLOW_ACTION_TYPE_VOID:
3471                         break;
3472                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3473                         conf.port_id = actions->conf;
3474                         if (conf.port_id->original)
3475                                 i = 0;
3476                         else
3477                                 for (i = 0; ptoi[i].ifindex; ++i)
3478                                         if (ptoi[i].port_id == conf.port_id->id)
3479                                                 break;
3480                         assert(ptoi[i].ifindex);
3481                         na_act_index =
3482                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3483                         assert(na_act_index);
3484                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "mirred");
3485                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3486                         assert(na_act);
3487                         if (encap.hdr) {
3488                                 assert(dev_flow->tcf.tunnel);
3489                                 dev_flow->tcf.tunnel->ifindex_ptr =
3490                                         &((struct tc_mirred *)
3491                                         mnl_attr_get_payload
3492                                         (mnl_nlmsg_get_payload_tail
3493                                                 (nlh)))->ifindex;
3494                         }
3495                         mnl_attr_put(nlh, TCA_MIRRED_PARMS,
3496                                      sizeof(struct tc_mirred),
3497                                      &(struct tc_mirred){
3498                                         .action = TC_ACT_STOLEN,
3499                                         .eaction = TCA_EGRESS_REDIR,
3500                                         .ifindex = ptoi[i].ifindex,
3501                                      });
3502                         mnl_attr_nest_end(nlh, na_act);
3503                         mnl_attr_nest_end(nlh, na_act_index);
3504                         break;
3505                 case RTE_FLOW_ACTION_TYPE_JUMP:
3506                         conf.jump = actions->conf;
3507                         na_act_index =
3508                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3509                         assert(na_act_index);
3510                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3511                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3512                         assert(na_act);
3513                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3514                                      sizeof(struct tc_gact),
3515                                      &(struct tc_gact){
3516                                         .action = TC_ACT_GOTO_CHAIN |
3517                                                   conf.jump->group,
3518                                      });
3519                         mnl_attr_nest_end(nlh, na_act);
3520                         mnl_attr_nest_end(nlh, na_act_index);
3521                         break;
3522                 case RTE_FLOW_ACTION_TYPE_DROP:
3523                         na_act_index =
3524                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3525                         assert(na_act_index);
3526                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3527                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3528                         assert(na_act);
3529                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3530                                      sizeof(struct tc_gact),
3531                                      &(struct tc_gact){
3532                                         .action = TC_ACT_SHOT,
3533                                      });
3534                         mnl_attr_nest_end(nlh, na_act);
3535                         mnl_attr_nest_end(nlh, na_act_index);
3536                         break;
3537                 case RTE_FLOW_ACTION_TYPE_COUNT:
3538                         /*
3539                          * Driver adds the count action implicitly for
3540                          * each rule it creates.
3541                          */
3542                         ret = flow_tcf_translate_action_count(dev,
3543                                                               dev_flow, error);
3544                         if (ret < 0)
3545                                 return ret;
3546                         break;
3547                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3548                         conf.of_push_vlan = NULL;
3549                         vlan_act = TCA_VLAN_ACT_POP;
3550                         goto action_of_vlan;
3551                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3552                         conf.of_push_vlan = actions->conf;
3553                         vlan_act = TCA_VLAN_ACT_PUSH;
3554                         goto action_of_vlan;
3555                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3556                         conf.of_set_vlan_vid = actions->conf;
3557                         if (na_vlan_id)
3558                                 goto override_na_vlan_id;
3559                         vlan_act = TCA_VLAN_ACT_MODIFY;
3560                         goto action_of_vlan;
3561                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3562                         conf.of_set_vlan_pcp = actions->conf;
3563                         if (na_vlan_priority)
3564                                 goto override_na_vlan_priority;
3565                         vlan_act = TCA_VLAN_ACT_MODIFY;
3566                         goto action_of_vlan;
3567 action_of_vlan:
3568                         na_act_index =
3569                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3570                         assert(na_act_index);
3571                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "vlan");
3572                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3573                         assert(na_act);
3574                         mnl_attr_put(nlh, TCA_VLAN_PARMS,
3575                                      sizeof(struct tc_vlan),
3576                                      &(struct tc_vlan){
3577                                         .action = TC_ACT_PIPE,
3578                                         .v_action = vlan_act,
3579                                      });
3580                         if (vlan_act == TCA_VLAN_ACT_POP) {
3581                                 mnl_attr_nest_end(nlh, na_act);
3582                                 mnl_attr_nest_end(nlh, na_act_index);
3583                                 break;
3584                         }
3585                         if (vlan_act == TCA_VLAN_ACT_PUSH)
3586                                 mnl_attr_put_u16(nlh,
3587                                                  TCA_VLAN_PUSH_VLAN_PROTOCOL,
3588                                                  conf.of_push_vlan->ethertype);
3589                         na_vlan_id = mnl_nlmsg_get_payload_tail(nlh);
3590                         mnl_attr_put_u16(nlh, TCA_VLAN_PAD, 0);
3591                         na_vlan_priority = mnl_nlmsg_get_payload_tail(nlh);
3592                         mnl_attr_put_u8(nlh, TCA_VLAN_PAD, 0);
3593                         mnl_attr_nest_end(nlh, na_act);
3594                         mnl_attr_nest_end(nlh, na_act_index);
3595                         if (actions->type ==
3596                             RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
3597 override_na_vlan_id:
3598                                 na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
3599                                 *(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
3600                                         rte_be_to_cpu_16
3601                                         (conf.of_set_vlan_vid->vlan_vid);
3602                         } else if (actions->type ==
3603                                    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
3604 override_na_vlan_priority:
3605                                 na_vlan_priority->nla_type =
3606                                         TCA_VLAN_PUSH_VLAN_PRIORITY;
3607                                 *(uint8_t *)mnl_attr_get_payload
3608                                         (na_vlan_priority) =
3609                                         conf.of_set_vlan_pcp->vlan_pcp;
3610                         }
3611                         break;
3612                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3613                         assert(decap.vxlan);
3614                         assert(dev_flow->tcf.tunnel);
3615                         dev_flow->tcf.tunnel->ifindex_ptr =
3616                                 (unsigned int *)&tcm->tcm_ifindex;
3617                         na_act_index =
3618                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3619                         assert(na_act_index);
3620                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3621                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3622                         assert(na_act);
3623                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3624                                 sizeof(struct tc_tunnel_key),
3625                                 &(struct tc_tunnel_key){
3626                                         .action = TC_ACT_PIPE,
3627                                         .t_action = TCA_TUNNEL_KEY_ACT_RELEASE,
3628                                         });
3629                         mnl_attr_nest_end(nlh, na_act);
3630                         mnl_attr_nest_end(nlh, na_act_index);
3631                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3632                         break;
3633                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3634                         assert(encap.vxlan);
3635                         flow_tcf_vxlan_encap_parse(actions, encap.vxlan);
3636                         na_act_index =
3637                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3638                         assert(na_act_index);
3639                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3640                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3641                         assert(na_act);
3642                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3643                                 sizeof(struct tc_tunnel_key),
3644                                 &(struct tc_tunnel_key){
3645                                         .action = TC_ACT_PIPE,
3646                                         .t_action = TCA_TUNNEL_KEY_ACT_SET,
3647                                         });
3648                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_UDP_DST)
3649                                 mnl_attr_put_u16(nlh,
3650                                          TCA_TUNNEL_KEY_ENC_DST_PORT,
3651                                          encap.vxlan->udp.dst);
3652                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_SRC)
3653                                 mnl_attr_put_u32(nlh,
3654                                          TCA_TUNNEL_KEY_ENC_IPV4_SRC,
3655                                          encap.vxlan->ipv4.src);
3656                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_DST)
3657                                 mnl_attr_put_u32(nlh,
3658                                          TCA_TUNNEL_KEY_ENC_IPV4_DST,
3659                                          encap.vxlan->ipv4.dst);
3660                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_SRC)
3661                                 mnl_attr_put(nlh,
3662                                          TCA_TUNNEL_KEY_ENC_IPV6_SRC,
3663                                          sizeof(encap.vxlan->ipv6.src),
3664                                          &encap.vxlan->ipv6.src);
3665                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_DST)
3666                                 mnl_attr_put(nlh,
3667                                          TCA_TUNNEL_KEY_ENC_IPV6_DST,
3668                                          sizeof(encap.vxlan->ipv6.dst),
3669                                          &encap.vxlan->ipv6.dst);
3670                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_VXLAN_VNI)
3671                                 mnl_attr_put_u32(nlh,
3672                                          TCA_TUNNEL_KEY_ENC_KEY_ID,
3673                                          vxlan_vni_as_be32
3674                                                 (encap.vxlan->vxlan.vni));
3675                         mnl_attr_put_u8(nlh, TCA_TUNNEL_KEY_NO_CSUM, 0);
3676                         mnl_attr_nest_end(nlh, na_act);
3677                         mnl_attr_nest_end(nlh, na_act_index);
3678                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3679                         break;
3680                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3681                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3682                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3683                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3684                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3685                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3686                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3687                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3688                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3689                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3690                         na_act_index =
3691                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3692                         flow_tcf_create_pedit_mnl_msg(nlh,
3693                                                       &actions, item_flags);
3694                         mnl_attr_nest_end(nlh, na_act_index);
3695                         break;
3696                 default:
3697                         return rte_flow_error_set(error, ENOTSUP,
3698                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3699                                                   actions,
3700                                                   "action not supported");
3701                 }
3702         }
3703         assert(na_flower);
3704         assert(na_flower_act);
3705         mnl_attr_nest_end(nlh, na_flower_act);
3706         dev_flow->tcf.ptc_flags = mnl_attr_get_payload
3707                                         (mnl_nlmsg_get_payload_tail(nlh));
3708         mnl_attr_put_u32(nlh, TCA_FLOWER_FLAGS, decap.vxlan ?
3709                                                 0 : TCA_CLS_FLAGS_SKIP_SW);
3710         mnl_attr_nest_end(nlh, na_flower);
3711         if (dev_flow->tcf.tunnel && dev_flow->tcf.tunnel->ifindex_ptr)
3712                 dev_flow->tcf.tunnel->ifindex_org =
3713                         *dev_flow->tcf.tunnel->ifindex_ptr;
3714         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3715         return 0;
3716 }
3717
3718 /**
3719  * Send Netlink message with acknowledgment.
3720  *
3721  * @param tcf
3722  *   Flow context to use.
3723  * @param nlh
3724  *   Message to send. This function always raises the NLM_F_ACK flag before
3725  *   sending.
3726  * @param[in] cb
3727  *   Callback handler for received message.
3728  * @param[in] arg
3729  *   Context pointer for callback handler.
3730  *
3731  * @return
3732  *   0 on success, a negative errno value otherwise and rte_errno is set.
3733  */
3734 static int
3735 flow_tcf_nl_ack(struct mlx5_flow_tcf_context *tcf,
3736                 struct nlmsghdr *nlh,
3737                 mnl_cb_t cb, void *arg)
3738 {
3739         unsigned int portid = mnl_socket_get_portid(tcf->nl);
3740         uint32_t seq = tcf->seq++;
3741         int ret, err = 0;
3742
3743         assert(tcf->nl);
3744         assert(tcf->buf);
3745         if (!seq) {
3746                 /* seq 0 is reserved for kernel event-driven notifications. */
3747                 seq = tcf->seq++;
3748         }
3749         nlh->nlmsg_seq = seq;
3750         nlh->nlmsg_flags |= NLM_F_ACK;
3751         ret = mnl_socket_sendto(tcf->nl, nlh, nlh->nlmsg_len);
3752         if (ret <= 0) {
3753                 /* Message send error occurres. */
3754                 rte_errno = errno;
3755                 return -rte_errno;
3756         }
3757         nlh = (struct nlmsghdr *)(tcf->buf);
3758         /*
3759          * The following loop postpones non-fatal errors until multipart
3760          * messages are complete.
3761          */
3762         while (true) {
3763                 ret = mnl_socket_recvfrom(tcf->nl, tcf->buf, tcf->buf_size);
3764                 if (ret < 0) {
3765                         err = errno;
3766                         /*
3767                          * In case of overflow Will receive till
3768                          * end of multipart message. We may lost part
3769                          * of reply messages but mark and return an error.
3770                          */
3771                         if (err != ENOSPC ||
3772                             !(nlh->nlmsg_flags & NLM_F_MULTI) ||
3773                             nlh->nlmsg_type == NLMSG_DONE)
3774                                 break;
3775                 } else {
3776                         ret = mnl_cb_run(nlh, ret, seq, portid, cb, arg);
3777                         if (!ret) {
3778                                 /*
3779                                  * libmnl returns 0 if DONE or
3780                                  * success ACK message found.
3781                                  */
3782                                 break;
3783                         }
3784                         if (ret < 0) {
3785                                 /*
3786                                  * ACK message with error found
3787                                  * or some error occurred.
3788                                  */
3789                                 err = errno;
3790                                 break;
3791                         }
3792                         /* We should continue receiving. */
3793                 }
3794         }
3795         if (!err)
3796                 return 0;
3797         rte_errno = err;
3798         return -err;
3799 }
3800
3801 #define MNL_BUF_EXTRA_SPACE 16
3802 #define MNL_REQUEST_SIZE_MIN 256
3803 #define MNL_REQUEST_SIZE_MAX 2048
3804 #define MNL_REQUEST_SIZE RTE_MIN(RTE_MAX(sysconf(_SC_PAGESIZE), \
3805                                  MNL_REQUEST_SIZE_MIN), MNL_REQUEST_SIZE_MAX)
3806
3807 /* Data structures used by flow_tcf_xxx_cb() routines. */
3808 struct tcf_nlcb_buf {
3809         LIST_ENTRY(tcf_nlcb_buf) next;
3810         uint32_t size;
3811         alignas(struct nlmsghdr)
3812         uint8_t msg[]; /**< Netlink message data. */
3813 };
3814
3815 struct tcf_nlcb_context {
3816         unsigned int ifindex; /**< Base interface index. */
3817         uint32_t bufsize;
3818         LIST_HEAD(, tcf_nlcb_buf) nlbuf;
3819 };
3820
3821 /**
3822  * Allocate space for netlink command in buffer list
3823  *
3824  * @param[in, out] ctx
3825  *   Pointer to callback context with command buffers list.
3826  * @param[in] size
3827  *   Required size of data buffer to be allocated.
3828  *
3829  * @return
3830  *   Pointer to allocated memory, aligned as message header.
3831  *   NULL if some error occurred.
3832  */
3833 static struct nlmsghdr *
3834 flow_tcf_alloc_nlcmd(struct tcf_nlcb_context *ctx, uint32_t size)
3835 {
3836         struct tcf_nlcb_buf *buf;
3837         struct nlmsghdr *nlh;
3838
3839         size = NLMSG_ALIGN(size);
3840         buf = LIST_FIRST(&ctx->nlbuf);
3841         if (buf && (buf->size + size) <= ctx->bufsize) {
3842                 nlh = (struct nlmsghdr *)&buf->msg[buf->size];
3843                 buf->size += size;
3844                 return nlh;
3845         }
3846         if (size > ctx->bufsize) {
3847                 DRV_LOG(WARNING, "netlink: too long command buffer requested");
3848                 return NULL;
3849         }
3850         buf = rte_malloc(__func__,
3851                         ctx->bufsize + sizeof(struct tcf_nlcb_buf),
3852                         alignof(struct tcf_nlcb_buf));
3853         if (!buf) {
3854                 DRV_LOG(WARNING, "netlink: no memory for command buffer");
3855                 return NULL;
3856         }
3857         LIST_INSERT_HEAD(&ctx->nlbuf, buf, next);
3858         buf->size = size;
3859         nlh = (struct nlmsghdr *)&buf->msg[0];
3860         return nlh;
3861 }
3862
3863 /**
3864  * Send the buffers with prepared netlink commands. Scans the list and
3865  * sends all found buffers. Buffers are sent and freed anyway in order
3866  * to prevent memory leakage if some every message in received packet.
3867  *
3868  * @param[in] tcf
3869  *   Context object initialized by mlx5_flow_tcf_context_create().
3870  * @param[in, out] ctx
3871  *   Pointer to callback context with command buffers list.
3872  *
3873  * @return
3874  *   Zero value on success, negative errno value otherwise
3875  *   and rte_errno is set.
3876  */
3877 static int
3878 flow_tcf_send_nlcmd(struct mlx5_flow_tcf_context *tcf,
3879                     struct tcf_nlcb_context *ctx)
3880 {
3881         struct tcf_nlcb_buf *bc = LIST_FIRST(&ctx->nlbuf);
3882         int ret = 0;
3883
3884         while (bc) {
3885                 struct tcf_nlcb_buf *bn = LIST_NEXT(bc, next);
3886                 struct nlmsghdr *nlh;
3887                 uint32_t msg = 0;
3888                 int rc;
3889
3890                 while (msg < bc->size) {
3891                         /*
3892                          * Send Netlink commands from buffer in one by one
3893                          * fashion. If we send multiple rule deletion commands
3894                          * in one Netlink message and some error occurs it may
3895                          * cause multiple ACK error messages and break sequence
3896                          * numbers of Netlink communication, because we expect
3897                          * the only one ACK reply.
3898                          */
3899                         assert((bc->size - msg) >= sizeof(struct nlmsghdr));
3900                         nlh = (struct nlmsghdr *)&bc->msg[msg];
3901                         assert((bc->size - msg) >= nlh->nlmsg_len);
3902                         msg += nlh->nlmsg_len;
3903                         rc = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
3904                         if (rc) {
3905                                 DRV_LOG(WARNING,
3906                                         "netlink: cleanup error %d", rc);
3907                                 if (!ret)
3908                                         ret = rc;
3909                         }
3910                 }
3911                 rte_free(bc);
3912                 bc = bn;
3913         }
3914         LIST_INIT(&ctx->nlbuf);
3915         return ret;
3916 }
3917
3918 /**
3919  * Collect local IP address rules with scope link attribute  on specified
3920  * network device. This is callback routine called by libmnl mnl_cb_run()
3921  * in loop for every message in received packet.
3922  *
3923  * @param[in] nlh
3924  *   Pointer to reply header.
3925  * @param[in, out] arg
3926  *   Opaque data pointer for this callback.
3927  *
3928  * @return
3929  *   A positive, nonzero value on success, negative errno value otherwise
3930  *   and rte_errno is set.
3931  */
3932 static int
3933 flow_tcf_collect_local_cb(const struct nlmsghdr *nlh, void *arg)
3934 {
3935         struct tcf_nlcb_context *ctx = arg;
3936         struct nlmsghdr *cmd;
3937         struct ifaddrmsg *ifa;
3938         struct nlattr *na;
3939         struct nlattr *na_local = NULL;
3940         struct nlattr *na_peer = NULL;
3941         unsigned char family;
3942         uint32_t size;
3943
3944         if (nlh->nlmsg_type != RTM_NEWADDR) {
3945                 rte_errno = EINVAL;
3946                 return -rte_errno;
3947         }
3948         ifa = mnl_nlmsg_get_payload(nlh);
3949         family = ifa->ifa_family;
3950         if (ifa->ifa_index != ctx->ifindex ||
3951             ifa->ifa_scope != RT_SCOPE_LINK ||
3952             !(ifa->ifa_flags & IFA_F_PERMANENT) ||
3953             (family != AF_INET && family != AF_INET6))
3954                 return 1;
3955         mnl_attr_for_each(na, nlh, sizeof(*ifa)) {
3956                 switch (mnl_attr_get_type(na)) {
3957                 case IFA_LOCAL:
3958                         na_local = na;
3959                         break;
3960                 case IFA_ADDRESS:
3961                         na_peer = na;
3962                         break;
3963                 }
3964                 if (na_local && na_peer)
3965                         break;
3966         }
3967         if (!na_local || !na_peer)
3968                 return 1;
3969         /* Local rule found with scope link, permanent and assigned peer. */
3970         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
3971                MNL_ALIGN(sizeof(struct ifaddrmsg)) +
3972                (family == AF_INET6 ? 2 * SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
3973                                    : 2 * SZ_NLATTR_TYPE_OF(uint32_t));
3974         cmd = flow_tcf_alloc_nlcmd(ctx, size);
3975         if (!cmd) {
3976                 rte_errno = ENOMEM;
3977                 return -rte_errno;
3978         }
3979         cmd = mnl_nlmsg_put_header(cmd);
3980         cmd->nlmsg_type = RTM_DELADDR;
3981         cmd->nlmsg_flags = NLM_F_REQUEST;
3982         ifa = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifa));
3983         ifa->ifa_flags = IFA_F_PERMANENT;
3984         ifa->ifa_scope = RT_SCOPE_LINK;
3985         ifa->ifa_index = ctx->ifindex;
3986         if (family == AF_INET) {
3987                 ifa->ifa_family = AF_INET;
3988                 ifa->ifa_prefixlen = 32;
3989                 mnl_attr_put_u32(cmd, IFA_LOCAL, mnl_attr_get_u32(na_local));
3990                 mnl_attr_put_u32(cmd, IFA_ADDRESS, mnl_attr_get_u32(na_peer));
3991         } else {
3992                 ifa->ifa_family = AF_INET6;
3993                 ifa->ifa_prefixlen = 128;
3994                 mnl_attr_put(cmd, IFA_LOCAL, IPV6_ADDR_LEN,
3995                         mnl_attr_get_payload(na_local));
3996                 mnl_attr_put(cmd, IFA_ADDRESS, IPV6_ADDR_LEN,
3997                         mnl_attr_get_payload(na_peer));
3998         }
3999         assert(size == cmd->nlmsg_len);
4000         return 1;
4001 }
4002
4003 /**
4004  * Cleanup the local IP addresses on outer interface.
4005  *
4006  * @param[in] tcf
4007  *   Context object initialized by mlx5_flow_tcf_context_create().
4008  * @param[in] ifindex
4009  *   Network inferface index to perform cleanup.
4010  */
4011 static void
4012 flow_tcf_encap_local_cleanup(struct mlx5_flow_tcf_context *tcf,
4013                             unsigned int ifindex)
4014 {
4015         struct nlmsghdr *nlh;
4016         struct ifaddrmsg *ifa;
4017         struct tcf_nlcb_context ctx = {
4018                 .ifindex = ifindex,
4019                 .bufsize = MNL_REQUEST_SIZE,
4020                 .nlbuf = LIST_HEAD_INITIALIZER(),
4021         };
4022         int ret;
4023
4024         assert(ifindex);
4025         /*
4026          * Seek and destroy leftovers of local IP addresses with
4027          * matching properties "scope link".
4028          */
4029         nlh = mnl_nlmsg_put_header(tcf->buf);
4030         nlh->nlmsg_type = RTM_GETADDR;
4031         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4032         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4033         ifa->ifa_family = AF_UNSPEC;
4034         ifa->ifa_index = ifindex;
4035         ifa->ifa_scope = RT_SCOPE_LINK;
4036         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_local_cb, &ctx);
4037         if (ret)
4038                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4039         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4040         if (ret)
4041                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4042 }
4043
4044 /**
4045  * Collect neigh permament rules on specified network device.
4046  * This is callback routine called by libmnl mnl_cb_run() in loop for
4047  * every message in received packet.
4048  *
4049  * @param[in] nlh
4050  *   Pointer to reply header.
4051  * @param[in, out] arg
4052  *   Opaque data pointer for this callback.
4053  *
4054  * @return
4055  *   A positive, nonzero value on success, negative errno value otherwise
4056  *   and rte_errno is set.
4057  */
4058 static int
4059 flow_tcf_collect_neigh_cb(const struct nlmsghdr *nlh, void *arg)
4060 {
4061         struct tcf_nlcb_context *ctx = arg;
4062         struct nlmsghdr *cmd;
4063         struct ndmsg *ndm;
4064         struct nlattr *na;
4065         struct nlattr *na_ip = NULL;
4066         struct nlattr *na_mac = NULL;
4067         unsigned char family;
4068         uint32_t size;
4069
4070         if (nlh->nlmsg_type != RTM_NEWNEIGH) {
4071                 rte_errno = EINVAL;
4072                 return -rte_errno;
4073         }
4074         ndm = mnl_nlmsg_get_payload(nlh);
4075         family = ndm->ndm_family;
4076         if (ndm->ndm_ifindex != (int)ctx->ifindex ||
4077            !(ndm->ndm_state & NUD_PERMANENT) ||
4078            (family != AF_INET && family != AF_INET6))
4079                 return 1;
4080         mnl_attr_for_each(na, nlh, sizeof(*ndm)) {
4081                 switch (mnl_attr_get_type(na)) {
4082                 case NDA_DST:
4083                         na_ip = na;
4084                         break;
4085                 case NDA_LLADDR:
4086                         na_mac = na;
4087                         break;
4088                 }
4089                 if (na_mac && na_ip)
4090                         break;
4091         }
4092         if (!na_mac || !na_ip)
4093                 return 1;
4094         /* Neigh rule with permenent attribute found. */
4095         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4096                MNL_ALIGN(sizeof(struct ndmsg)) +
4097                SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) +
4098                (family == AF_INET6 ? SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
4099                                    : SZ_NLATTR_TYPE_OF(uint32_t));
4100         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4101         if (!cmd) {
4102                 rte_errno = ENOMEM;
4103                 return -rte_errno;
4104         }
4105         cmd = mnl_nlmsg_put_header(cmd);
4106         cmd->nlmsg_type = RTM_DELNEIGH;
4107         cmd->nlmsg_flags = NLM_F_REQUEST;
4108         ndm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ndm));
4109         ndm->ndm_ifindex = ctx->ifindex;
4110         ndm->ndm_state = NUD_PERMANENT;
4111         ndm->ndm_flags = 0;
4112         ndm->ndm_type = 0;
4113         if (family == AF_INET) {
4114                 ndm->ndm_family = AF_INET;
4115                 mnl_attr_put_u32(cmd, NDA_DST, mnl_attr_get_u32(na_ip));
4116         } else {
4117                 ndm->ndm_family = AF_INET6;
4118                 mnl_attr_put(cmd, NDA_DST, IPV6_ADDR_LEN,
4119                              mnl_attr_get_payload(na_ip));
4120         }
4121         mnl_attr_put(cmd, NDA_LLADDR, ETHER_ADDR_LEN,
4122                      mnl_attr_get_payload(na_mac));
4123         assert(size == cmd->nlmsg_len);
4124         return 1;
4125 }
4126
4127 /**
4128  * Cleanup the neigh rules on outer interface.
4129  *
4130  * @param[in] tcf
4131  *   Context object initialized by mlx5_flow_tcf_context_create().
4132  * @param[in] ifindex
4133  *   Network inferface index to perform cleanup.
4134  */
4135 static void
4136 flow_tcf_encap_neigh_cleanup(struct mlx5_flow_tcf_context *tcf,
4137                             unsigned int ifindex)
4138 {
4139         struct nlmsghdr *nlh;
4140         struct ndmsg *ndm;
4141         struct tcf_nlcb_context ctx = {
4142                 .ifindex = ifindex,
4143                 .bufsize = MNL_REQUEST_SIZE,
4144                 .nlbuf = LIST_HEAD_INITIALIZER(),
4145         };
4146         int ret;
4147
4148         assert(ifindex);
4149         /* Seek and destroy leftovers of neigh rules. */
4150         nlh = mnl_nlmsg_put_header(tcf->buf);
4151         nlh->nlmsg_type = RTM_GETNEIGH;
4152         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4153         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4154         ndm->ndm_family = AF_UNSPEC;
4155         ndm->ndm_ifindex = ifindex;
4156         ndm->ndm_state = NUD_PERMANENT;
4157         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_neigh_cb, &ctx);
4158         if (ret)
4159                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4160         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4161         if (ret)
4162                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4163 }
4164
4165 /**
4166  * Collect indices of VXLAN encap/decap interfaces associated with device.
4167  * This is callback routine called by libmnl mnl_cb_run() in loop for
4168  * every message in received packet.
4169  *
4170  * @param[in] nlh
4171  *   Pointer to reply header.
4172  * @param[in, out] arg
4173  *   Opaque data pointer for this callback.
4174  *
4175  * @return
4176  *   A positive, nonzero value on success, negative errno value otherwise
4177  *   and rte_errno is set.
4178  */
4179 static int
4180 flow_tcf_collect_vxlan_cb(const struct nlmsghdr *nlh, void *arg)
4181 {
4182         struct tcf_nlcb_context *ctx = arg;
4183         struct nlmsghdr *cmd;
4184         struct ifinfomsg *ifm;
4185         struct nlattr *na;
4186         struct nlattr *na_info = NULL;
4187         struct nlattr *na_vxlan = NULL;
4188         bool found = false;
4189         unsigned int vxindex;
4190         uint32_t size;
4191
4192         if (nlh->nlmsg_type != RTM_NEWLINK) {
4193                 rte_errno = EINVAL;
4194                 return -rte_errno;
4195         }
4196         ifm = mnl_nlmsg_get_payload(nlh);
4197         if (!ifm->ifi_index) {
4198                 rte_errno = EINVAL;
4199                 return -rte_errno;
4200         }
4201         mnl_attr_for_each(na, nlh, sizeof(*ifm))
4202                 if (mnl_attr_get_type(na) == IFLA_LINKINFO) {
4203                         na_info = na;
4204                         break;
4205                 }
4206         if (!na_info)
4207                 return 1;
4208         mnl_attr_for_each_nested(na, na_info) {
4209                 switch (mnl_attr_get_type(na)) {
4210                 case IFLA_INFO_KIND:
4211                         if (!strncmp("vxlan", mnl_attr_get_str(na),
4212                                      mnl_attr_get_len(na)))
4213                                 found = true;
4214                         break;
4215                 case IFLA_INFO_DATA:
4216                         na_vxlan = na;
4217                         break;
4218                 }
4219                 if (found && na_vxlan)
4220                         break;
4221         }
4222         if (!found || !na_vxlan)
4223                 return 1;
4224         found = false;
4225         mnl_attr_for_each_nested(na, na_vxlan) {
4226                 if (mnl_attr_get_type(na) == IFLA_VXLAN_LINK &&
4227                     mnl_attr_get_u32(na) == ctx->ifindex) {
4228                         found = true;
4229                         break;
4230                 }
4231         }
4232         if (!found)
4233                 return 1;
4234         /* Attached VXLAN device found, store the command to delete. */
4235         vxindex = ifm->ifi_index;
4236         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4237                MNL_ALIGN(sizeof(struct ifinfomsg));
4238         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4239         if (!cmd) {
4240                 rte_errno = ENOMEM;
4241                 return -rte_errno;
4242         }
4243         cmd = mnl_nlmsg_put_header(cmd);
4244         cmd->nlmsg_type = RTM_DELLINK;
4245         cmd->nlmsg_flags = NLM_F_REQUEST;
4246         ifm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifm));
4247         ifm->ifi_family = AF_UNSPEC;
4248         ifm->ifi_index = vxindex;
4249         assert(size == cmd->nlmsg_len);
4250         return 1;
4251 }
4252
4253 /**
4254  * Cleanup the outer interface. Removes all found vxlan devices
4255  * attached to specified index, flushes the meigh and local IP
4256  * datavase.
4257  *
4258  * @param[in] tcf
4259  *   Context object initialized by mlx5_flow_tcf_context_create().
4260  * @param[in] ifindex
4261  *   Network inferface index to perform cleanup.
4262  */
4263 static void
4264 flow_tcf_encap_iface_cleanup(struct mlx5_flow_tcf_context *tcf,
4265                             unsigned int ifindex)
4266 {
4267         struct nlmsghdr *nlh;
4268         struct ifinfomsg *ifm;
4269         struct tcf_nlcb_context ctx = {
4270                 .ifindex = ifindex,
4271                 .bufsize = MNL_REQUEST_SIZE,
4272                 .nlbuf = LIST_HEAD_INITIALIZER(),
4273         };
4274         int ret;
4275
4276         assert(ifindex);
4277         /*
4278          * Seek and destroy leftover VXLAN encap/decap interfaces with
4279          * matching properties.
4280          */
4281         nlh = mnl_nlmsg_put_header(tcf->buf);
4282         nlh->nlmsg_type = RTM_GETLINK;
4283         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4284         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4285         ifm->ifi_family = AF_UNSPEC;
4286         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_vxlan_cb, &ctx);
4287         if (ret)
4288                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4289         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4290         if (ret)
4291                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4292 }
4293
4294 /**
4295  * Emit Netlink message to add/remove local address to the outer device.
4296  * The address being added is visible within the link only (scope link).
4297  *
4298  * Note that an implicit route is maintained by the kernel due to the
4299  * presence of a peer address (IFA_ADDRESS).
4300  *
4301  * These rules are used for encapsultion only and allow to assign
4302  * the outer tunnel source IP address.
4303  *
4304  * @param[in] tcf
4305  *   Libmnl socket context object.
4306  * @param[in] encap
4307  *   Encapsulation properties (source address and its peer).
4308  * @param[in] ifindex
4309  *   Network interface to apply rule.
4310  * @param[in] enable
4311  *   Toggle between add and remove.
4312  * @param[out] error
4313  *   Perform verbose error reporting if not NULL.
4314  *
4315  * @return
4316  *   0 on success, a negative errno value otherwise and rte_errno is set.
4317  */
4318 static int
4319 flow_tcf_rule_local(struct mlx5_flow_tcf_context *tcf,
4320                     const struct flow_tcf_vxlan_encap *encap,
4321                     unsigned int ifindex,
4322                     bool enable,
4323                     struct rte_flow_error *error)
4324 {
4325         struct nlmsghdr *nlh;
4326         struct ifaddrmsg *ifa;
4327         alignas(struct nlmsghdr)
4328         uint8_t buf[mnl_nlmsg_size(sizeof(*ifa) + 128)];
4329
4330         nlh = mnl_nlmsg_put_header(buf);
4331         nlh->nlmsg_type = enable ? RTM_NEWADDR : RTM_DELADDR;
4332         nlh->nlmsg_flags =
4333                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4334         nlh->nlmsg_seq = 0;
4335         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4336         ifa->ifa_flags = IFA_F_PERMANENT;
4337         ifa->ifa_scope = RT_SCOPE_LINK;
4338         ifa->ifa_index = ifindex;
4339         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4340                 ifa->ifa_family = AF_INET;
4341                 ifa->ifa_prefixlen = 32;
4342                 mnl_attr_put_u32(nlh, IFA_LOCAL, encap->ipv4.src);
4343                 if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST)
4344                         mnl_attr_put_u32(nlh, IFA_ADDRESS,
4345                                               encap->ipv4.dst);
4346         } else {
4347                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4348                 ifa->ifa_family = AF_INET6;
4349                 ifa->ifa_prefixlen = 128;
4350                 mnl_attr_put(nlh, IFA_LOCAL,
4351                                   sizeof(encap->ipv6.src),
4352                                   &encap->ipv6.src);
4353                 if (encap->mask & FLOW_TCF_ENCAP_IPV6_DST)
4354                         mnl_attr_put(nlh, IFA_ADDRESS,
4355                                           sizeof(encap->ipv6.dst),
4356                                           &encap->ipv6.dst);
4357         }
4358         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4359                 return 0;
4360         return rte_flow_error_set(error, rte_errno,
4361                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4362                                   "netlink: cannot complete IFA request"
4363                                   " (ip addr add)");
4364 }
4365
4366 /**
4367  * Emit Netlink message to add/remove neighbor.
4368  *
4369  * @param[in] tcf
4370  *   Libmnl socket context object.
4371  * @param[in] encap
4372  *   Encapsulation properties (destination address).
4373  * @param[in] ifindex
4374  *   Network interface.
4375  * @param[in] enable
4376  *   Toggle between add and remove.
4377  * @param[out] error
4378  *   Perform verbose error reporting if not NULL.
4379  *
4380  * @return
4381  *   0 on success, a negative errno value otherwise and rte_errno is set.
4382  */
4383 static int
4384 flow_tcf_rule_neigh(struct mlx5_flow_tcf_context *tcf,
4385                      const struct flow_tcf_vxlan_encap *encap,
4386                      unsigned int ifindex,
4387                      bool enable,
4388                      struct rte_flow_error *error)
4389 {
4390         struct nlmsghdr *nlh;
4391         struct ndmsg *ndm;
4392         alignas(struct nlmsghdr)
4393         uint8_t buf[mnl_nlmsg_size(sizeof(*ndm) + 128)];
4394
4395         nlh = mnl_nlmsg_put_header(buf);
4396         nlh->nlmsg_type = enable ? RTM_NEWNEIGH : RTM_DELNEIGH;
4397         nlh->nlmsg_flags =
4398                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4399         nlh->nlmsg_seq = 0;
4400         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4401         ndm->ndm_ifindex = ifindex;
4402         ndm->ndm_state = NUD_PERMANENT;
4403         ndm->ndm_flags = 0;
4404         ndm->ndm_type = 0;
4405         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4406                 ndm->ndm_family = AF_INET;
4407                 mnl_attr_put_u32(nlh, NDA_DST, encap->ipv4.dst);
4408         } else {
4409                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4410                 ndm->ndm_family = AF_INET6;
4411                 mnl_attr_put(nlh, NDA_DST, sizeof(encap->ipv6.dst),
4412                                                  &encap->ipv6.dst);
4413         }
4414         if (encap->mask & FLOW_TCF_ENCAP_ETH_SRC && enable)
4415                 DRV_LOG(WARNING,
4416                         "outer ethernet source address cannot be "
4417                         "forced for VXLAN encapsulation");
4418         if (encap->mask & FLOW_TCF_ENCAP_ETH_DST)
4419                 mnl_attr_put(nlh, NDA_LLADDR, sizeof(encap->eth.dst),
4420                                                     &encap->eth.dst);
4421         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4422                 return 0;
4423         return rte_flow_error_set(error, rte_errno,
4424                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4425                                   "netlink: cannot complete ND request"
4426                                   " (ip neigh)");
4427 }
4428
4429 /**
4430  * Manage the local IP addresses and their peers IP addresses on the
4431  * outer interface for encapsulation purposes. The kernel searches the
4432  * appropriate device for tunnel egress traffic using the outer source
4433  * IP, this IP should be assigned to the outer network device, otherwise
4434  * kernel rejects the rule.
4435  *
4436  * Adds or removes the addresses using the Netlink command like this:
4437  *   ip addr add <src_ip> peer <dst_ip> scope link dev <ifouter>
4438  *
4439  * The addresses are local to the netdev ("scope link"), this reduces
4440  * the risk of conflicts. Note that an implicit route is maintained by
4441  * the kernel due to the presence of a peer address (IFA_ADDRESS).
4442  *
4443  * @param[in] tcf
4444  *   Libmnl socket context object.
4445  * @param[in] vtep
4446  *   VTEP object, contains rule database and ifouter index.
4447  * @param[in] dev_flow
4448  *   Flow object, contains the tunnel parameters (for encap only).
4449  * @param[in] enable
4450  *   Toggle between add and remove.
4451  * @param[out] error
4452  *   Perform verbose error reporting if not NULL.
4453  *
4454  * @return
4455  *   0 on success, a negative errno value otherwise and rte_errno is set.
4456  */
4457 static int
4458 flow_tcf_encap_local(struct mlx5_flow_tcf_context *tcf,
4459                      struct tcf_vtep *vtep,
4460                      struct mlx5_flow *dev_flow,
4461                      bool enable,
4462                      struct rte_flow_error *error)
4463 {
4464         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4465         struct tcf_local_rule *rule;
4466         bool found = false;
4467         int ret;
4468
4469         assert(encap);
4470         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4471         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4472                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_DST);
4473                 LIST_FOREACH(rule, &vtep->local, next) {
4474                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_SRC &&
4475                             encap->ipv4.src == rule->ipv4.src &&
4476                             encap->ipv4.dst == rule->ipv4.dst) {
4477                                 found = true;
4478                                 break;
4479                         }
4480                 }
4481         } else {
4482                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4483                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4484                 LIST_FOREACH(rule, &vtep->local, next) {
4485                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_SRC &&
4486                             !memcmp(&encap->ipv6.src, &rule->ipv6.src,
4487                                             sizeof(encap->ipv6.src)) &&
4488                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4489                                             sizeof(encap->ipv6.dst))) {
4490                                 found = true;
4491                                 break;
4492                         }
4493                 }
4494         }
4495         if (found) {
4496                 if (enable) {
4497                         rule->refcnt++;
4498                         return 0;
4499                 }
4500                 if (!rule->refcnt || !--rule->refcnt) {
4501                         LIST_REMOVE(rule, next);
4502                         return flow_tcf_rule_local(tcf, encap,
4503                                         vtep->ifouter, false, error);
4504                 }
4505                 return 0;
4506         }
4507         if (!enable) {
4508                 DRV_LOG(WARNING, "disabling not existing local rule");
4509                 rte_flow_error_set(error, ENOENT,
4510                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4511                                    "disabling not existing local rule");
4512                 return -ENOENT;
4513         }
4514         rule = rte_zmalloc(__func__, sizeof(struct tcf_local_rule),
4515                                 alignof(struct tcf_local_rule));
4516         if (!rule) {
4517                 rte_flow_error_set(error, ENOMEM,
4518                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4519                                    "unable to allocate memory for local rule");
4520                 return -rte_errno;
4521         }
4522         *rule = (struct tcf_local_rule){.refcnt = 0,
4523                                         .mask = 0,
4524                                         };
4525         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4526                 rule->mask = FLOW_TCF_ENCAP_IPV4_SRC
4527                            | FLOW_TCF_ENCAP_IPV4_DST;
4528                 rule->ipv4.src = encap->ipv4.src;
4529                 rule->ipv4.dst = encap->ipv4.dst;
4530         } else {
4531                 rule->mask = FLOW_TCF_ENCAP_IPV6_SRC
4532                            | FLOW_TCF_ENCAP_IPV6_DST;
4533                 memcpy(&rule->ipv6.src, &encap->ipv6.src, IPV6_ADDR_LEN);
4534                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4535         }
4536         ret = flow_tcf_rule_local(tcf, encap, vtep->ifouter, true, error);
4537         if (ret) {
4538                 rte_free(rule);
4539                 return ret;
4540         }
4541         rule->refcnt++;
4542         LIST_INSERT_HEAD(&vtep->local, rule, next);
4543         return 0;
4544 }
4545
4546 /**
4547  * Manage the destination MAC/IP addresses neigh database, kernel uses
4548  * this one to determine the destination MAC address within encapsulation
4549  * header. Adds or removes the entries using the Netlink command like this:
4550  *   ip neigh add dev <ifouter> lladdr <dst_mac> to <dst_ip> nud permanent
4551  *
4552  * @param[in] tcf
4553  *   Libmnl socket context object.
4554  * @param[in] vtep
4555  *   VTEP object, contains rule database and ifouter index.
4556  * @param[in] dev_flow
4557  *   Flow object, contains the tunnel parameters (for encap only).
4558  * @param[in] enable
4559  *   Toggle between add and remove.
4560  * @param[out] error
4561  *   Perform verbose error reporting if not NULL.
4562  *
4563  * @return
4564  *   0 on success, a negative errno value otherwise and rte_errno is set.
4565  */
4566 static int
4567 flow_tcf_encap_neigh(struct mlx5_flow_tcf_context *tcf,
4568                      struct tcf_vtep *vtep,
4569                      struct mlx5_flow *dev_flow,
4570                      bool enable,
4571                      struct rte_flow_error *error)
4572 {
4573         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4574         struct tcf_neigh_rule *rule;
4575         bool found = false;
4576         int ret;
4577
4578         assert(encap);
4579         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4580         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4581                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_SRC);
4582                 LIST_FOREACH(rule, &vtep->neigh, next) {
4583                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_DST &&
4584                             encap->ipv4.dst == rule->ipv4.dst) {
4585                                 found = true;
4586                                 break;
4587                         }
4588                 }
4589         } else {
4590                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4591                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4592                 LIST_FOREACH(rule, &vtep->neigh, next) {
4593                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_DST &&
4594                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4595                                                 sizeof(encap->ipv6.dst))) {
4596                                 found = true;
4597                                 break;
4598                         }
4599                 }
4600         }
4601         if (found) {
4602                 if (memcmp(&encap->eth.dst, &rule->eth,
4603                            sizeof(encap->eth.dst))) {
4604                         DRV_LOG(WARNING, "Destination MAC differs"
4605                                          " in neigh rule");
4606                         rte_flow_error_set(error, EEXIST,
4607                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4608                                            NULL, "Different MAC address"
4609                                            " neigh rule for the same"
4610                                            " destination IP");
4611                                         return -EEXIST;
4612                 }
4613                 if (enable) {
4614                         rule->refcnt++;
4615                         return 0;
4616                 }
4617                 if (!rule->refcnt || !--rule->refcnt) {
4618                         LIST_REMOVE(rule, next);
4619                         return flow_tcf_rule_neigh(tcf, encap,
4620                                                    vtep->ifouter,
4621                                                    false, error);
4622                 }
4623                 return 0;
4624         }
4625         if (!enable) {
4626                 DRV_LOG(WARNING, "Disabling not existing neigh rule");
4627                 rte_flow_error_set(error, ENOENT,
4628                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4629                                    "unable to allocate memory for neigh rule");
4630                 return -ENOENT;
4631         }
4632         rule = rte_zmalloc(__func__, sizeof(struct tcf_neigh_rule),
4633                                 alignof(struct tcf_neigh_rule));
4634         if (!rule) {
4635                 rte_flow_error_set(error, ENOMEM,
4636                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4637                                    "unable to allocate memory for neigh rule");
4638                 return -rte_errno;
4639         }
4640         *rule = (struct tcf_neigh_rule){.refcnt = 0,
4641                                         .mask = 0,
4642                                         };
4643         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4644                 rule->mask = FLOW_TCF_ENCAP_IPV4_DST;
4645                 rule->ipv4.dst = encap->ipv4.dst;
4646         } else {
4647                 rule->mask = FLOW_TCF_ENCAP_IPV6_DST;
4648                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4649         }
4650         memcpy(&rule->eth, &encap->eth.dst, sizeof(rule->eth));
4651         ret = flow_tcf_rule_neigh(tcf, encap, vtep->ifouter, true, error);
4652         if (ret) {
4653                 rte_free(rule);
4654                 return ret;
4655         }
4656         rule->refcnt++;
4657         LIST_INSERT_HEAD(&vtep->neigh, rule, next);
4658         return 0;
4659 }
4660
4661 /* VTEP device list is shared between PMD port instances. */
4662 static LIST_HEAD(, tcf_vtep) vtep_list_vxlan = LIST_HEAD_INITIALIZER();
4663 static pthread_mutex_t vtep_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4664
4665 /**
4666  * Deletes VTEP network device.
4667  *
4668  * @param[in] tcf
4669  *   Context object initialized by mlx5_flow_tcf_context_create().
4670  * @param[in] vtep
4671  *   Object represinting the network device to delete. Memory
4672  *   allocated for this object is freed by routine.
4673  */
4674 static void
4675 flow_tcf_vtep_delete(struct mlx5_flow_tcf_context *tcf,
4676                      struct tcf_vtep *vtep)
4677 {
4678         struct nlmsghdr *nlh;
4679         struct ifinfomsg *ifm;
4680         alignas(struct nlmsghdr)
4681         uint8_t buf[mnl_nlmsg_size(MNL_ALIGN(sizeof(*ifm))) +
4682                     MNL_BUF_EXTRA_SPACE];
4683         int ret;
4684
4685         assert(!vtep->refcnt);
4686         /* Delete only ifaces those we actually created. */
4687         if (vtep->created && vtep->ifindex) {
4688                 DRV_LOG(INFO, "VTEP delete (%d)", vtep->ifindex);
4689                 nlh = mnl_nlmsg_put_header(buf);
4690                 nlh->nlmsg_type = RTM_DELLINK;
4691                 nlh->nlmsg_flags = NLM_F_REQUEST;
4692                 ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4693                 ifm->ifi_family = AF_UNSPEC;
4694                 ifm->ifi_index = vtep->ifindex;
4695                 assert(sizeof(buf) >= nlh->nlmsg_len);
4696                 ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4697                 if (ret)
4698                         DRV_LOG(WARNING, "netlink: error deleting vxlan"
4699                                          " encap/decap ifindex %u",
4700                                          ifm->ifi_index);
4701         }
4702         rte_free(vtep);
4703 }
4704
4705 /**
4706  * Creates VTEP network device.
4707  *
4708  * @param[in] tcf
4709  *   Context object initialized by mlx5_flow_tcf_context_create().
4710  * @param[in] ifouter
4711  *   Outer interface to attach new-created VXLAN device
4712  *   If zero the VXLAN device will not be attached to any device.
4713  *   These VTEPs are used for decapsulation and can be precreated
4714  *   and shared between processes.
4715  * @param[in] port
4716  *   UDP port of created VTEP device.
4717  * @param[out] error
4718  *   Perform verbose error reporting if not NULL.
4719  *
4720  * @return
4721  * Pointer to created device structure on success,
4722  * NULL otherwise and rte_errno is set.
4723  */
4724 #ifdef HAVE_IFLA_VXLAN_COLLECT_METADATA
4725 static struct tcf_vtep*
4726 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf,
4727                      unsigned int ifouter,
4728                      uint16_t port, struct rte_flow_error *error)
4729 {
4730         struct tcf_vtep *vtep;
4731         struct nlmsghdr *nlh;
4732         struct ifinfomsg *ifm;
4733         char name[sizeof(MLX5_VXLAN_DEVICE_PFX) + 24];
4734         alignas(struct nlmsghdr)
4735         uint8_t buf[mnl_nlmsg_size(sizeof(*ifm)) +
4736                     SZ_NLATTR_DATA_OF(sizeof(name)) +
4737                     SZ_NLATTR_NEST * 2 +
4738                     SZ_NLATTR_STRZ_OF("vxlan") +
4739                     SZ_NLATTR_DATA_OF(sizeof(uint32_t)) +
4740                     SZ_NLATTR_DATA_OF(sizeof(uint16_t)) +
4741                     SZ_NLATTR_DATA_OF(sizeof(uint8_t)) * 3 +
4742                     MNL_BUF_EXTRA_SPACE];
4743         struct nlattr *na_info;
4744         struct nlattr *na_vxlan;
4745         rte_be16_t vxlan_port = rte_cpu_to_be_16(port);
4746         int ret;
4747
4748         vtep = rte_zmalloc(__func__, sizeof(*vtep), alignof(struct tcf_vtep));
4749         if (!vtep) {
4750                 rte_flow_error_set(error, ENOMEM,
4751                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4752                                    "unable to allocate memory for VTEP");
4753                 return NULL;
4754         }
4755         *vtep = (struct tcf_vtep){
4756                         .port = port,
4757                         .local = LIST_HEAD_INITIALIZER(),
4758                         .neigh = LIST_HEAD_INITIALIZER(),
4759         };
4760         memset(buf, 0, sizeof(buf));
4761         nlh = mnl_nlmsg_put_header(buf);
4762         nlh->nlmsg_type = RTM_NEWLINK;
4763         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE  | NLM_F_EXCL;
4764         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4765         ifm->ifi_family = AF_UNSPEC;
4766         ifm->ifi_type = 0;
4767         ifm->ifi_index = 0;
4768         ifm->ifi_flags = IFF_UP;
4769         ifm->ifi_change = 0xffffffff;
4770         snprintf(name, sizeof(name), "%s%u", MLX5_VXLAN_DEVICE_PFX, port);
4771         mnl_attr_put_strz(nlh, IFLA_IFNAME, name);
4772         na_info = mnl_attr_nest_start(nlh, IFLA_LINKINFO);
4773         assert(na_info);
4774         mnl_attr_put_strz(nlh, IFLA_INFO_KIND, "vxlan");
4775         na_vxlan = mnl_attr_nest_start(nlh, IFLA_INFO_DATA);
4776         if (ifouter)
4777                 mnl_attr_put_u32(nlh, IFLA_VXLAN_LINK, ifouter);
4778         assert(na_vxlan);
4779         mnl_attr_put_u8(nlh, IFLA_VXLAN_COLLECT_METADATA, 1);
4780         mnl_attr_put_u8(nlh, IFLA_VXLAN_UDP_ZERO_CSUM6_RX, 1);
4781         mnl_attr_put_u8(nlh, IFLA_VXLAN_LEARNING, 0);
4782         mnl_attr_put_u16(nlh, IFLA_VXLAN_PORT, vxlan_port);
4783         mnl_attr_nest_end(nlh, na_vxlan);
4784         mnl_attr_nest_end(nlh, na_info);
4785         assert(sizeof(buf) >= nlh->nlmsg_len);
4786         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4787         if (ret) {
4788                 DRV_LOG(WARNING,
4789                         "netlink: VTEP %s create failure (%d)",
4790                         name, rte_errno);
4791                 if (rte_errno != EEXIST || ifouter)
4792                         /*
4793                          * Some unhandled error occurred or device is
4794                          * for encapsulation and cannot be shared.
4795                          */
4796                         goto error;
4797         } else {
4798                 /*
4799                  * Mark device we actually created.
4800                  * We should explicitly delete
4801                  * when we do not need it anymore.
4802                  */
4803                 vtep->created = 1;
4804         }
4805         /* Try to get ifindex of created of pre-existing device. */
4806         ret = if_nametoindex(name);
4807         if (!ret) {
4808                 DRV_LOG(WARNING,
4809                         "VTEP %s failed to get index (%d)", name, errno);
4810                 rte_flow_error_set
4811                         (error, -errno,
4812                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4813                          "netlink: failed to retrieve VTEP ifindex");
4814                 goto error;
4815         }
4816         vtep->ifindex = ret;
4817         vtep->ifouter = ifouter;
4818         memset(buf, 0, sizeof(buf));
4819         nlh = mnl_nlmsg_put_header(buf);
4820         nlh->nlmsg_type = RTM_NEWLINK;
4821         nlh->nlmsg_flags = NLM_F_REQUEST;
4822         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4823         ifm->ifi_family = AF_UNSPEC;
4824         ifm->ifi_type = 0;
4825         ifm->ifi_index = vtep->ifindex;
4826         ifm->ifi_flags = IFF_UP;
4827         ifm->ifi_change = IFF_UP;
4828         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4829         if (ret) {
4830                 rte_flow_error_set(error, -errno,
4831                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4832                                    "netlink: failed to set VTEP link up");
4833                 DRV_LOG(WARNING, "netlink: VTEP %s set link up failure (%d)",
4834                         name, rte_errno);
4835                 goto clean;
4836         }
4837         ret = mlx5_flow_tcf_init(tcf, vtep->ifindex, error);
4838         if (ret) {
4839                 DRV_LOG(WARNING, "VTEP %s init failure (%d)", name, rte_errno);
4840                 goto clean;
4841         }
4842         DRV_LOG(INFO, "VTEP create (%d, %d)", vtep->port, vtep->ifindex);
4843         vtep->refcnt = 1;
4844         return vtep;
4845 clean:
4846         flow_tcf_vtep_delete(tcf, vtep);
4847         return NULL;
4848 error:
4849         rte_free(vtep);
4850         return NULL;
4851 }
4852 #else
4853 static struct tcf_vtep*
4854 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf __rte_unused,
4855                      unsigned int ifouter __rte_unused,
4856                      uint16_t port __rte_unused,
4857                      struct rte_flow_error *error)
4858 {
4859         rte_flow_error_set(error, ENOTSUP,
4860                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4861                            "netlink: failed to create VTEP, "
4862                            "vxlan metadata are not supported by kernel");
4863         return NULL;
4864 }
4865 #endif /* HAVE_IFLA_VXLAN_COLLECT_METADATA */
4866
4867 /**
4868  * Acquire target interface index for VXLAN tunneling decapsulation.
4869  * In order to share the UDP port within the other interfaces the
4870  * VXLAN device created as not attached to any interface (if created).
4871  *
4872  * @param[in] tcf
4873  *   Context object initialized by mlx5_flow_tcf_context_create().
4874  * @param[in] dev_flow
4875  *   Flow tcf object with tunnel structure pointer set.
4876  * @param[out] error
4877  *   Perform verbose error reporting if not NULL.
4878  * @return
4879  *   Interface descriptor pointer on success,
4880  *   NULL otherwise and rte_errno is set.
4881  */
4882 static struct tcf_vtep*
4883 flow_tcf_decap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4884                             struct mlx5_flow *dev_flow,
4885                             struct rte_flow_error *error)
4886 {
4887         struct tcf_vtep *vtep;
4888         uint16_t port = dev_flow->tcf.vxlan_decap->udp_port;
4889
4890         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4891                 if (vtep->port == port)
4892                         break;
4893         }
4894         if (vtep && vtep->ifouter) {
4895                 rte_flow_error_set(error, -errno,
4896                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4897                                    "Failed to create decap VTEP with specified"
4898                                    " UDP port, atatched device exists");
4899                 return NULL;
4900         }
4901         if (vtep) {
4902                 /* Device exists, just increment the reference counter. */
4903                 vtep->refcnt++;
4904                 assert(vtep->ifindex);
4905                 return vtep;
4906         }
4907         /* No decapsulation device exists, try to create the new one. */
4908         vtep = flow_tcf_vtep_create(tcf, 0, port, error);
4909         if (vtep)
4910                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
4911         return vtep;
4912 }
4913
4914 /**
4915  * Aqcuire target interface index for VXLAN tunneling encapsulation.
4916  *
4917  * @param[in] tcf
4918  *   Context object initialized by mlx5_flow_tcf_context_create().
4919  * @param[in] ifouter
4920  *   Network interface index to attach VXLAN encap device to.
4921  * @param[in] dev_flow
4922  *   Flow tcf object with tunnel structure pointer set.
4923  * @param[out] error
4924  *   Perform verbose error reporting if not NULL.
4925  * @return
4926  *   Interface descriptor pointer on success,
4927  *   NULL otherwise and rte_errno is set.
4928  */
4929 static struct tcf_vtep*
4930 flow_tcf_encap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4931                             unsigned int ifouter,
4932                             struct mlx5_flow *dev_flow __rte_unused,
4933                             struct rte_flow_error *error)
4934 {
4935         static uint16_t encap_port = MLX5_VXLAN_PORT_MIN - 1;
4936         struct tcf_vtep *vtep;
4937         int ret;
4938
4939         assert(ifouter);
4940         /* Look whether the attached VTEP for encap is created. */
4941         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4942                 if (vtep->ifouter == ifouter)
4943                         break;
4944         }
4945         if (vtep) {
4946                 /* VTEP already exists, just increment the reference. */
4947                 vtep->refcnt++;
4948         } else {
4949                 uint16_t pcnt;
4950
4951                 /* Not found, we should create the new attached VTEP. */
4952                 flow_tcf_encap_iface_cleanup(tcf, ifouter);
4953                 flow_tcf_encap_local_cleanup(tcf, ifouter);
4954                 flow_tcf_encap_neigh_cleanup(tcf, ifouter);
4955                 for (pcnt = 0; pcnt <= (MLX5_VXLAN_PORT_MAX
4956                                      - MLX5_VXLAN_PORT_MIN); pcnt++) {
4957                         encap_port++;
4958                         /* Wraparound the UDP port index. */
4959                         if (encap_port < MLX5_VXLAN_PORT_MIN ||
4960                             encap_port > MLX5_VXLAN_PORT_MAX)
4961                                 encap_port = MLX5_VXLAN_PORT_MIN;
4962                         /* Check whether UDP port is in already in use. */
4963                         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4964                                 if (vtep->port == encap_port)
4965                                         break;
4966                         }
4967                         if (vtep) {
4968                                 /* Port is in use, try the next one. */
4969                                 vtep = NULL;
4970                                 continue;
4971                         }
4972                         vtep = flow_tcf_vtep_create(tcf, ifouter,
4973                                                     encap_port, error);
4974                         if (vtep) {
4975                                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
4976                                 break;
4977                         }
4978                         if (rte_errno != EEXIST)
4979                                 break;
4980                 }
4981                 if (!vtep)
4982                         return NULL;
4983         }
4984         assert(vtep->ifouter == ifouter);
4985         assert(vtep->ifindex);
4986         /* Create local ipaddr with peer to specify the outer IPs. */
4987         ret = flow_tcf_encap_local(tcf, vtep, dev_flow, true, error);
4988         if (!ret) {
4989                 /* Create neigh rule to specify outer destination MAC. */
4990                 ret = flow_tcf_encap_neigh(tcf, vtep, dev_flow, true, error);
4991                 if (ret)
4992                         flow_tcf_encap_local(tcf, vtep,
4993                                              dev_flow, false, error);
4994         }
4995         if (ret) {
4996                 if (--vtep->refcnt == 0)
4997                         flow_tcf_vtep_delete(tcf, vtep);
4998                 return NULL;
4999         }
5000         return vtep;
5001 }
5002
5003 /**
5004  * Acquires target interface index for tunneling of any type.
5005  * Creates the new VTEP if needed.
5006  *
5007  * @param[in] tcf
5008  *   Context object initialized by mlx5_flow_tcf_context_create().
5009  * @param[in] ifouter
5010  *   Network interface index to attach VXLAN encap device to.
5011  * @param[in] dev_flow
5012  *   Flow tcf object with tunnel structure pointer set.
5013  * @param[out] error
5014  *   Perform verbose error reporting if not NULL.
5015  * @return
5016  *   Interface descriptor pointer on success,
5017  *   NULL otherwise and rte_errno is set.
5018  */
5019 static struct tcf_vtep*
5020 flow_tcf_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
5021                       unsigned int ifouter,
5022                       struct mlx5_flow *dev_flow,
5023                       struct rte_flow_error *error)
5024 {
5025         struct tcf_vtep *vtep = NULL;
5026
5027         assert(dev_flow->tcf.tunnel);
5028         pthread_mutex_lock(&vtep_list_mutex);
5029         switch (dev_flow->tcf.tunnel->type) {
5030         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5031                 vtep = flow_tcf_encap_vtep_acquire(tcf, ifouter,
5032                                                   dev_flow, error);
5033                 break;
5034         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5035                 vtep = flow_tcf_decap_vtep_acquire(tcf, dev_flow, error);
5036                 break;
5037         default:
5038                 rte_flow_error_set(error, ENOTSUP,
5039                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5040                                    "unsupported tunnel type");
5041                 break;
5042         }
5043         pthread_mutex_unlock(&vtep_list_mutex);
5044         return vtep;
5045 }
5046
5047 /**
5048  * Release tunneling interface by ifindex. Decrements reference
5049  * counter and actually removes the device if counter is zero.
5050  *
5051  * @param[in] tcf
5052  *   Context object initialized by mlx5_flow_tcf_context_create().
5053  * @param[in] vtep
5054  *   VTEP device descriptor structure.
5055  * @param[in] dev_flow
5056  *   Flow tcf object with tunnel structure pointer set.
5057  */
5058 static void
5059 flow_tcf_vtep_release(struct mlx5_flow_tcf_context *tcf,
5060                       struct tcf_vtep *vtep,
5061                       struct mlx5_flow *dev_flow)
5062 {
5063         assert(dev_flow->tcf.tunnel);
5064         pthread_mutex_lock(&vtep_list_mutex);
5065         switch (dev_flow->tcf.tunnel->type) {
5066         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5067                 break;
5068         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5069                 /* Remove the encap ancillary rules first. */
5070                 flow_tcf_encap_neigh(tcf, vtep, dev_flow, false, NULL);
5071                 flow_tcf_encap_local(tcf, vtep, dev_flow, false, NULL);
5072                 break;
5073         default:
5074                 assert(false);
5075                 DRV_LOG(WARNING, "Unsupported tunnel type");
5076                 break;
5077         }
5078         assert(vtep->refcnt);
5079         if (--vtep->refcnt == 0) {
5080                 LIST_REMOVE(vtep, next);
5081                 flow_tcf_vtep_delete(tcf, vtep);
5082         }
5083         pthread_mutex_unlock(&vtep_list_mutex);
5084 }
5085
5086 struct tcf_nlcb_query {
5087         uint32_t handle;
5088         uint32_t tc_flags;
5089         uint32_t flags_valid:1;
5090 };
5091
5092 /**
5093  * Collect queried rule attributes. This is callback routine called by
5094  * libmnl mnl_cb_run() in loop for every message in received packet.
5095  * Current implementation collects the flower flags only.
5096  *
5097  * @param[in] nlh
5098  *   Pointer to reply header.
5099  * @param[in, out] arg
5100  *   Context pointer for this callback.
5101  *
5102  * @return
5103  *   A positive, nonzero value on success (required by libmnl
5104  *   to continue messages processing).
5105  */
5106 static int
5107 flow_tcf_collect_query_cb(const struct nlmsghdr *nlh, void *arg)
5108 {
5109         struct tcf_nlcb_query *query = arg;
5110         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
5111         struct nlattr *na, *na_opt;
5112         bool flower = false;
5113
5114         if (nlh->nlmsg_type != RTM_NEWTFILTER ||
5115             tcm->tcm_handle != query->handle)
5116                 return 1;
5117         mnl_attr_for_each(na, nlh, sizeof(*tcm)) {
5118                 switch (mnl_attr_get_type(na)) {
5119                 case TCA_KIND:
5120                         if (strcmp(mnl_attr_get_payload(na), "flower")) {
5121                                 /* Not flower filter, drop entire message. */
5122                                 return 1;
5123                         }
5124                         flower = true;
5125                         break;
5126                 case TCA_OPTIONS:
5127                         if (!flower) {
5128                                 /* Not flower options, drop entire message. */
5129                                 return 1;
5130                         }
5131                         /* Check nested flower options. */
5132                         mnl_attr_for_each_nested(na_opt, na) {
5133                                 switch (mnl_attr_get_type(na_opt)) {
5134                                 case TCA_FLOWER_FLAGS:
5135                                         query->flags_valid = 1;
5136                                         query->tc_flags =
5137                                                 mnl_attr_get_u32(na_opt);
5138                                         break;
5139                                 }
5140                         }
5141                         break;
5142                 }
5143         }
5144         return 1;
5145 }
5146
5147 /**
5148  * Query a TC flower rule flags via netlink.
5149  *
5150  * @param[in] tcf
5151  *   Context object initialized by mlx5_flow_tcf_context_create().
5152  * @param[in] dev_flow
5153  *   Pointer to the flow.
5154  * @param[out] pflags
5155  *   pointer to the data retrieved by the query.
5156  *
5157  * @return
5158  *   0 on success, a negative errno value otherwise.
5159  */
5160 static int
5161 flow_tcf_query_flags(struct mlx5_flow_tcf_context *tcf,
5162                      struct mlx5_flow *dev_flow,
5163                      uint32_t *pflags)
5164 {
5165         struct nlmsghdr *nlh;
5166         struct tcmsg *tcm;
5167         struct tcf_nlcb_query query = {
5168                 .handle = dev_flow->tcf.tcm->tcm_handle,
5169         };
5170
5171         nlh = mnl_nlmsg_put_header(tcf->buf);
5172         nlh->nlmsg_type = RTM_GETTFILTER;
5173         nlh->nlmsg_flags = NLM_F_REQUEST;
5174         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5175         memcpy(tcm, dev_flow->tcf.tcm, sizeof(*tcm));
5176         /*
5177          * Ignore Netlink error for filter query operations.
5178          * The reply length is sent by kernel as errno.
5179          * Just check we got the flags option.
5180          */
5181         flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_query_cb, &query);
5182         if (!query.flags_valid) {
5183                 *pflags = 0;
5184                 return -ENOENT;
5185         }
5186         *pflags = query.tc_flags;
5187         return 0;
5188 }
5189
5190 /**
5191  * Query and check the in_hw set for specified rule.
5192  *
5193  * @param[in] tcf
5194  *   Context object initialized by mlx5_flow_tcf_context_create().
5195  * @param[in] dev_flow
5196  *   Pointer to the flow to check.
5197  *
5198  * @return
5199  *   0 on success, a negative errno value otherwise.
5200  */
5201 static int
5202 flow_tcf_check_inhw(struct mlx5_flow_tcf_context *tcf,
5203                     struct mlx5_flow *dev_flow)
5204 {
5205         uint32_t flags;
5206         int ret;
5207
5208         ret = flow_tcf_query_flags(tcf, dev_flow, &flags);
5209         if (ret)
5210                 return ret;
5211         return  (flags & TCA_CLS_FLAGS_IN_HW) ? 0 : -ENOENT;
5212 }
5213
5214 /**
5215  * Remove flow from E-Switch by sending Netlink message.
5216  *
5217  * @param[in] dev
5218  *   Pointer to Ethernet device.
5219  * @param[in, out] flow
5220  *   Pointer to the sub flow.
5221  */
5222 static void
5223 flow_tcf_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5224 {
5225         struct priv *priv = dev->data->dev_private;
5226         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5227         struct mlx5_flow *dev_flow;
5228         struct nlmsghdr *nlh;
5229
5230         if (!flow)
5231                 return;
5232         dev_flow = LIST_FIRST(&flow->dev_flows);
5233         if (!dev_flow)
5234                 return;
5235         /* E-Switch flow can't be expanded. */
5236         assert(!LIST_NEXT(dev_flow, next));
5237         if (dev_flow->tcf.applied) {
5238                 nlh = dev_flow->tcf.nlh;
5239                 nlh->nlmsg_type = RTM_DELTFILTER;
5240                 nlh->nlmsg_flags = NLM_F_REQUEST;
5241                 flow_tcf_nl_ack(ctx, nlh, NULL, NULL);
5242                 if (dev_flow->tcf.tunnel) {
5243                         assert(dev_flow->tcf.tunnel->vtep);
5244                         flow_tcf_vtep_release(ctx,
5245                                 dev_flow->tcf.tunnel->vtep,
5246                                 dev_flow);
5247                         dev_flow->tcf.tunnel->vtep = NULL;
5248                 }
5249                 dev_flow->tcf.applied = 0;
5250         }
5251 }
5252
5253 /**
5254  * Apply flow to E-Switch by sending Netlink message.
5255  *
5256  * @param[in] dev
5257  *   Pointer to Ethernet device.
5258  * @param[in, out] flow
5259  *   Pointer to the sub flow.
5260  * @param[out] error
5261  *   Pointer to the error structure.
5262  *
5263  * @return
5264  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5265  */
5266 static int
5267 flow_tcf_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5268                struct rte_flow_error *error)
5269 {
5270         struct priv *priv = dev->data->dev_private;
5271         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5272         struct mlx5_flow *dev_flow;
5273         struct nlmsghdr *nlh;
5274
5275         dev_flow = LIST_FIRST(&flow->dev_flows);
5276         /* E-Switch flow can't be expanded. */
5277         assert(!LIST_NEXT(dev_flow, next));
5278         if (dev_flow->tcf.applied)
5279                 return 0;
5280         nlh = dev_flow->tcf.nlh;
5281         nlh->nlmsg_type = RTM_NEWTFILTER;
5282         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5283         if (dev_flow->tcf.tunnel) {
5284                 /*
5285                  * Replace the interface index, target for
5286                  * encapsulation, source for decapsulation.
5287                  */
5288                 assert(!dev_flow->tcf.tunnel->vtep);
5289                 assert(dev_flow->tcf.tunnel->ifindex_ptr);
5290                 /* Acquire actual VTEP device when rule is being applied. */
5291                 dev_flow->tcf.tunnel->vtep =
5292                         flow_tcf_vtep_acquire(ctx,
5293                                         dev_flow->tcf.tunnel->ifindex_org,
5294                                         dev_flow, error);
5295                 if (!dev_flow->tcf.tunnel->vtep)
5296                         return -rte_errno;
5297                 DRV_LOG(INFO, "Replace ifindex: %d->%d",
5298                                 dev_flow->tcf.tunnel->vtep->ifindex,
5299                                 dev_flow->tcf.tunnel->ifindex_org);
5300                 *dev_flow->tcf.tunnel->ifindex_ptr =
5301                         dev_flow->tcf.tunnel->vtep->ifindex;
5302         }
5303         if (!flow_tcf_nl_ack(ctx, nlh, NULL, NULL)) {
5304                 dev_flow->tcf.applied = 1;
5305                 if (*dev_flow->tcf.ptc_flags & TCA_CLS_FLAGS_SKIP_SW)
5306                         return 0;
5307                 /*
5308                  * Rule was applied without skip_sw flag set.
5309                  * We should check whether the rule was acctually
5310                  * accepted by hardware (have look at in_hw flag).
5311                  */
5312                 if (flow_tcf_check_inhw(ctx, dev_flow)) {
5313                         flow_tcf_remove(dev, flow);
5314                         return rte_flow_error_set
5315                                 (error, ENOENT,
5316                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5317                                  "netlink: rule has no in_hw flag set");
5318                 }
5319                 return 0;
5320         }
5321         if (dev_flow->tcf.tunnel) {
5322                 /* Rollback the VTEP configuration if rule apply failed. */
5323                 assert(dev_flow->tcf.tunnel->vtep);
5324                 flow_tcf_vtep_release(ctx, dev_flow->tcf.tunnel->vtep,
5325                                       dev_flow);
5326                 dev_flow->tcf.tunnel->vtep = NULL;
5327         }
5328         return rte_flow_error_set(error, rte_errno,
5329                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5330                                   "netlink: failed to create TC flow rule");
5331 }
5332
5333 /**
5334  * Remove flow from E-Switch and release resources of the device flow.
5335  *
5336  * @param[in] dev
5337  *   Pointer to Ethernet device.
5338  * @param[in, out] flow
5339  *   Pointer to the sub flow.
5340  */
5341 static void
5342 flow_tcf_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5343 {
5344         struct mlx5_flow *dev_flow;
5345
5346         if (!flow)
5347                 return;
5348         flow_tcf_remove(dev, flow);
5349         if (flow->counter) {
5350                 if (--flow->counter->ref_cnt == 0) {
5351                         rte_free(flow->counter);
5352                         flow->counter = NULL;
5353                 }
5354         }
5355         dev_flow = LIST_FIRST(&flow->dev_flows);
5356         if (!dev_flow)
5357                 return;
5358         /* E-Switch flow can't be expanded. */
5359         assert(!LIST_NEXT(dev_flow, next));
5360         LIST_REMOVE(dev_flow, next);
5361         rte_free(dev_flow);
5362 }
5363
5364 /**
5365  * Helper routine for figuring the space size required for a parse buffer.
5366  *
5367  * @param array
5368  *   array of values to use.
5369  * @param idx
5370  *   Current location in array.
5371  * @param value
5372  *   Value to compare with.
5373  *
5374  * @return
5375  *   The maximum between the given value and the array value on index.
5376  */
5377 static uint16_t
5378 flow_tcf_arr_val_max(uint16_t array[], int idx, uint16_t value)
5379 {
5380         return idx < 0 ? (value) : RTE_MAX((array)[idx], value);
5381 }
5382
5383 /**
5384  * Parse rtnetlink message attributes filling the attribute table with the info
5385  * retrieved.
5386  *
5387  * @param tb
5388  *   Attribute table to be filled.
5389  * @param[out] max
5390  *   Maxinum entry in the attribute table.
5391  * @param rte
5392  *   The attributes section in the message to be parsed.
5393  * @param len
5394  *   The length of the attributes section in the message.
5395  */
5396 static void
5397 flow_tcf_nl_parse_rtattr(struct rtattr *tb[], int max,
5398                          struct rtattr *rta, int len)
5399 {
5400         unsigned short type;
5401         memset(tb, 0, sizeof(struct rtattr *) * (max + 1));
5402         while (RTA_OK(rta, len)) {
5403                 type = rta->rta_type;
5404                 if (type <= max && !tb[type])
5405                         tb[type] = rta;
5406                 rta = RTA_NEXT(rta, len);
5407         }
5408 }
5409
5410 /**
5411  * Extract flow counters from flower action.
5412  *
5413  * @param rta
5414  *   flower action stats properties in the Netlink message received.
5415  * @param rta_type
5416  *   The backward sequence of rta_types, as written in the attribute table,
5417  *   we need to traverse in order to get to the requested object.
5418  * @param idx
5419  *   Current location in rta_type table.
5420  * @param[out] data
5421  *   data holding the count statistics of the rte_flow retrieved from
5422  *   the message.
5423  *
5424  * @return
5425  *   0 if data was found and retrieved, -1 otherwise.
5426  */
5427 static int
5428 flow_tcf_nl_action_stats_parse_and_get(struct rtattr *rta,
5429                                        uint16_t rta_type[], int idx,
5430                                        struct gnet_stats_basic *data)
5431 {
5432         int tca_stats_max = flow_tcf_arr_val_max(rta_type, idx,
5433                                                  TCA_STATS_BASIC);
5434         struct rtattr *tbs[tca_stats_max + 1];
5435
5436         if (rta == NULL || idx < 0)
5437                 return -1;
5438         flow_tcf_nl_parse_rtattr(tbs, tca_stats_max,
5439                                  RTA_DATA(rta), RTA_PAYLOAD(rta));
5440         switch (rta_type[idx]) {
5441         case TCA_STATS_BASIC:
5442                 if (tbs[TCA_STATS_BASIC]) {
5443                         memcpy(data, RTA_DATA(tbs[TCA_STATS_BASIC]),
5444                                RTE_MIN(RTA_PAYLOAD(tbs[TCA_STATS_BASIC]),
5445                                sizeof(*data)));
5446                         return 0;
5447                 }
5448                 break;
5449         default:
5450                 break;
5451         }
5452         return -1;
5453 }
5454
5455 /**
5456  * Parse flower single action retrieving the requested action attribute,
5457  * if found.
5458  *
5459  * @param arg
5460  *   flower action properties in the Netlink message received.
5461  * @param rta_type
5462  *   The backward sequence of rta_types, as written in the attribute table,
5463  *   we need to traverse in order to get to the requested object.
5464  * @param idx
5465  *   Current location in rta_type table.
5466  * @param[out] data
5467  *   Count statistics retrieved from the message query.
5468  *
5469  * @return
5470  *   0 if data was found and retrieved, -1 otherwise.
5471  */
5472 static int
5473 flow_tcf_nl_parse_one_action_and_get(struct rtattr *arg,
5474                                      uint16_t rta_type[], int idx, void *data)
5475 {
5476         int tca_act_max = flow_tcf_arr_val_max(rta_type, idx, TCA_ACT_STATS);
5477         struct rtattr *tb[tca_act_max + 1];
5478
5479         if (arg == NULL || idx < 0)
5480                 return -1;
5481         flow_tcf_nl_parse_rtattr(tb, tca_act_max,
5482                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5483         if (tb[TCA_ACT_KIND] == NULL)
5484                 return -1;
5485         switch (rta_type[idx]) {
5486         case TCA_ACT_STATS:
5487                 if (tb[TCA_ACT_STATS])
5488                         return flow_tcf_nl_action_stats_parse_and_get
5489                                         (tb[TCA_ACT_STATS],
5490                                          rta_type, --idx,
5491                                          (struct gnet_stats_basic *)data);
5492                 break;
5493         default:
5494                 break;
5495         }
5496         return -1;
5497 }
5498
5499 /**
5500  * Parse flower action section in the message retrieving the requested
5501  * attribute from the first action that provides it.
5502  *
5503  * @param opt
5504  *   flower section in the Netlink message received.
5505  * @param rta_type
5506  *   The backward sequence of rta_types, as written in the attribute table,
5507  *   we need to traverse in order to get to the requested object.
5508  * @param idx
5509  *   Current location in rta_type table.
5510  * @param[out] data
5511  *   data retrieved from the message query.
5512  *
5513  * @return
5514  *   0 if data was found and retrieved, -1 otherwise.
5515  */
5516 static int
5517 flow_tcf_nl_action_parse_and_get(struct rtattr *arg,
5518                                  uint16_t rta_type[], int idx, void *data)
5519 {
5520         struct rtattr *tb[TCA_ACT_MAX_PRIO + 1];
5521         int i;
5522
5523         if (arg == NULL || idx < 0)
5524                 return -1;
5525         flow_tcf_nl_parse_rtattr(tb, TCA_ACT_MAX_PRIO,
5526                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5527         switch (rta_type[idx]) {
5528         /*
5529          * flow counters are stored in the actions defined by the flow
5530          * and not in the flow itself, therefore we need to traverse the
5531          * flower chain of actions in search for them.
5532          *
5533          * Note that the index is not decremented here.
5534          */
5535         case TCA_ACT_STATS:
5536                 for (i = 0; i <= TCA_ACT_MAX_PRIO; i++) {
5537                         if (tb[i] &&
5538                         !flow_tcf_nl_parse_one_action_and_get(tb[i],
5539                                                               rta_type,
5540                                                               idx, data))
5541                                 return 0;
5542                 }
5543                 break;
5544         default:
5545                 break;
5546         }
5547         return -1;
5548 }
5549
5550 /**
5551  * Parse flower classifier options in the message, retrieving the requested
5552  * attribute if found.
5553  *
5554  * @param opt
5555  *   flower section in the Netlink message received.
5556  * @param rta_type
5557  *   The backward sequence of rta_types, as written in the attribute table,
5558  *   we need to traverse in order to get to the requested object.
5559  * @param idx
5560  *   Current location in rta_type table.
5561  * @param[out] data
5562  *   data retrieved from the message query.
5563  *
5564  * @return
5565  *   0 if data was found and retrieved, -1 otherwise.
5566  */
5567 static int
5568 flow_tcf_nl_opts_parse_and_get(struct rtattr *opt,
5569                                uint16_t rta_type[], int idx, void *data)
5570 {
5571         int tca_flower_max = flow_tcf_arr_val_max(rta_type, idx,
5572                                                   TCA_FLOWER_ACT);
5573         struct rtattr *tb[tca_flower_max + 1];
5574
5575         if (!opt || idx < 0)
5576                 return -1;
5577         flow_tcf_nl_parse_rtattr(tb, tca_flower_max,
5578                                  RTA_DATA(opt), RTA_PAYLOAD(opt));
5579         switch (rta_type[idx]) {
5580         case TCA_FLOWER_ACT:
5581                 if (tb[TCA_FLOWER_ACT])
5582                         return flow_tcf_nl_action_parse_and_get
5583                                                         (tb[TCA_FLOWER_ACT],
5584                                                          rta_type, --idx, data);
5585                 break;
5586         default:
5587                 break;
5588         }
5589         return -1;
5590 }
5591
5592 /**
5593  * Parse Netlink reply on filter query, retrieving the flow counters.
5594  *
5595  * @param nlh
5596  *   Message received from Netlink.
5597  * @param rta_type
5598  *   The backward sequence of rta_types, as written in the attribute table,
5599  *   we need to traverse in order to get to the requested object.
5600  * @param idx
5601  *   Current location in rta_type table.
5602  * @param[out] data
5603  *   data retrieved from the message query.
5604  *
5605  * @return
5606  *   0 if data was found and retrieved, -1 otherwise.
5607  */
5608 static int
5609 flow_tcf_nl_filter_parse_and_get(struct nlmsghdr *cnlh,
5610                                  uint16_t rta_type[], int idx, void *data)
5611 {
5612         struct nlmsghdr *nlh = cnlh;
5613         struct tcmsg *t = NLMSG_DATA(nlh);
5614         int len = nlh->nlmsg_len;
5615         int tca_max = flow_tcf_arr_val_max(rta_type, idx, TCA_OPTIONS);
5616         struct rtattr *tb[tca_max + 1];
5617
5618         if (idx < 0)
5619                 return -1;
5620         if (nlh->nlmsg_type != RTM_NEWTFILTER &&
5621             nlh->nlmsg_type != RTM_GETTFILTER &&
5622             nlh->nlmsg_type != RTM_DELTFILTER)
5623                 return -1;
5624         len -= NLMSG_LENGTH(sizeof(*t));
5625         if (len < 0)
5626                 return -1;
5627         flow_tcf_nl_parse_rtattr(tb, tca_max, TCA_RTA(t), len);
5628         /* Not a TC flower flow - bail out */
5629         if (!tb[TCA_KIND] ||
5630             strcmp(RTA_DATA(tb[TCA_KIND]), "flower"))
5631                 return -1;
5632         switch (rta_type[idx]) {
5633         case TCA_OPTIONS:
5634                 if (tb[TCA_OPTIONS])
5635                         return flow_tcf_nl_opts_parse_and_get(tb[TCA_OPTIONS],
5636                                                               rta_type,
5637                                                               --idx, data);
5638                 break;
5639         default:
5640                 break;
5641         }
5642         return -1;
5643 }
5644
5645 /**
5646  * A callback to parse Netlink reply on TC flower query.
5647  *
5648  * @param nlh
5649  *   Message received from Netlink.
5650  * @param[out] data
5651  *   Pointer to data area to be filled by the parsing routine.
5652  *   assumed to be a pointer to struct flow_tcf_stats_basic.
5653  *
5654  * @return
5655  *   MNL_CB_OK value.
5656  */
5657 static int
5658 flow_tcf_nl_message_get_stats_basic(const struct nlmsghdr *nlh, void *data)
5659 {
5660         /*
5661          * The backward sequence of rta_types to pass in order to get
5662          *  to the counters.
5663          */
5664         uint16_t rta_type[] = { TCA_STATS_BASIC, TCA_ACT_STATS,
5665                                 TCA_FLOWER_ACT, TCA_OPTIONS };
5666         struct flow_tcf_stats_basic *sb_data = data;
5667         union {
5668                 const struct nlmsghdr *c;
5669                 struct nlmsghdr *nc;
5670         } tnlh = { .c = nlh };
5671
5672         if (!flow_tcf_nl_filter_parse_and_get(tnlh.nc, rta_type,
5673                                               RTE_DIM(rta_type) - 1,
5674                                               (void *)&sb_data->counters))
5675                 sb_data->valid = true;
5676         return MNL_CB_OK;
5677 }
5678
5679 /**
5680  * Query a TC flower rule for its statistics via netlink.
5681  *
5682  * @param[in] dev
5683  *   Pointer to Ethernet device.
5684  * @param[in] flow
5685  *   Pointer to the sub flow.
5686  * @param[out] data
5687  *   data retrieved by the query.
5688  * @param[out] error
5689  *   Perform verbose error reporting if not NULL.
5690  *
5691  * @return
5692  *   0 on success, a negative errno value otherwise and rte_errno is set.
5693  */
5694 static int
5695 flow_tcf_query_count(struct rte_eth_dev *dev,
5696                           struct rte_flow *flow,
5697                           void *data,
5698                           struct rte_flow_error *error)
5699 {
5700         struct flow_tcf_stats_basic sb_data;
5701         struct rte_flow_query_count *qc = data;
5702         struct priv *priv = dev->data->dev_private;
5703         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5704         struct mnl_socket *nl = ctx->nl;
5705         struct mlx5_flow *dev_flow;
5706         struct nlmsghdr *nlh;
5707         uint32_t seq = priv->tcf_context->seq++;
5708         ssize_t ret;
5709         assert(qc);
5710
5711         memset(&sb_data, 0, sizeof(sb_data));
5712         dev_flow = LIST_FIRST(&flow->dev_flows);
5713         /* E-Switch flow can't be expanded. */
5714         assert(!LIST_NEXT(dev_flow, next));
5715         if (!dev_flow->flow->counter)
5716                 goto notsup_exit;
5717         nlh = dev_flow->tcf.nlh;
5718         nlh->nlmsg_type = RTM_GETTFILTER;
5719         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_ECHO;
5720         nlh->nlmsg_seq = seq;
5721         if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) == -1)
5722                 goto error_exit;
5723         do {
5724                 ret = mnl_socket_recvfrom(nl, ctx->buf, ctx->buf_size);
5725                 if (ret <= 0)
5726                         break;
5727                 ret = mnl_cb_run(ctx->buf, ret, seq,
5728                                  mnl_socket_get_portid(nl),
5729                                  flow_tcf_nl_message_get_stats_basic,
5730                                  (void *)&sb_data);
5731         } while (ret > 0);
5732         /* Return the delta from last reset. */
5733         if (sb_data.valid) {
5734                 /* Return the delta from last reset. */
5735                 qc->hits_set = 1;
5736                 qc->bytes_set = 1;
5737                 qc->hits = sb_data.counters.packets - flow->counter->hits;
5738                 qc->bytes = sb_data.counters.bytes - flow->counter->bytes;
5739                 if (qc->reset) {
5740                         flow->counter->hits = sb_data.counters.packets;
5741                         flow->counter->bytes = sb_data.counters.bytes;
5742                 }
5743                 return 0;
5744         }
5745         return rte_flow_error_set(error, EINVAL,
5746                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5747                                   NULL,
5748                                   "flow does not have counter");
5749 error_exit:
5750         return rte_flow_error_set
5751                         (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5752                          NULL, "netlink: failed to read flow rule counters");
5753 notsup_exit:
5754         return rte_flow_error_set
5755                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5756                          NULL, "counters are not available.");
5757 }
5758
5759 /**
5760  * Query a flow.
5761  *
5762  * @see rte_flow_query()
5763  * @see rte_flow_ops
5764  */
5765 static int
5766 flow_tcf_query(struct rte_eth_dev *dev,
5767                struct rte_flow *flow,
5768                const struct rte_flow_action *actions,
5769                void *data,
5770                struct rte_flow_error *error)
5771 {
5772         int ret = -EINVAL;
5773
5774         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5775                 switch (actions->type) {
5776                 case RTE_FLOW_ACTION_TYPE_VOID:
5777                         break;
5778                 case RTE_FLOW_ACTION_TYPE_COUNT:
5779                         ret = flow_tcf_query_count(dev, flow, data, error);
5780                         break;
5781                 default:
5782                         return rte_flow_error_set(error, ENOTSUP,
5783                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5784                                                   actions,
5785                                                   "action not supported");
5786                 }
5787         }
5788         return ret;
5789 }
5790
5791 const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops = {
5792         .validate = flow_tcf_validate,
5793         .prepare = flow_tcf_prepare,
5794         .translate = flow_tcf_translate,
5795         .apply = flow_tcf_apply,
5796         .remove = flow_tcf_remove,
5797         .destroy = flow_tcf_destroy,
5798         .query = flow_tcf_query,
5799 };
5800
5801 /**
5802  * Create and configure a libmnl socket for Netlink flow rules.
5803  *
5804  * @return
5805  *   A valid libmnl socket object pointer on success, NULL otherwise and
5806  *   rte_errno is set.
5807  */
5808 static struct mnl_socket *
5809 flow_tcf_mnl_socket_create(void)
5810 {
5811         struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
5812
5813         if (nl) {
5814                 mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
5815                                       sizeof(int));
5816                 if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
5817                         return nl;
5818         }
5819         rte_errno = errno;
5820         if (nl)
5821                 mnl_socket_close(nl);
5822         return NULL;
5823 }
5824
5825 /**
5826  * Destroy a libmnl socket.
5827  *
5828  * @param nl
5829  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5830  */
5831 static void
5832 flow_tcf_mnl_socket_destroy(struct mnl_socket *nl)
5833 {
5834         if (nl)
5835                 mnl_socket_close(nl);
5836 }
5837
5838 /**
5839  * Initialize ingress qdisc of a given network interface.
5840  *
5841  * @param ctx
5842  *   Pointer to tc-flower context to use.
5843  * @param ifindex
5844  *   Index of network interface to initialize.
5845  * @param[out] error
5846  *   Perform verbose error reporting if not NULL.
5847  *
5848  * @return
5849  *   0 on success, a negative errno value otherwise and rte_errno is set.
5850  */
5851 int
5852 mlx5_flow_tcf_init(struct mlx5_flow_tcf_context *ctx,
5853                    unsigned int ifindex, struct rte_flow_error *error)
5854 {
5855         struct nlmsghdr *nlh;
5856         struct tcmsg *tcm;
5857         alignas(struct nlmsghdr)
5858         uint8_t buf[mnl_nlmsg_size(sizeof(*tcm)) +
5859                     SZ_NLATTR_STRZ_OF("ingress") +
5860                     MNL_BUF_EXTRA_SPACE];
5861
5862         /* Destroy existing ingress qdisc and everything attached to it. */
5863         nlh = mnl_nlmsg_put_header(buf);
5864         nlh->nlmsg_type = RTM_DELQDISC;
5865         nlh->nlmsg_flags = NLM_F_REQUEST;
5866         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5867         tcm->tcm_family = AF_UNSPEC;
5868         tcm->tcm_ifindex = ifindex;
5869         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5870         tcm->tcm_parent = TC_H_INGRESS;
5871         assert(sizeof(buf) >= nlh->nlmsg_len);
5872         /* Ignore errors when qdisc is already absent. */
5873         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL) &&
5874             rte_errno != EINVAL && rte_errno != ENOENT)
5875                 return rte_flow_error_set(error, rte_errno,
5876                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5877                                           "netlink: failed to remove ingress"
5878                                           " qdisc");
5879         /* Create fresh ingress qdisc. */
5880         nlh = mnl_nlmsg_put_header(buf);
5881         nlh->nlmsg_type = RTM_NEWQDISC;
5882         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5883         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5884         tcm->tcm_family = AF_UNSPEC;
5885         tcm->tcm_ifindex = ifindex;
5886         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5887         tcm->tcm_parent = TC_H_INGRESS;
5888         mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
5889         assert(sizeof(buf) >= nlh->nlmsg_len);
5890         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL))
5891                 return rte_flow_error_set(error, rte_errno,
5892                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5893                                           "netlink: failed to create ingress"
5894                                           " qdisc");
5895         return 0;
5896 }
5897
5898 /**
5899  * Create libmnl context for Netlink flow rules.
5900  *
5901  * @return
5902  *   A valid libmnl socket object pointer on success, NULL otherwise and
5903  *   rte_errno is set.
5904  */
5905 struct mlx5_flow_tcf_context *
5906 mlx5_flow_tcf_context_create(void)
5907 {
5908         struct mlx5_flow_tcf_context *ctx = rte_zmalloc(__func__,
5909                                                         sizeof(*ctx),
5910                                                         sizeof(uint32_t));
5911         if (!ctx)
5912                 goto error;
5913         ctx->nl = flow_tcf_mnl_socket_create();
5914         if (!ctx->nl)
5915                 goto error;
5916         ctx->buf_size = MNL_SOCKET_BUFFER_SIZE;
5917         ctx->buf = rte_zmalloc(__func__,
5918                                ctx->buf_size, sizeof(uint32_t));
5919         if (!ctx->buf)
5920                 goto error;
5921         ctx->seq = random();
5922         return ctx;
5923 error:
5924         mlx5_flow_tcf_context_destroy(ctx);
5925         return NULL;
5926 }
5927
5928 /**
5929  * Destroy a libmnl context.
5930  *
5931  * @param ctx
5932  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5933  */
5934 void
5935 mlx5_flow_tcf_context_destroy(struct mlx5_flow_tcf_context *ctx)
5936 {
5937         if (!ctx)
5938                 return;
5939         flow_tcf_mnl_socket_destroy(ctx->nl);
5940         rte_free(ctx->buf);
5941         rte_free(ctx);
5942 }