drivers/net/mlx5/mlx5_flow_tcf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 6WIND S.A.
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <assert.h>
   7 #include <errno.h>
   8 #include <libmnl/libmnl.h>
   9 #include <linux/gen_stats.h>
  10 #include <linux/if_ether.h>
  11 #include <linux/netlink.h>
  12 #include <linux/pkt_cls.h>
  13 #include <linux/pkt_sched.h>
  14 #include <linux/rtnetlink.h>
  15 #include <linux/tc_act/tc_gact.h>
  16 #include <linux/tc_act/tc_mirred.h>
  17 #include <netinet/in.h>
  18 #include <stdalign.h>
  19 #include <stdbool.h>
  20 #include <stddef.h>
  21 #include <stdint.h>
  22 #include <stdlib.h>
  23 #include <sys/socket.h>
  24
  25 #include <rte_byteorder.h>
  26 #include <rte_errno.h>
  27 #include <rte_ether.h>
  28 #include <rte_flow.h>
  29 #include <rte_malloc.h>
  30 #include <rte_common.h>
  31
  32 #include "mlx5.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_autoconf.h"
  35
  36 #ifdef HAVE_TC_ACT_VLAN
  37
  38 #include <linux/tc_act/tc_vlan.h>
  39
  40 #else /* HAVE_TC_ACT_VLAN */
  41
  42 #define TCA_VLAN_ACT_POP 1
  43 #define TCA_VLAN_ACT_PUSH 2
  44 #define TCA_VLAN_ACT_MODIFY 3
  45 #define TCA_VLAN_PARMS 2
  46 #define TCA_VLAN_PUSH_VLAN_ID 3
  47 #define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
  48 #define TCA_VLAN_PAD 5
  49 #define TCA_VLAN_PUSH_VLAN_PRIORITY 6
  50
  51 struct tc_vlan {
  52         tc_gen;
  53         int v_action;
  54 };
  55
  56 #endif /* HAVE_TC_ACT_VLAN */
  57
  58 #ifdef HAVE_TC_ACT_PEDIT
  59
  60 #include <linux/tc_act/tc_pedit.h>
  61
  62 #else /* HAVE_TC_ACT_VLAN */
  63
  64 enum {
  65         TCA_PEDIT_UNSPEC,
  66         TCA_PEDIT_TM,
  67         TCA_PEDIT_PARMS,
  68         TCA_PEDIT_PAD,
  69         TCA_PEDIT_PARMS_EX,
  70         TCA_PEDIT_KEYS_EX,
  71         TCA_PEDIT_KEY_EX,
  72         __TCA_PEDIT_MAX
  73 };
  74
  75 enum {
  76         TCA_PEDIT_KEY_EX_HTYPE = 1,
  77         TCA_PEDIT_KEY_EX_CMD = 2,
  78         __TCA_PEDIT_KEY_EX_MAX
  79 };
  80
  81 enum pedit_header_type {
  82         TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK = 0,
  83         TCA_PEDIT_KEY_EX_HDR_TYPE_ETH = 1,
  84         TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 = 2,
  85         TCA_PEDIT_KEY_EX_HDR_TYPE_IP6 = 3,
  86         TCA_PEDIT_KEY_EX_HDR_TYPE_TCP = 4,
  87         TCA_PEDIT_KEY_EX_HDR_TYPE_UDP = 5,
  88         __PEDIT_HDR_TYPE_MAX,
  89 };
  90
  91 enum pedit_cmd {
  92         TCA_PEDIT_KEY_EX_CMD_SET = 0,
  93         TCA_PEDIT_KEY_EX_CMD_ADD = 1,
  94         __PEDIT_CMD_MAX,
  95 };
  96
  97 struct tc_pedit_key {
  98         __u32 mask; /* AND */
  99         __u32 val; /*XOR */
 100         __u32 off; /*offset */
 101         __u32 at;
 102         __u32 offmask;
 103         __u32 shift;
 104 };
 105
 106 __extension__
 107 struct tc_pedit_sel {
 108         tc_gen;
 109         unsigned char nkeys;
 110         unsigned char flags;
 111         struct tc_pedit_key keys[0];
 112 };
 113
 114 #endif /* HAVE_TC_ACT_VLAN */
 115
 116 #ifdef HAVE_TC_ACT_TUNNEL_KEY
 117
 118 #include <linux/tc_act/tc_tunnel_key.h>
 119
 120 #ifndef HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT
 121 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 122 #endif
 123
 124 #ifndef HAVE_TCA_TUNNEL_KEY_NO_CSUM
 125 #define TCA_TUNNEL_KEY_NO_CSUM 10
 126 #endif
 127
 128 #else /* HAVE_TC_ACT_TUNNEL_KEY */
 129
 130 #define TCA_ACT_TUNNEL_KEY 17
 131 #define TCA_TUNNEL_KEY_ACT_SET 1
 132 #define TCA_TUNNEL_KEY_ACT_RELEASE 2
 133 #define TCA_TUNNEL_KEY_PARMS 2
 134 #define TCA_TUNNEL_KEY_ENC_IPV4_SRC 3
 135 #define TCA_TUNNEL_KEY_ENC_IPV4_DST 4
 136 #define TCA_TUNNEL_KEY_ENC_IPV6_SRC 5
 137 #define TCA_TUNNEL_KEY_ENC_IPV6_DST 6
 138 #define TCA_TUNNEL_KEY_ENC_KEY_ID 7
 139 #define TCA_TUNNEL_KEY_ENC_DST_PORT 9
 140 #define TCA_TUNNEL_KEY_NO_CSUM 10
 141
 142 struct tc_tunnel_key {
 143         tc_gen;
 144         int t_action;
 145 };
 146
 147 #endif /* HAVE_TC_ACT_TUNNEL_KEY */
 148
 149 /* Normally found in linux/netlink.h. */
 150 #ifndef NETLINK_CAP_ACK
 151 #define NETLINK_CAP_ACK 10
 152 #endif
 153
 154 /* Normally found in linux/pkt_sched.h. */
 155 #ifndef TC_H_MIN_INGRESS
 156 #define TC_H_MIN_INGRESS 0xfff2u
 157 #endif
 158
 159 /* Normally found in linux/pkt_cls.h. */
 160 #ifndef TCA_CLS_FLAGS_SKIP_SW
 161 #define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
 162 #endif
 163 #ifndef TCA_CLS_FLAGS_IN_HW
 164 #define TCA_CLS_FLAGS_IN_HW (1 << 2)
 165 #endif
 166 #ifndef HAVE_TCA_CHAIN
 167 #define TCA_CHAIN 11
 168 #endif
 169 #ifndef HAVE_TCA_FLOWER_ACT
 170 #define TCA_FLOWER_ACT 3
 171 #endif
 172 #ifndef HAVE_TCA_FLOWER_FLAGS
 173 #define TCA_FLOWER_FLAGS 22
 174 #endif
 175 #ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
 176 #define TCA_FLOWER_KEY_ETH_TYPE 8
 177 #endif
 178 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
 179 #define TCA_FLOWER_KEY_ETH_DST 4
 180 #endif
 181 #ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
 182 #define TCA_FLOWER_KEY_ETH_DST_MASK 5
 183 #endif
 184 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
 185 #define TCA_FLOWER_KEY_ETH_SRC 6
 186 #endif
 187 #ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
 188 #define TCA_FLOWER_KEY_ETH_SRC_MASK 7
 189 #endif
 190 #ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
 191 #define TCA_FLOWER_KEY_IP_PROTO 9
 192 #endif
 193 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
 194 #define TCA_FLOWER_KEY_IPV4_SRC 10
 195 #endif
 196 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
 197 #define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
 198 #endif
 199 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
 200 #define TCA_FLOWER_KEY_IPV4_DST 12
 201 #endif
 202 #ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
 203 #define TCA_FLOWER_KEY_IPV4_DST_MASK 13
 204 #endif
 205 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
 206 #define TCA_FLOWER_KEY_IPV6_SRC 14
 207 #endif
 208 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
 209 #define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
 210 #endif
 211 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
 212 #define TCA_FLOWER_KEY_IPV6_DST 16
 213 #endif
 214 #ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
 215 #define TCA_FLOWER_KEY_IPV6_DST_MASK 17
 216 #endif
 217 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
 218 #define TCA_FLOWER_KEY_TCP_SRC 18
 219 #endif
 220 #ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
 221 #define TCA_FLOWER_KEY_TCP_SRC_MASK 35
 222 #endif
 223 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
 224 #define TCA_FLOWER_KEY_TCP_DST 19
 225 #endif
 226 #ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
 227 #define TCA_FLOWER_KEY_TCP_DST_MASK 36
 228 #endif
 229 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
 230 #define TCA_FLOWER_KEY_UDP_SRC 20
 231 #endif
 232 #ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
 233 #define TCA_FLOWER_KEY_UDP_SRC_MASK 37
 234 #endif
 235 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
 236 #define TCA_FLOWER_KEY_UDP_DST 21
 237 #endif
 238 #ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
 239 #define TCA_FLOWER_KEY_UDP_DST_MASK 38
 240 #endif
 241 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
 242 #define TCA_FLOWER_KEY_VLAN_ID 23
 243 #endif
 244 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
 245 #define TCA_FLOWER_KEY_VLAN_PRIO 24
 246 #endif
 247 #ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
 248 #define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
 249 #endif
 250 #ifndef HAVE_TCA_FLOWER_KEY_ENC_KEY_ID
 251 #define TCA_FLOWER_KEY_ENC_KEY_ID 26
 252 #endif
 253 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC
 254 #define TCA_FLOWER_KEY_ENC_IPV4_SRC 27
 255 #endif
 256 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK
 257 #define TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK 28
 258 #endif
 259 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST
 260 #define TCA_FLOWER_KEY_ENC_IPV4_DST 29
 261 #endif
 262 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK
 263 #define TCA_FLOWER_KEY_ENC_IPV4_DST_MASK 30
 264 #endif
 265 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC
 266 #define TCA_FLOWER_KEY_ENC_IPV6_SRC 31
 267 #endif
 268 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK
 269 #define TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK 32
 270 #endif
 271 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST
 272 #define TCA_FLOWER_KEY_ENC_IPV6_DST 33
 273 #endif
 274 #ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK
 275 #define TCA_FLOWER_KEY_ENC_IPV6_DST_MASK 34
 276 #endif
 277 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT
 278 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT 43
 279 #endif
 280 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK
 281 #define TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK 44
 282 #endif
 283 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT
 284 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT 45
 285 #endif
 286 #ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK
 287 #define TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK 46
 288 #endif
 289 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS
 290 #define TCA_FLOWER_KEY_TCP_FLAGS 71
 291 #endif
 292 #ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK
 293 #define TCA_FLOWER_KEY_TCP_FLAGS_MASK 72
 294 #endif
 295 #ifndef HAVE_TC_ACT_GOTO_CHAIN
 296 #define TC_ACT_GOTO_CHAIN 0x20000000
 297 #endif
 298
 299 #ifndef IPV6_ADDR_LEN
 300 #define IPV6_ADDR_LEN 16
 301 #endif
 302
 303 #ifndef IPV4_ADDR_LEN
 304 #define IPV4_ADDR_LEN 4
 305 #endif
 306
 307 #ifndef TP_PORT_LEN
 308 #define TP_PORT_LEN 2 /* Transport Port (UDP/TCP) Length */
 309 #endif
 310
 311 #ifndef TTL_LEN
 312 #define TTL_LEN 1
 313 #endif
 314
 315 #ifndef TCA_ACT_MAX_PRIO
 316 #define TCA_ACT_MAX_PRIO 32
 317 #endif
 318
 319 /** UDP port range of VXLAN devices created by driver. */
 320 #define MLX5_VXLAN_PORT_MIN 30000
 321 #define MLX5_VXLAN_PORT_MAX 60000
 322 #define MLX5_VXLAN_DEVICE_PFX "vmlx_"
 323
 324 /** Tunnel action type, used for @p type in header structure. */
 325 enum flow_tcf_tunact_type {
 326         FLOW_TCF_TUNACT_VXLAN_DECAP,
 327         FLOW_TCF_TUNACT_VXLAN_ENCAP,
 328 };
 329
 330 /** Flags used for @p mask in tunnel action encap descriptors. */
 331 #define FLOW_TCF_ENCAP_ETH_SRC (1u << 0)
 332 #define FLOW_TCF_ENCAP_ETH_DST (1u << 1)
 333 #define FLOW_TCF_ENCAP_IPV4_SRC (1u << 2)
 334 #define FLOW_TCF_ENCAP_IPV4_DST (1u << 3)
 335 #define FLOW_TCF_ENCAP_IPV6_SRC (1u << 4)
 336 #define FLOW_TCF_ENCAP_IPV6_DST (1u << 5)
 337 #define FLOW_TCF_ENCAP_UDP_SRC (1u << 6)
 338 #define FLOW_TCF_ENCAP_UDP_DST (1u << 7)
 339 #define FLOW_TCF_ENCAP_VXLAN_VNI (1u << 8)
 340
 341 /**
 342  * Structure for holding netlink context.
 343  * Note the size of the message buffer which is MNL_SOCKET_BUFFER_SIZE.
 344  * Using this (8KB) buffer size ensures that netlink messages will never be
 345  * truncated.
 346  */
 347 struct mlx5_flow_tcf_context {
 348         struct mnl_socket *nl; /* NETLINK_ROUTE libmnl socket. */
 349         uint32_t seq; /* Message sequence number. */
 350         uint32_t buf_size; /* Message buffer size. */
 351         uint8_t *buf; /* Message buffer. */
 352 };
 353
 354 /**
 355  * Neigh rule structure. The neigh rule is applied via Netlink to
 356  * outer tunnel iface in order to provide destination MAC address
 357  * for the VXLAN encapsultion. The neigh rule is implicitly related
 358  * to the Flow itself and can be shared by multiple Flows.
 359  */
 360 struct tcf_neigh_rule {
 361         LIST_ENTRY(tcf_neigh_rule) next;
 362         uint32_t refcnt;
 363         struct ether_addr eth;
 364         uint16_t mask;
 365         union {
 366                 struct {
 367                         rte_be32_t dst;
 368                 } ipv4;
 369                 struct {
 370                         uint8_t dst[IPV6_ADDR_LEN];
 371                 } ipv6;
 372         };
 373 };
 374
 375 /**
 376  * Local rule structure. The local rule is applied via Netlink to
 377  * outer tunnel iface in order to provide local and peer IP addresses
 378  * of the VXLAN tunnel for encapsulation. The local rule is implicitly
 379  * related to the Flow itself and can be shared by multiple Flows.
 380  */
 381 struct tcf_local_rule {
 382         LIST_ENTRY(tcf_local_rule) next;
 383         uint32_t refcnt;
 384         uint16_t mask;
 385         union {
 386                 struct {
 387                         rte_be32_t dst;
 388                         rte_be32_t src;
 389                 } ipv4;
 390                 struct {
 391                         uint8_t dst[IPV6_ADDR_LEN];
 392                         uint8_t src[IPV6_ADDR_LEN];
 393                 } ipv6;
 394         };
 395 };
 396
 397 /** VXLAN virtual netdev. */
 398 struct tcf_vtep {
 399         LIST_ENTRY(tcf_vtep) next;
 400         LIST_HEAD(, tcf_neigh_rule) neigh;
 401         LIST_HEAD(, tcf_local_rule) local;
 402         uint32_t refcnt;
 403         unsigned int ifindex; /**< Own interface index. */
 404         unsigned int ifouter; /**< Index of device attached to. */
 405         uint16_t port;
 406         uint8_t created;
 407 };
 408
 409 /** Tunnel descriptor header, common for all tunnel types. */
 410 struct flow_tcf_tunnel_hdr {
 411         uint32_t type; /**< Tunnel action type. */
 412         struct tcf_vtep *vtep; /**< Virtual tunnel endpoint device. */
 413         unsigned int ifindex_org; /**< Original dst/src interface */
 414         unsigned int *ifindex_ptr; /**< Interface ptr in message. */
 415 };
 416
 417 struct flow_tcf_vxlan_decap {
 418         struct flow_tcf_tunnel_hdr hdr;
 419         uint16_t udp_port;
 420 };
 421
 422 struct flow_tcf_vxlan_encap {
 423         struct flow_tcf_tunnel_hdr hdr;
 424         uint32_t mask;
 425         struct {
 426                 struct ether_addr dst;
 427                 struct ether_addr src;
 428         } eth;
 429         union {
 430                 struct {
 431                         rte_be32_t dst;
 432                         rte_be32_t src;
 433                 } ipv4;
 434                 struct {
 435                         uint8_t dst[IPV6_ADDR_LEN];
 436                         uint8_t src[IPV6_ADDR_LEN];
 437                 } ipv6;
 438         };
 439 struct {
 440                 rte_be16_t src;
 441                 rte_be16_t dst;
 442         } udp;
 443         struct {
 444                 uint8_t vni[3];
 445         } vxlan;
 446 };
 447
 448 /** Structure used when extracting the values of a flow counters
 449  * from a netlink message.
 450  */
 451 struct flow_tcf_stats_basic {
 452         bool valid;
 453         struct gnet_stats_basic counters;
 454 };
 455
 456 /** Empty masks for known item types. */
 457 static const union {
 458         struct rte_flow_item_port_id port_id;
 459         struct rte_flow_item_eth eth;
 460         struct rte_flow_item_vlan vlan;
 461         struct rte_flow_item_ipv4 ipv4;
 462         struct rte_flow_item_ipv6 ipv6;
 463         struct rte_flow_item_tcp tcp;
 464         struct rte_flow_item_udp udp;
 465         struct rte_flow_item_vxlan vxlan;
 466 } flow_tcf_mask_empty = {
 467         {0},
 468 };
 469
 470 /** Supported masks for known item types. */
 471 static const struct {
 472         struct rte_flow_item_port_id port_id;
 473         struct rte_flow_item_eth eth;
 474         struct rte_flow_item_vlan vlan;
 475         struct rte_flow_item_ipv4 ipv4;
 476         struct rte_flow_item_ipv6 ipv6;
 477         struct rte_flow_item_tcp tcp;
 478         struct rte_flow_item_udp udp;
 479         struct rte_flow_item_vxlan vxlan;
 480 } flow_tcf_mask_supported = {
 481         .port_id = {
 482                 .id = 0xffffffff,
 483         },
 484         .eth = {
 485                 .type = RTE_BE16(0xffff),
 486                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 487                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 488         },
 489         .vlan = {
 490                 /* PCP and VID only, no DEI. */
 491                 .tci = RTE_BE16(0xefff),
 492                 .inner_type = RTE_BE16(0xffff),
 493         },
 494         .ipv4.hdr = {
 495                 .next_proto_id = 0xff,
 496                 .src_addr = RTE_BE32(0xffffffff),
 497                 .dst_addr = RTE_BE32(0xffffffff),
 498         },
 499         .ipv6.hdr = {
 500                 .proto = 0xff,
 501                 .src_addr =
 502                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 503                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 504                 .dst_addr =
 505                         "\xff\xff\xff\xff\xff\xff\xff\xff"
 506                         "\xff\xff\xff\xff\xff\xff\xff\xff",
 507         },
 508         .tcp.hdr = {
 509                 .src_port = RTE_BE16(0xffff),
 510                 .dst_port = RTE_BE16(0xffff),
 511                 .tcp_flags = 0xff,
 512         },
 513         .udp.hdr = {
 514                 .src_port = RTE_BE16(0xffff),
 515                 .dst_port = RTE_BE16(0xffff),
 516         },
 517         .vxlan = {
 518                .vni = "\xff\xff\xff",
 519         },
 520 };
 521
 522 #define SZ_NLATTR_HDR MNL_ALIGN(sizeof(struct nlattr))
 523 #define SZ_NLATTR_NEST SZ_NLATTR_HDR
 524 #define SZ_NLATTR_DATA_OF(len) MNL_ALIGN(SZ_NLATTR_HDR + (len))
 525 #define SZ_NLATTR_TYPE_OF(typ) SZ_NLATTR_DATA_OF(sizeof(typ))
 526 #define SZ_NLATTR_STRZ_OF(str) SZ_NLATTR_DATA_OF(strlen(str) + 1)
 527
 528 #define PTOI_TABLE_SZ_MAX(dev) (mlx5_dev_to_port_id((dev)->device, NULL, 0) + 2)
 529
 530 /** DPDK port to network interface index (ifindex) conversion. */
 531 struct flow_tcf_ptoi {
 532         uint16_t port_id; /**< DPDK port ID. */
 533         unsigned int ifindex; /**< Network interface index. */
 534 };
 535
 536 /* Due to a limitation on driver/FW. */
 537 #define MLX5_TCF_GROUP_ID_MAX 3
 538
 539 /*
 540  * Due to a limitation on driver/FW, priority ranges from 1 to 16 in kernel.
 541  * Priority in rte_flow attribute starts from 0 and is added by 1 in
 542  * translation. This is subject to be changed to determine the max priority
 543  * based on trial-and-error like Verbs driver once the restriction is lifted or
 544  * the range is extended.
 545  */
 546 #define MLX5_TCF_GROUP_PRIORITY_MAX 15
 547
 548 #define MLX5_TCF_FATE_ACTIONS \
 549         (MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
 550          MLX5_FLOW_ACTION_JUMP)
 551
 552 #define MLX5_TCF_VLAN_ACTIONS \
 553         (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN | \
 554          MLX5_FLOW_ACTION_OF_SET_VLAN_VID | MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
 555
 556 #define MLX5_TCF_VXLAN_ACTIONS \
 557         (MLX5_FLOW_ACTION_VXLAN_ENCAP | MLX5_FLOW_ACTION_VXLAN_DECAP)
 558
 559 #define MLX5_TCF_PEDIT_ACTIONS \
 560         (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST | \
 561          MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST | \
 562          MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST | \
 563          MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL | \
 564          MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)
 565
 566 #define MLX5_TCF_CONFIG_ACTIONS \
 567         (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_JUMP | \
 568          MLX5_FLOW_ACTION_OF_PUSH_VLAN | MLX5_FLOW_ACTION_OF_SET_VLAN_VID | \
 569          MLX5_FLOW_ACTION_OF_SET_VLAN_PCP | \
 570          (MLX5_TCF_PEDIT_ACTIONS & ~MLX5_FLOW_ACTION_DEC_TTL))
 571
 572 #define MAX_PEDIT_KEYS 128
 573 #define SZ_PEDIT_KEY_VAL 4
 574
 575 #define NUM_OF_PEDIT_KEYS(sz) \
 576         (((sz) / SZ_PEDIT_KEY_VAL) + (((sz) % SZ_PEDIT_KEY_VAL) ? 1 : 0))
 577
 578 struct pedit_key_ex {
 579         enum pedit_header_type htype;
 580         enum pedit_cmd cmd;
 581 };
 582
 583 struct pedit_parser {
 584         struct tc_pedit_sel sel;
 585         struct tc_pedit_key keys[MAX_PEDIT_KEYS];
 586         struct pedit_key_ex keys_ex[MAX_PEDIT_KEYS];
 587 };
 588
 589 /**
 590  * Create space for using the implicitly created TC flow counter.
 591  *
 592  * @param[in] dev
 593  *   Pointer to the Ethernet device structure.
 594  *
 595  * @return
 596  *   A pointer to the counter data structure, NULL otherwise and
 597  *   rte_errno is set.
 598  */
 599 static struct mlx5_flow_counter *
 600 flow_tcf_counter_new(void)
 601 {
 602         struct mlx5_flow_counter *cnt;
 603
 604         /*
 605          * eswitch counter cannot be shared and its id is unknown.
 606          * currently returning all with id 0.
 607          * in the future maybe better to switch to unique numbers.
 608          */
 609         struct mlx5_flow_counter tmpl = {
 610                 .ref_cnt = 1,
 611         };
 612         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 613         if (!cnt) {
 614                 rte_errno = ENOMEM;
 615                 return NULL;
 616         }
 617         *cnt = tmpl;
 618         /* Implicit counter, do not add to list. */
 619         return cnt;
 620 }
 621
 622 /**
 623  * Set pedit key of MAC address
 624  *
 625  * @param[in] actions
 626  *   pointer to action specification
 627  * @param[in,out] p_parser
 628  *   pointer to pedit_parser
 629  */
 630 static void
 631 flow_tcf_pedit_key_set_mac(const struct rte_flow_action *actions,
 632                            struct pedit_parser *p_parser)
 633 {
 634         int idx = p_parser->sel.nkeys;
 635         uint32_t off = actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
 636                                         offsetof(struct ether_hdr, s_addr) :
 637                                         offsetof(struct ether_hdr, d_addr);
 638         const struct rte_flow_action_set_mac *conf =
 639                 (const struct rte_flow_action_set_mac *)actions->conf;
 640
 641         p_parser->keys[idx].off = off;
 642         p_parser->keys[idx].mask = ~UINT32_MAX;
 643         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 644         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 645         memcpy(&p_parser->keys[idx].val,
 646                 conf->mac_addr, SZ_PEDIT_KEY_VAL);
 647         idx++;
 648         p_parser->keys[idx].off = off + SZ_PEDIT_KEY_VAL;
 649         p_parser->keys[idx].mask = 0xFFFF0000;
 650         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
 651         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 652         memcpy(&p_parser->keys[idx].val,
 653                 conf->mac_addr + SZ_PEDIT_KEY_VAL,
 654                 ETHER_ADDR_LEN - SZ_PEDIT_KEY_VAL);
 655         p_parser->sel.nkeys = (++idx);
 656 }
 657
 658 /**
 659  * Set pedit key of decrease/set ttl
 660  *
 661  * @param[in] actions
 662  *   pointer to action specification
 663  * @param[in,out] p_parser
 664  *   pointer to pedit_parser
 665  * @param[in] item_flags
 666  *   flags of all items presented
 667  */
 668 static void
 669 flow_tcf_pedit_key_set_dec_ttl(const struct rte_flow_action *actions,
 670                                 struct pedit_parser *p_parser,
 671                                 uint64_t item_flags)
 672 {
 673         int idx = p_parser->sel.nkeys;
 674
 675         p_parser->keys[idx].mask = 0xFFFFFF00;
 676         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) {
 677                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 678                 p_parser->keys[idx].off =
 679                         offsetof(struct ipv4_hdr, time_to_live);
 680         }
 681         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6) {
 682                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 683                 p_parser->keys[idx].off =
 684                         offsetof(struct ipv6_hdr, hop_limits);
 685         }
 686         if (actions->type == RTE_FLOW_ACTION_TYPE_DEC_TTL) {
 687                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_ADD;
 688                 p_parser->keys[idx].val = 0x000000FF;
 689         } else {
 690                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 691                 p_parser->keys[idx].val =
 692                         (__u32)((const struct rte_flow_action_set_ttl *)
 693                          actions->conf)->ttl_value;
 694         }
 695         p_parser->sel.nkeys = (++idx);
 696 }
 697
 698 /**
 699  * Set pedit key of transport (TCP/UDP) port value
 700  *
 701  * @param[in] actions
 702  *   pointer to action specification
 703  * @param[in,out] p_parser
 704  *   pointer to pedit_parser
 705  * @param[in] item_flags
 706  *   flags of all items presented
 707  */
 708 static void
 709 flow_tcf_pedit_key_set_tp_port(const struct rte_flow_action *actions,
 710                                 struct pedit_parser *p_parser,
 711                                 uint64_t item_flags)
 712 {
 713         int idx = p_parser->sel.nkeys;
 714
 715         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 716                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_UDP;
 717         if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 718                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_TCP;
 719         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 720         /* offset of src/dst port is same for TCP and UDP */
 721         p_parser->keys[idx].off =
 722                 actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
 723                 offsetof(struct tcp_hdr, src_port) :
 724                 offsetof(struct tcp_hdr, dst_port);
 725         p_parser->keys[idx].mask = 0xFFFF0000;
 726         p_parser->keys[idx].val =
 727                 (__u32)((const struct rte_flow_action_set_tp *)
 728                                 actions->conf)->port;
 729         p_parser->sel.nkeys = (++idx);
 730 }
 731
 732 /**
 733  * Set pedit key of ipv6 address
 734  *
 735  * @param[in] actions
 736  *   pointer to action specification
 737  * @param[in,out] p_parser
 738  *   pointer to pedit_parser
 739  */
 740 static void
 741 flow_tcf_pedit_key_set_ipv6_addr(const struct rte_flow_action *actions,
 742                                  struct pedit_parser *p_parser)
 743 {
 744         int idx = p_parser->sel.nkeys;
 745         int keys = NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 746         int off_base =
 747                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
 748                 offsetof(struct ipv6_hdr, src_addr) :
 749                 offsetof(struct ipv6_hdr, dst_addr);
 750         const struct rte_flow_action_set_ipv6 *conf =
 751                 (const struct rte_flow_action_set_ipv6 *)actions->conf;
 752
 753         for (int i = 0; i < keys; i++, idx++) {
 754                 p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
 755                 p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 756                 p_parser->keys[idx].off = off_base + i * SZ_PEDIT_KEY_VAL;
 757                 p_parser->keys[idx].mask = ~UINT32_MAX;
 758                 memcpy(&p_parser->keys[idx].val,
 759                         conf->ipv6_addr + i *  SZ_PEDIT_KEY_VAL,
 760                         SZ_PEDIT_KEY_VAL);
 761         }
 762         p_parser->sel.nkeys += keys;
 763 }
 764
 765 /**
 766  * Set pedit key of ipv4 address
 767  *
 768  * @param[in] actions
 769  *   pointer to action specification
 770  * @param[in,out] p_parser
 771  *   pointer to pedit_parser
 772  */
 773 static void
 774 flow_tcf_pedit_key_set_ipv4_addr(const struct rte_flow_action *actions,
 775                                  struct pedit_parser *p_parser)
 776 {
 777         int idx = p_parser->sel.nkeys;
 778
 779         p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
 780         p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
 781         p_parser->keys[idx].off =
 782                 actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
 783                 offsetof(struct ipv4_hdr, src_addr) :
 784                 offsetof(struct ipv4_hdr, dst_addr);
 785         p_parser->keys[idx].mask = ~UINT32_MAX;
 786         p_parser->keys[idx].val =
 787                 ((const struct rte_flow_action_set_ipv4 *)
 788                  actions->conf)->ipv4_addr;
 789         p_parser->sel.nkeys = (++idx);
 790 }
 791
 792 /**
 793  * Create the pedit's na attribute in netlink message
 794  * on pre-allocate message buffer
 795  *
 796  * @param[in,out] nl
 797  *   pointer to pre-allocated netlink message buffer
 798  * @param[in,out] actions
 799  *   pointer to pointer of actions specification.
 800  * @param[in,out] action_flags
 801  *   pointer to actions flags
 802  * @param[in] item_flags
 803  *   flags of all item presented
 804  */
 805 static void
 806 flow_tcf_create_pedit_mnl_msg(struct nlmsghdr *nl,
 807                               const struct rte_flow_action **actions,
 808                               uint64_t item_flags)
 809 {
 810         struct pedit_parser p_parser;
 811         struct nlattr *na_act_options;
 812         struct nlattr *na_pedit_keys;
 813
 814         memset(&p_parser, 0, sizeof(p_parser));
 815         mnl_attr_put_strz(nl, TCA_ACT_KIND, "pedit");
 816         na_act_options = mnl_attr_nest_start(nl, TCA_ACT_OPTIONS);
 817         /* all modify header actions should be in one tc-pedit action */
 818         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 819                 switch ((*actions)->type) {
 820                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 821                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 822                         flow_tcf_pedit_key_set_ipv4_addr(*actions, &p_parser);
 823                         break;
 824                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 825                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 826                         flow_tcf_pedit_key_set_ipv6_addr(*actions, &p_parser);
 827                         break;
 828                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 829                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 830                         flow_tcf_pedit_key_set_tp_port(*actions,
 831                                                         &p_parser, item_flags);
 832                         break;
 833                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 834                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 835                         flow_tcf_pedit_key_set_dec_ttl(*actions,
 836                                                         &p_parser, item_flags);
 837                         break;
 838                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 839                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 840                         flow_tcf_pedit_key_set_mac(*actions, &p_parser);
 841                         break;
 842                 default:
 843                         goto pedit_mnl_msg_done;
 844                 }
 845         }
 846 pedit_mnl_msg_done:
 847         p_parser.sel.action = TC_ACT_PIPE;
 848         mnl_attr_put(nl, TCA_PEDIT_PARMS_EX,
 849                      sizeof(p_parser.sel) +
 850                      p_parser.sel.nkeys * sizeof(struct tc_pedit_key),
 851                      &p_parser);
 852         na_pedit_keys =
 853                 mnl_attr_nest_start(nl, TCA_PEDIT_KEYS_EX | NLA_F_NESTED);
 854         for (int i = 0; i < p_parser.sel.nkeys; i++) {
 855                 struct nlattr *na_pedit_key =
 856                         mnl_attr_nest_start(nl,
 857                                             TCA_PEDIT_KEY_EX | NLA_F_NESTED);
 858                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_HTYPE,
 859                                  p_parser.keys_ex[i].htype);
 860                 mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_CMD,
 861                                  p_parser.keys_ex[i].cmd);
 862                 mnl_attr_nest_end(nl, na_pedit_key);
 863         }
 864         mnl_attr_nest_end(nl, na_pedit_keys);
 865         mnl_attr_nest_end(nl, na_act_options);
 866         (*actions)--;
 867 }
 868
 869 /**
 870  * Calculate max memory size of one TC-pedit actions.
 871  * One TC-pedit action can contain set of keys each defining
 872  * a rewrite element (rte_flow action)
 873  *
 874  * @param[in,out] actions
 875  *   actions specification.
 876  * @param[in,out] action_flags
 877  *   actions flags
 878  * @param[in,out] size
 879  *   accumulated size
 880  * @return
 881  *   Max memory size of one TC-pedit action
 882  */
 883 static int
 884 flow_tcf_get_pedit_actions_size(const struct rte_flow_action **actions,
 885                                 uint64_t *action_flags)
 886 {
 887         int pedit_size = 0;
 888         int keys = 0;
 889         uint64_t flags = 0;
 890
 891         pedit_size += SZ_NLATTR_NEST + /* na_act_index. */
 892                       SZ_NLATTR_STRZ_OF("pedit") +
 893                       SZ_NLATTR_NEST; /* TCA_ACT_OPTIONS. */
 894         for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
 895                 switch ((*actions)->type) {
 896                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
 897                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 898                         flags |= MLX5_FLOW_ACTION_SET_IPV4_SRC;
 899                         break;
 900                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
 901                         keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
 902                         flags |= MLX5_FLOW_ACTION_SET_IPV4_DST;
 903                         break;
 904                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
 905                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 906                         flags |= MLX5_FLOW_ACTION_SET_IPV6_SRC;
 907                         break;
 908                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
 909                         keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
 910                         flags |= MLX5_FLOW_ACTION_SET_IPV6_DST;
 911                         break;
 912                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
 913                         /* TCP is as same as UDP */
 914                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 915                         flags |= MLX5_FLOW_ACTION_SET_TP_SRC;
 916                         break;
 917                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
 918                         /* TCP is as same as UDP */
 919                         keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
 920                         flags |= MLX5_FLOW_ACTION_SET_TP_DST;
 921                         break;
 922                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
 923                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 924                         flags |= MLX5_FLOW_ACTION_SET_TTL;
 925                         break;
 926                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
 927                         keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
 928                         flags |= MLX5_FLOW_ACTION_DEC_TTL;
 929                         break;
 930                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
 931                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 932                         flags |= MLX5_FLOW_ACTION_SET_MAC_SRC;
 933                         break;
 934                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
 935                         keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
 936                         flags |= MLX5_FLOW_ACTION_SET_MAC_DST;
 937                         break;
 938                 default:
 939                         goto get_pedit_action_size_done;
 940                 }
 941         }
 942 get_pedit_action_size_done:
 943         /* TCA_PEDIT_PARAMS_EX */
 944         pedit_size +=
 945                 SZ_NLATTR_DATA_OF(sizeof(struct tc_pedit_sel) +
 946                                   keys * sizeof(struct tc_pedit_key));
 947         pedit_size += SZ_NLATTR_NEST; /* TCA_PEDIT_KEYS */
 948         pedit_size += keys *
 949                       /* TCA_PEDIT_KEY_EX + HTYPE + CMD */
 950                       (SZ_NLATTR_NEST + SZ_NLATTR_DATA_OF(2) +
 951                        SZ_NLATTR_DATA_OF(2));
 952         (*action_flags) |= flags;
 953         (*actions)--;
 954         return pedit_size;
 955 }
 956
 957 /**
 958  * Retrieve mask for pattern item.
 959  *
 960  * This function does basic sanity checks on a pattern item in order to
 961  * return the most appropriate mask for it.
 962  *
 963  * @param[in] item
 964  *   Item specification.
 965  * @param[in] mask_default
 966  *   Default mask for pattern item as specified by the flow API.
 967  * @param[in] mask_supported
 968  *   Mask fields supported by the implementation.
 969  * @param[in] mask_empty
 970  *   Empty mask to return when there is no specification.
 971  * @param[out] error
 972  *   Perform verbose error reporting if not NULL.
 973  *
 974  * @return
 975  *   Either @p item->mask or one of the mask parameters on success, NULL
 976  *   otherwise and rte_errno is set.
 977  */
 978 static const void *
 979 flow_tcf_item_mask(const struct rte_flow_item *item, const void *mask_default,
 980                    const void *mask_supported, const void *mask_empty,
 981                    size_t mask_size, struct rte_flow_error *error)
 982 {
 983         const uint8_t *mask;
 984         size_t i;
 985
 986         /* item->last and item->mask cannot exist without item->spec. */
 987         if (!item->spec && (item->mask || item->last)) {
 988                 rte_flow_error_set(error, EINVAL,
 989                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 990                                    "\"mask\" or \"last\" field provided without"
 991                                    " a corresponding \"spec\"");
 992                 return NULL;
 993         }
 994         /* No spec, no mask, no problem. */
 995         if (!item->spec)
 996                 return mask_empty;
 997         mask = item->mask ? item->mask : mask_default;
 998         assert(mask);
 999         /*
1000          * Single-pass check to make sure that:
1001          * - Mask is supported, no bits are set outside mask_supported.
1002          * - Both item->spec and item->last are included in mask.
1003          */
1004         for (i = 0; i != mask_size; ++i) {
1005                 if (!mask[i])
1006                         continue;
1007                 if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
1008                     ((const uint8_t *)mask_supported)[i]) {
1009                         rte_flow_error_set(error, ENOTSUP,
1010                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1011                                            "unsupported field found"
1012                                            " in \"mask\"");
1013                         return NULL;
1014                 }
1015                 if (item->last &&
1016                     (((const uint8_t *)item->spec)[i] & mask[i]) !=
1017                     (((const uint8_t *)item->last)[i] & mask[i])) {
1018                         rte_flow_error_set(error, EINVAL,
1019                                            RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1020                                            item->last,
1021                                            "range between \"spec\" and \"last\""
1022                                            " not comprised in \"mask\"");
1023                         return NULL;
1024                 }
1025         }
1026         return mask;
1027 }
1028
1029 /**
1030  * Build a conversion table between port ID and ifindex.
1031  *
1032  * @param[in] dev
1033  *   Pointer to Ethernet device.
1034  * @param[out] ptoi
1035  *   Pointer to ptoi table.
1036  * @param[in] len
1037  *   Size of ptoi table provided.
1038  *
1039  * @return
1040  *   Size of ptoi table filled.
1041  */
1042 static unsigned int
1043 flow_tcf_build_ptoi_table(struct rte_eth_dev *dev, struct flow_tcf_ptoi *ptoi,
1044                           unsigned int len)
1045 {
1046         unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
1047         uint16_t port_id[n + 1];
1048         unsigned int i;
1049         unsigned int own = 0;
1050
1051         /* At least one port is needed when no switch domain is present. */
1052         if (!n) {
1053                 n = 1;
1054                 port_id[0] = dev->data->port_id;
1055         } else {
1056                 n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
1057         }
1058         if (n > len)
1059                 return 0;
1060         for (i = 0; i != n; ++i) {
1061                 struct rte_eth_dev_info dev_info;
1062
1063                 rte_eth_dev_info_get(port_id[i], &dev_info);
1064                 if (port_id[i] == dev->data->port_id)
1065                         own = i;
1066                 ptoi[i].port_id = port_id[i];
1067                 ptoi[i].ifindex = dev_info.if_index;
1068         }
1069         /* Ensure first entry of ptoi[] is the current device. */
1070         if (own) {
1071                 ptoi[n] = ptoi[0];
1072                 ptoi[0] = ptoi[own];
1073                 ptoi[own] = ptoi[n];
1074         }
1075         /* An entry with zero ifindex terminates ptoi[]. */
1076         ptoi[n].port_id = 0;
1077         ptoi[n].ifindex = 0;
1078         return n;
1079 }
1080
1081 /**
1082  * Verify the @p attr will be correctly understood by the E-switch.
1083  *
1084  * @param[in] attr
1085  *   Pointer to flow attributes
1086  * @param[out] error
1087  *   Pointer to error structure.
1088  *
1089  * @return
1090  *   0 on success, a negative errno value otherwise and rte_errno is set.
1091  */
1092 static int
1093 flow_tcf_validate_attributes(const struct rte_flow_attr *attr,
1094                              struct rte_flow_error *error)
1095 {
1096         /*
1097          * Supported attributes: groups, some priorities and ingress only.
1098          * group is supported only if kernel supports chain. Don't care about
1099          * transfer as it is the caller's problem.
1100          */
1101         if (attr->group > MLX5_TCF_GROUP_ID_MAX)
1102                 return rte_flow_error_set(error, ENOTSUP,
1103                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
1104                                           "group ID larger than "
1105                                           RTE_STR(MLX5_TCF_GROUP_ID_MAX)
1106                                           " isn't supported");
1107         else if (attr->priority > MLX5_TCF_GROUP_PRIORITY_MAX)
1108                 return rte_flow_error_set(error, ENOTSUP,
1109                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1110                                           attr,
1111                                           "priority more than "
1112                                           RTE_STR(MLX5_TCF_GROUP_PRIORITY_MAX)
1113                                           " is not supported");
1114         if (!attr->ingress)
1115                 return rte_flow_error_set(error, EINVAL,
1116                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1117                                           attr, "only ingress is supported");
1118         if (attr->egress)
1119                 return rte_flow_error_set(error, ENOTSUP,
1120                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1121                                           attr, "egress is not supported");
1122         return 0;
1123 }
1124
1125 /**
1126  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_ETH item for E-Switch.
1127  * The routine checks the L2 fields to be used in encapsulation header.
1128  *
1129  * @param[in] item
1130  *   Pointer to the item structure.
1131  * @param[out] error
1132  *   Pointer to the error structure.
1133  *
1134  * @return
1135  *   0 on success, a negative errno value otherwise and rte_errno is set.
1136  **/
1137 static int
1138 flow_tcf_validate_vxlan_encap_eth(const struct rte_flow_item *item,
1139                                   struct rte_flow_error *error)
1140 {
1141         const struct rte_flow_item_eth *spec = item->spec;
1142         const struct rte_flow_item_eth *mask = item->mask;
1143
1144         if (!spec) {
1145                 /*
1146                  * Specification for L2 addresses can be empty
1147                  * because these ones are optional and not
1148                  * required directly by tc rule. Kernel tries
1149                  * to resolve these ones on its own
1150                  */
1151                 return 0;
1152         }
1153         if (!mask) {
1154                 /* If mask is not specified use the default one. */
1155                 mask = &rte_flow_item_eth_mask;
1156         }
1157         if (memcmp(&mask->dst,
1158                    &flow_tcf_mask_empty.eth.dst,
1159                    sizeof(flow_tcf_mask_empty.eth.dst))) {
1160                 if (memcmp(&mask->dst,
1161                            &rte_flow_item_eth_mask.dst,
1162                            sizeof(rte_flow_item_eth_mask.dst)))
1163                         return rte_flow_error_set
1164                                 (error, ENOTSUP,
1165                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1166                                  "no support for partial mask on"
1167                                  " \"eth.dst\" field");
1168         }
1169         if (memcmp(&mask->src,
1170                    &flow_tcf_mask_empty.eth.src,
1171                    sizeof(flow_tcf_mask_empty.eth.src))) {
1172                 if (memcmp(&mask->src,
1173                            &rte_flow_item_eth_mask.src,
1174                            sizeof(rte_flow_item_eth_mask.src)))
1175                         return rte_flow_error_set
1176                                 (error, ENOTSUP,
1177                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1178                                  "no support for partial mask on"
1179                                  " \"eth.src\" field");
1180         }
1181         if (mask->type != RTE_BE16(0x0000)) {
1182                 if (mask->type != RTE_BE16(0xffff))
1183                         return rte_flow_error_set
1184                                 (error, ENOTSUP,
1185                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1186                                  "no support for partial mask on"
1187                                  " \"eth.type\" field");
1188                 DRV_LOG(WARNING,
1189                         "outer ethernet type field"
1190                         " cannot be forced for vxlan"
1191                         " encapsulation, parameter ignored");
1192         }
1193         return 0;
1194 }
1195
1196 /**
1197  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV4 item for E-Switch.
1198  * The routine checks the IPv4 fields to be used in encapsulation header.
1199  *
1200  * @param[in] item
1201  *   Pointer to the item structure.
1202  * @param[out] error
1203  *   Pointer to the error structure.
1204  *
1205  * @return
1206  *   0 on success, a negative errno value otherwise and rte_errno is set.
1207  **/
1208 static int
1209 flow_tcf_validate_vxlan_encap_ipv4(const struct rte_flow_item *item,
1210                                    struct rte_flow_error *error)
1211 {
1212         const struct rte_flow_item_ipv4 *spec = item->spec;
1213         const struct rte_flow_item_ipv4 *mask = item->mask;
1214
1215         if (!spec) {
1216                 /*
1217                  * Specification for IP addresses cannot be empty
1218                  * because it is required by tunnel_key parameter.
1219                  */
1220                 return rte_flow_error_set(error, EINVAL,
1221                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1222                                           "NULL outer ipv4 address"
1223                                           " specification for vxlan"
1224                                           " encapsulation");
1225         }
1226         if (!mask)
1227                 mask = &rte_flow_item_ipv4_mask;
1228         if (mask->hdr.dst_addr != RTE_BE32(0x00000000)) {
1229                 if (mask->hdr.dst_addr != RTE_BE32(0xffffffff))
1230                         return rte_flow_error_set
1231                                 (error, ENOTSUP,
1232                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1233                                  "no support for partial mask on"
1234                                  " \"ipv4.hdr.dst_addr\" field"
1235                                  " for vxlan encapsulation");
1236                 /* More IPv4 address validations can be put here. */
1237         } else {
1238                 /*
1239                  * Kernel uses the destination IP address to determine
1240                  * the routing path and obtain the MAC destination
1241                  * address, so IP destination address must be
1242                  * specified in the tc rule.
1243                  */
1244                 return rte_flow_error_set(error, EINVAL,
1245                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1246                                           "outer ipv4 destination address"
1247                                           " must be specified for"
1248                                           " vxlan encapsulation");
1249         }
1250         if (mask->hdr.src_addr != RTE_BE32(0x00000000)) {
1251                 if (mask->hdr.src_addr != RTE_BE32(0xffffffff))
1252                         return rte_flow_error_set
1253                                 (error, ENOTSUP,
1254                                  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1255                                  "no support for partial mask on"
1256                                  " \"ipv4.hdr.src_addr\" field"
1257                                  " for vxlan encapsulation");
1258                 /* More IPv4 address validations can be put here. */
1259         } else {
1260                 /*
1261                  * Kernel uses the source IP address to select the
1262                  * interface for egress encapsulated traffic, so
1263                  * it must be specified in the tc rule.
1264                  */
1265                 return rte_flow_error_set(error, EINVAL,
1266                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1267                                           "outer ipv4 source address"
1268                                           " must be specified for"
1269                                           " vxlan encapsulation");
1270         }
1271         return 0;
1272 }
1273
1274 /**
1275  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_IPV6 item for E-Switch.
1276  * The routine checks the IPv6 fields to be used in encapsulation header.
1277  *
1278  * @param[in] item
1279  *   Pointer to the item structure.
1280  * @param[out] error
1281  *   Pointer to the error structure.
1282  *
1283  * @return
1284  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1285  **/
1286 static int
1287 flow_tcf_validate_vxlan_encap_ipv6(const struct rte_flow_item *item,
1288                                    struct rte_flow_error *error)
1289 {
1290         const struct rte_flow_item_ipv6 *spec = item->spec;
1291         const struct rte_flow_item_ipv6 *mask = item->mask;
1292
1293         if (!spec) {
1294                 /*
1295                  * Specification for IP addresses cannot be empty
1296                  * because it is required by tunnel_key parameter.
1297                  */
1298                 return rte_flow_error_set(error, EINVAL,
1299                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1300                                           "NULL outer ipv6 address"
1301                                           " specification for"
1302                                           " vxlan encapsulation");
1303         }
1304         if (!mask)
1305                 mask = &rte_flow_item_ipv6_mask;
1306         if (memcmp(&mask->hdr.dst_addr,
1307                    &flow_tcf_mask_empty.ipv6.hdr.dst_addr,
1308                    IPV6_ADDR_LEN)) {
1309                 if (memcmp(&mask->hdr.dst_addr,
1310                            &rte_flow_item_ipv6_mask.hdr.dst_addr,
1311                            IPV6_ADDR_LEN))
1312                         return rte_flow_error_set
1313                                         (error, ENOTSUP,
1314                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1315                                          "no support for partial mask on"
1316                                          " \"ipv6.hdr.dst_addr\" field"
1317                                          " for vxlan encapsulation");
1318                 /* More IPv6 address validations can be put here. */
1319         } else {
1320                 /*
1321                  * Kernel uses the destination IP address to determine
1322                  * the routing path and obtain the MAC destination
1323                  * address (heigh or gate), so IP destination address
1324                  * must be specified within the tc rule.
1325                  */
1326                 return rte_flow_error_set(error, EINVAL,
1327                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1328                                           "outer ipv6 destination address"
1329                                           " must be specified for"
1330                                           " vxlan encapsulation");
1331         }
1332         if (memcmp(&mask->hdr.src_addr,
1333                    &flow_tcf_mask_empty.ipv6.hdr.src_addr,
1334                    IPV6_ADDR_LEN)) {
1335                 if (memcmp(&mask->hdr.src_addr,
1336                            &rte_flow_item_ipv6_mask.hdr.src_addr,
1337                            IPV6_ADDR_LEN))
1338                         return rte_flow_error_set
1339                                         (error, ENOTSUP,
1340                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1341                                          "no support for partial mask on"
1342                                          " \"ipv6.hdr.src_addr\" field"
1343                                          " for vxlan encapsulation");
1344                 /* More L3 address validation can be put here. */
1345         } else {
1346                 /*
1347                  * Kernel uses the source IP address to select the
1348                  * interface for egress encapsulated traffic, so
1349                  * it must be specified in the tc rule.
1350                  */
1351                 return rte_flow_error_set(error, EINVAL,
1352                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1353                                           "outer L3 source address"
1354                                           " must be specified for"
1355                                           " vxlan encapsulation");
1356         }
1357         return 0;
1358 }
1359
1360 /**
1361  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_UDP item for E-Switch.
1362  * The routine checks the UDP fields to be used in encapsulation header.
1363  *
1364  * @param[in] item
1365  *   Pointer to the item structure.
1366  * @param[out] error
1367  *   Pointer to the error structure.
1368  *
1369  * @return
1370  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1371  **/
1372 static int
1373 flow_tcf_validate_vxlan_encap_udp(const struct rte_flow_item *item,
1374                                   struct rte_flow_error *error)
1375 {
1376         const struct rte_flow_item_udp *spec = item->spec;
1377         const struct rte_flow_item_udp *mask = item->mask;
1378
1379         if (!spec) {
1380                 /*
1381                  * Specification for UDP ports cannot be empty
1382                  * because it is required by tunnel_key parameter.
1383                  */
1384                 return rte_flow_error_set(error, EINVAL,
1385                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1386                                           "NULL UDP port specification "
1387                                           " for vxlan encapsulation");
1388         }
1389         if (!mask)
1390                 mask = &rte_flow_item_udp_mask;
1391         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1392                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1393                         return rte_flow_error_set
1394                                         (error, ENOTSUP,
1395                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1396                                          "no support for partial mask on"
1397                                          " \"udp.hdr.dst_port\" field"
1398                                          " for vxlan encapsulation");
1399                 if (!spec->hdr.dst_port)
1400                         return rte_flow_error_set
1401                                         (error, EINVAL,
1402                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
1403                                          "outer UDP remote port cannot be"
1404                                          " 0 for vxlan encapsulation");
1405         } else {
1406                 return rte_flow_error_set(error, EINVAL,
1407                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1408                                           "outer UDP remote port"
1409                                           " must be specified for"
1410                                           " vxlan encapsulation");
1411         }
1412         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1413                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1414                         return rte_flow_error_set
1415                                         (error, ENOTSUP,
1416                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1417                                          "no support for partial mask on"
1418                                          " \"udp.hdr.src_port\" field"
1419                                          " for vxlan encapsulation");
1420                 DRV_LOG(WARNING,
1421                         "outer UDP source port cannot be"
1422                         " forced for vxlan encapsulation,"
1423                         " parameter ignored");
1424         }
1425         return 0;
1426 }
1427
1428 /**
1429  * Validate VXLAN_ENCAP action RTE_FLOW_ITEM_TYPE_VXLAN item for E-Switch.
1430  * The routine checks the VNIP fields to be used in encapsulation header.
1431  *
1432  * @param[in] item
1433  *   Pointer to the item structure.
1434  * @param[out] error
1435  *   Pointer to the error structure.
1436  *
1437  * @return
1438  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1439  **/
1440 static int
1441 flow_tcf_validate_vxlan_encap_vni(const struct rte_flow_item *item,
1442                                   struct rte_flow_error *error)
1443 {
1444         const struct rte_flow_item_vxlan *spec = item->spec;
1445         const struct rte_flow_item_vxlan *mask = item->mask;
1446
1447         if (!spec) {
1448                 /* Outer VNI is required by tunnel_key parameter. */
1449                 return rte_flow_error_set(error, EINVAL,
1450                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1451                                           "NULL VNI specification"
1452                                           " for vxlan encapsulation");
1453         }
1454         if (!mask)
1455                 mask = &rte_flow_item_vxlan_mask;
1456         if (!mask->vni[0] && !mask->vni[1] && !mask->vni[2])
1457                 return rte_flow_error_set(error, EINVAL,
1458                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1459                                           "outer VNI must be specified "
1460                                           "for vxlan encapsulation");
1461         if (mask->vni[0] != 0xff ||
1462             mask->vni[1] != 0xff ||
1463             mask->vni[2] != 0xff)
1464                 return rte_flow_error_set(error, ENOTSUP,
1465                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1466                                           "no support for partial mask on"
1467                                           " \"vxlan.vni\" field");
1468
1469         if (!spec->vni[0] && !spec->vni[1] && !spec->vni[2])
1470                 return rte_flow_error_set(error, EINVAL,
1471                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1472                                           "vxlan vni cannot be 0");
1473         return 0;
1474 }
1475
1476 /**
1477  * Validate VXLAN_ENCAP action item list for E-Switch.
1478  * The routine checks items to be used in encapsulation header.
1479  *
1480  * @param[in] action
1481  *   Pointer to the VXLAN_ENCAP action structure.
1482  * @param[out] error
1483  *   Pointer to the error structure.
1484  *
1485  * @return
1486  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1487  **/
1488 static int
1489 flow_tcf_validate_vxlan_encap(const struct rte_flow_action *action,
1490                               struct rte_flow_error *error)
1491 {
1492         const struct rte_flow_item *items;
1493         int ret;
1494         uint32_t item_flags = 0;
1495
1496         if (!action->conf)
1497                 return rte_flow_error_set(error, EINVAL,
1498                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1499                                           "Missing vxlan tunnel"
1500                                           " action configuration");
1501         items = ((const struct rte_flow_action_vxlan_encap *)
1502                                         action->conf)->definition;
1503         if (!items)
1504                 return rte_flow_error_set(error, EINVAL,
1505                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1506                                           "Missing vxlan tunnel"
1507                                           " encapsulation parameters");
1508         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1509                 switch (items->type) {
1510                 case RTE_FLOW_ITEM_TYPE_VOID:
1511                         break;
1512                 case RTE_FLOW_ITEM_TYPE_ETH:
1513                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1514                                                           error);
1515                         if (ret < 0)
1516                                 return ret;
1517                         ret = flow_tcf_validate_vxlan_encap_eth(items, error);
1518                         if (ret < 0)
1519                                 return ret;
1520                         item_flags |= MLX5_FLOW_LAYER_OUTER_L2;
1521                         break;
1522                 break;
1523                 case RTE_FLOW_ITEM_TYPE_IPV4:
1524                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1525                                                            error);
1526                         if (ret < 0)
1527                                 return ret;
1528                         ret = flow_tcf_validate_vxlan_encap_ipv4(items, error);
1529                         if (ret < 0)
1530                                 return ret;
1531                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1532                         break;
1533                 case RTE_FLOW_ITEM_TYPE_IPV6:
1534                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1535                                                            error);
1536                         if (ret < 0)
1537                                 return ret;
1538                         ret = flow_tcf_validate_vxlan_encap_ipv6(items, error);
1539                         if (ret < 0)
1540                                 return ret;
1541                         item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1542                         break;
1543                 case RTE_FLOW_ITEM_TYPE_UDP:
1544                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1545                                                            0xFF, error);
1546                         if (ret < 0)
1547                                 return ret;
1548                         ret = flow_tcf_validate_vxlan_encap_udp(items, error);
1549                         if (ret < 0)
1550                                 return ret;
1551                         item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
1552                         break;
1553                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1554                         ret = mlx5_flow_validate_item_vxlan(items,
1555                                                             item_flags, error);
1556                         if (ret < 0)
1557                                 return ret;
1558                         ret = flow_tcf_validate_vxlan_encap_vni(items, error);
1559                         if (ret < 0)
1560                                 return ret;
1561                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1562                         break;
1563                 default:
1564                         return rte_flow_error_set
1565                                         (error, ENOTSUP,
1566                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1567                                          "vxlan encap item not supported");
1568                 }
1569         }
1570         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
1571                 return rte_flow_error_set(error, EINVAL,
1572                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1573                                           "no outer IP layer found"
1574                                           " for vxlan encapsulation");
1575         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1576                 return rte_flow_error_set(error, EINVAL,
1577                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1578                                           "no outer UDP layer found"
1579                                           " for vxlan encapsulation");
1580         if (!(item_flags & MLX5_FLOW_LAYER_VXLAN))
1581                 return rte_flow_error_set(error, EINVAL,
1582                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1583                                           "no VXLAN VNI found"
1584                                           " for vxlan encapsulation");
1585         return 0;
1586 }
1587
1588 /**
1589  * Validate outer RTE_FLOW_ITEM_TYPE_UDP item if tunnel item
1590  * RTE_FLOW_ITEM_TYPE_VXLAN is present in item list.
1591  *
1592  * @param[in] udp
1593  *   Outer UDP layer item (if any, NULL otherwise).
1594  * @param[out] error
1595  *   Pointer to the error structure.
1596  *
1597  * @return
1598  *   0 on success, a negative errno value otherwise and rte_errno is set.
1599  **/
1600 static int
1601 flow_tcf_validate_vxlan_decap_udp(const struct rte_flow_item *udp,
1602                                   struct rte_flow_error *error)
1603 {
1604         const struct rte_flow_item_udp *spec = udp->spec;
1605         const struct rte_flow_item_udp *mask = udp->mask;
1606
1607         if (!spec)
1608                 /*
1609                  * Specification for UDP ports cannot be empty
1610                  * because it is required as decap parameter.
1611                  */
1612                 return rte_flow_error_set(error, EINVAL,
1613                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1614                                           "NULL UDP port specification"
1615                                           " for VXLAN decapsulation");
1616         if (!mask)
1617                 mask = &rte_flow_item_udp_mask;
1618         if (mask->hdr.dst_port != RTE_BE16(0x0000)) {
1619                 if (mask->hdr.dst_port != RTE_BE16(0xffff))
1620                         return rte_flow_error_set
1621                                         (error, ENOTSUP,
1622                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1623                                          "no support for partial mask on"
1624                                          " \"udp.hdr.dst_port\" field");
1625                 if (!spec->hdr.dst_port)
1626                         return rte_flow_error_set
1627                                         (error, EINVAL,
1628                                          RTE_FLOW_ERROR_TYPE_ITEM, udp,
1629                                          "zero decap local UDP port");
1630         } else {
1631                 return rte_flow_error_set(error, EINVAL,
1632                                           RTE_FLOW_ERROR_TYPE_ITEM, udp,
1633                                           "outer UDP destination port must be "
1634                                           "specified for vxlan decapsulation");
1635         }
1636         if (mask->hdr.src_port != RTE_BE16(0x0000)) {
1637                 if (mask->hdr.src_port != RTE_BE16(0xffff))
1638                         return rte_flow_error_set
1639                                         (error, ENOTSUP,
1640                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1641                                          "no support for partial mask on"
1642                                          " \"udp.hdr.src_port\" field");
1643                 DRV_LOG(WARNING,
1644                         "outer UDP local port cannot be "
1645                         "forced for VXLAN encapsulation, "
1646                         "parameter ignored");
1647         }
1648         return 0;
1649 }
1650
1651 /**
1652  * Validate flow for E-Switch.
1653  *
1654  * @param[in] priv
1655  *   Pointer to the priv structure.
1656  * @param[in] attr
1657  *   Pointer to the flow attributes.
1658  * @param[in] items
1659  *   Pointer to the list of items.
1660  * @param[in] actions
1661  *   Pointer to the list of actions.
1662  * @param[out] error
1663  *   Pointer to the error structure.
1664  *
1665  * @return
1666  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1667  */
1668 static int
1669 flow_tcf_validate(struct rte_eth_dev *dev,
1670                   const struct rte_flow_attr *attr,
1671                   const struct rte_flow_item items[],
1672                   const struct rte_flow_action actions[],
1673                   struct rte_flow_error *error)
1674 {
1675         union {
1676                 const struct rte_flow_item_port_id *port_id;
1677                 const struct rte_flow_item_eth *eth;
1678                 const struct rte_flow_item_vlan *vlan;
1679                 const struct rte_flow_item_ipv4 *ipv4;
1680                 const struct rte_flow_item_ipv6 *ipv6;
1681                 const struct rte_flow_item_tcp *tcp;
1682                 const struct rte_flow_item_udp *udp;
1683                 const struct rte_flow_item_vxlan *vxlan;
1684         } spec, mask;
1685         union {
1686                 const struct rte_flow_action_port_id *port_id;
1687                 const struct rte_flow_action_jump *jump;
1688                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
1689                 const struct rte_flow_action_of_set_vlan_vid *
1690                         of_set_vlan_vid;
1691                 const struct rte_flow_action_of_set_vlan_pcp *
1692                         of_set_vlan_pcp;
1693                 const struct rte_flow_action_vxlan_encap *vxlan_encap;
1694                 const struct rte_flow_action_set_ipv4 *set_ipv4;
1695                 const struct rte_flow_action_set_ipv6 *set_ipv6;
1696         } conf;
1697         const struct rte_flow_item *outer_udp = NULL;
1698         uint64_t item_flags = 0;
1699         uint64_t action_flags = 0;
1700         uint8_t next_protocol = -1;
1701         unsigned int tcm_ifindex = 0;
1702         uint8_t pedit_validated = 0;
1703         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
1704         struct rte_eth_dev *port_id_dev = NULL;
1705         bool in_port_id_set;
1706         int ret;
1707
1708         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
1709                                                 PTOI_TABLE_SZ_MAX(dev)));
1710         ret = flow_tcf_validate_attributes(attr, error);
1711         if (ret < 0)
1712                 return ret;
1713         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1714                 unsigned int i;
1715                 uint64_t current_action_flag = 0;
1716
1717                 switch (actions->type) {
1718                 case RTE_FLOW_ACTION_TYPE_VOID:
1719                         break;
1720                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
1721                         current_action_flag = MLX5_FLOW_ACTION_PORT_ID;
1722                         if (!actions->conf)
1723                                 break;
1724                         conf.port_id = actions->conf;
1725                         if (conf.port_id->original)
1726                                 i = 0;
1727                         else
1728                                 for (i = 0; ptoi[i].ifindex; ++i)
1729                                         if (ptoi[i].port_id == conf.port_id->id)
1730                                                 break;
1731                         if (!ptoi[i].ifindex)
1732                                 return rte_flow_error_set
1733                                         (error, ENODEV,
1734                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1735                                          conf.port_id,
1736                                          "missing data to convert port ID to"
1737                                          " ifindex");
1738                         port_id_dev = &rte_eth_devices[conf.port_id->id];
1739                         break;
1740                 case RTE_FLOW_ACTION_TYPE_JUMP:
1741                         current_action_flag = MLX5_FLOW_ACTION_JUMP;
1742                         if (!actions->conf)
1743                                 break;
1744                         conf.jump = actions->conf;
1745                         if (attr->group >= conf.jump->group)
1746                                 return rte_flow_error_set
1747                                         (error, ENOTSUP,
1748                                          RTE_FLOW_ERROR_TYPE_ACTION,
1749                                          actions,
1750                                          "can jump only to a group forward");
1751                         break;
1752                 case RTE_FLOW_ACTION_TYPE_DROP:
1753                         current_action_flag = MLX5_FLOW_ACTION_DROP;
1754                         break;
1755                 case RTE_FLOW_ACTION_TYPE_COUNT:
1756                         break;
1757                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
1758                         current_action_flag = MLX5_FLOW_ACTION_OF_POP_VLAN;
1759                         break;
1760                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1761                         rte_be16_t ethertype;
1762
1763                         current_action_flag = MLX5_FLOW_ACTION_OF_PUSH_VLAN;
1764                         if (!actions->conf)
1765                                 break;
1766                         conf.of_push_vlan = actions->conf;
1767                         ethertype = conf.of_push_vlan->ethertype;
1768                         if (ethertype != RTE_BE16(ETH_P_8021Q) &&
1769                             ethertype != RTE_BE16(ETH_P_8021AD))
1770                                 return rte_flow_error_set
1771                                         (error, EINVAL,
1772                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1773                                          "vlan push TPID must be "
1774                                          "802.1Q or 802.1AD");
1775                         break;
1776                 }
1777                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
1778                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1779                                 return rte_flow_error_set
1780                                         (error, ENOTSUP,
1781                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1782                                          "vlan modify is not supported,"
1783                                          " set action must follow push action");
1784                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
1785                         break;
1786                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
1787                         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1788                                 return rte_flow_error_set
1789                                         (error, ENOTSUP,
1790                                          RTE_FLOW_ERROR_TYPE_ACTION, actions,
1791                                          "vlan modify is not supported,"
1792                                          " set action must follow push action");
1793                         current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
1794                         break;
1795                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1796                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_DECAP;
1797                         break;
1798                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1799                         ret = flow_tcf_validate_vxlan_encap(actions, error);
1800                         if (ret < 0)
1801                                 return ret;
1802                         current_action_flag = MLX5_FLOW_ACTION_VXLAN_ENCAP;
1803                         break;
1804                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
1805                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_SRC;
1806                         break;
1807                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
1808                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_DST;
1809                         break;
1810                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
1811                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_SRC;
1812                         break;
1813                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
1814                         current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_DST;
1815                         break;
1816                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
1817                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_SRC;
1818                         break;
1819                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
1820                         current_action_flag = MLX5_FLOW_ACTION_SET_TP_DST;
1821                         break;
1822                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
1823                         current_action_flag = MLX5_FLOW_ACTION_SET_TTL;
1824                         break;
1825                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
1826                         current_action_flag = MLX5_FLOW_ACTION_DEC_TTL;
1827                         break;
1828                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
1829                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_SRC;
1830                         break;
1831                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
1832                         current_action_flag = MLX5_FLOW_ACTION_SET_MAC_DST;
1833                         break;
1834                 default:
1835                         return rte_flow_error_set(error, ENOTSUP,
1836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1837                                                   actions,
1838                                                   "action not supported");
1839                 }
1840                 if (current_action_flag & MLX5_TCF_CONFIG_ACTIONS) {
1841                         if (!actions->conf)
1842                                 return rte_flow_error_set
1843                                         (error, EINVAL,
1844                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1845                                          actions,
1846                                          "action configuration not set");
1847                 }
1848                 if ((current_action_flag & MLX5_TCF_PEDIT_ACTIONS) &&
1849                     pedit_validated)
1850                         return rte_flow_error_set(error, ENOTSUP,
1851                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1852                                                   actions,
1853                                                   "set actions should be "
1854                                                   "listed successively");
1855                 if ((current_action_flag & ~MLX5_TCF_PEDIT_ACTIONS) &&
1856                     (action_flags & MLX5_TCF_PEDIT_ACTIONS))
1857                         pedit_validated = 1;
1858                 if ((current_action_flag & MLX5_TCF_FATE_ACTIONS) &&
1859                     (action_flags & MLX5_TCF_FATE_ACTIONS))
1860                         return rte_flow_error_set(error, EINVAL,
1861                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1862                                                   actions,
1863                                                   "can't have multiple fate"
1864                                                   " actions");
1865                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1866                     (action_flags & MLX5_TCF_VXLAN_ACTIONS))
1867                         return rte_flow_error_set(error, EINVAL,
1868                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1869                                                   actions,
1870                                                   "can't have multiple vxlan"
1871                                                   " actions");
1872                 if ((current_action_flag & MLX5_TCF_VXLAN_ACTIONS) &&
1873                     (action_flags & MLX5_TCF_VLAN_ACTIONS))
1874                         return rte_flow_error_set(error, ENOTSUP,
1875                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1876                                                   actions,
1877                                                   "can't have vxlan and vlan"
1878                                                   " actions in the same rule");
1879                 action_flags |= current_action_flag;
1880         }
1881         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1882                 unsigned int i;
1883
1884                 switch (items->type) {
1885                 case RTE_FLOW_ITEM_TYPE_VOID:
1886                         break;
1887                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
1888                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1889                                 return rte_flow_error_set
1890                                         (error, ENOTSUP,
1891                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1892                                          "inner tunnel port id"
1893                                          " item is not supported");
1894                         mask.port_id = flow_tcf_item_mask
1895                                 (items, &rte_flow_item_port_id_mask,
1896                                  &flow_tcf_mask_supported.port_id,
1897                                  &flow_tcf_mask_empty.port_id,
1898                                  sizeof(flow_tcf_mask_supported.port_id),
1899                                  error);
1900                         if (!mask.port_id)
1901                                 return -rte_errno;
1902                         if (mask.port_id == &flow_tcf_mask_empty.port_id) {
1903                                 in_port_id_set = 1;
1904                                 break;
1905                         }
1906                         spec.port_id = items->spec;
1907                         if (mask.port_id->id && mask.port_id->id != 0xffffffff)
1908                                 return rte_flow_error_set
1909                                         (error, ENOTSUP,
1910                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1911                                          mask.port_id,
1912                                          "no support for partial mask on"
1913                                          " \"id\" field");
1914                         if (!mask.port_id->id)
1915                                 i = 0;
1916                         else
1917                                 for (i = 0; ptoi[i].ifindex; ++i)
1918                                         if (ptoi[i].port_id == spec.port_id->id)
1919                                                 break;
1920                         if (!ptoi[i].ifindex)
1921                                 return rte_flow_error_set
1922                                         (error, ENODEV,
1923                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1924                                          spec.port_id,
1925                                          "missing data to convert port ID to"
1926                                          " ifindex");
1927                         if (in_port_id_set && ptoi[i].ifindex != tcm_ifindex)
1928                                 return rte_flow_error_set
1929                                         (error, ENOTSUP,
1930                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1931                                          spec.port_id,
1932                                          "cannot match traffic for"
1933                                          " several port IDs through"
1934                                          " a single flow rule");
1935                         tcm_ifindex = ptoi[i].ifindex;
1936                         in_port_id_set = 1;
1937                         break;
1938                 case RTE_FLOW_ITEM_TYPE_ETH:
1939                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1940                                                           error);
1941                         if (ret < 0)
1942                                 return ret;
1943                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
1944                                       MLX5_FLOW_LAYER_INNER_L2 :
1945                                       MLX5_FLOW_LAYER_OUTER_L2;
1946                         /* TODO:
1947                          * Redundant check due to different supported mask.
1948                          * Same for the rest of items.
1949                          */
1950                         mask.eth = flow_tcf_item_mask
1951                                 (items, &rte_flow_item_eth_mask,
1952                                  &flow_tcf_mask_supported.eth,
1953                                  &flow_tcf_mask_empty.eth,
1954                                  sizeof(flow_tcf_mask_supported.eth),
1955                                  error);
1956                         if (!mask.eth)
1957                                 return -rte_errno;
1958                         if (mask.eth->type && mask.eth->type !=
1959                             RTE_BE16(0xffff))
1960                                 return rte_flow_error_set
1961                                         (error, ENOTSUP,
1962                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1963                                          mask.eth,
1964                                          "no support for partial mask on"
1965                                          " \"type\" field");
1966                         break;
1967                 case RTE_FLOW_ITEM_TYPE_VLAN:
1968                         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1969                                 return rte_flow_error_set
1970                                         (error, ENOTSUP,
1971                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1972                                          "inner tunnel VLAN"
1973                                          " is not supported");
1974                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1975                                                            error);
1976                         if (ret < 0)
1977                                 return ret;
1978                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
1979                         mask.vlan = flow_tcf_item_mask
1980                                 (items, &rte_flow_item_vlan_mask,
1981                                  &flow_tcf_mask_supported.vlan,
1982                                  &flow_tcf_mask_empty.vlan,
1983                                  sizeof(flow_tcf_mask_supported.vlan),
1984                                  error);
1985                         if (!mask.vlan)
1986                                 return -rte_errno;
1987                         if ((mask.vlan->tci & RTE_BE16(0xe000) &&
1988                              (mask.vlan->tci & RTE_BE16(0xe000)) !=
1989                               RTE_BE16(0xe000)) ||
1990                             (mask.vlan->tci & RTE_BE16(0x0fff) &&
1991                              (mask.vlan->tci & RTE_BE16(0x0fff)) !=
1992                               RTE_BE16(0x0fff)) ||
1993                             (mask.vlan->inner_type &&
1994                              mask.vlan->inner_type != RTE_BE16(0xffff)))
1995                                 return rte_flow_error_set
1996                                         (error, ENOTSUP,
1997                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1998                                          mask.vlan,
1999                                          "no support for partial masks on"
2000                                          " \"tci\" (PCP and VID parts) and"
2001                                          " \"inner_type\" fields");
2002                         break;
2003                 case RTE_FLOW_ITEM_TYPE_IPV4:
2004                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2005                                                            error);
2006                         if (ret < 0)
2007                                 return ret;
2008                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2009                                       MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2010                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2011                         mask.ipv4 = flow_tcf_item_mask
2012                                 (items, &rte_flow_item_ipv4_mask,
2013                                  &flow_tcf_mask_supported.ipv4,
2014                                  &flow_tcf_mask_empty.ipv4,
2015                                  sizeof(flow_tcf_mask_supported.ipv4),
2016                                  error);
2017                         if (!mask.ipv4)
2018                                 return -rte_errno;
2019                         if (mask.ipv4->hdr.next_proto_id &&
2020                             mask.ipv4->hdr.next_proto_id != 0xff)
2021                                 return rte_flow_error_set
2022                                         (error, ENOTSUP,
2023                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2024                                          mask.ipv4,
2025                                          "no support for partial mask on"
2026                                          " \"hdr.next_proto_id\" field");
2027                         else if (mask.ipv4->hdr.next_proto_id)
2028                                 next_protocol =
2029                                         ((const struct rte_flow_item_ipv4 *)
2030                                          (items->spec))->hdr.next_proto_id;
2031                         break;
2032                 case RTE_FLOW_ITEM_TYPE_IPV6:
2033                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2034                                                            error);
2035                         if (ret < 0)
2036                                 return ret;
2037                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2038                                       MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2039                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2040                         mask.ipv6 = flow_tcf_item_mask
2041                                 (items, &rte_flow_item_ipv6_mask,
2042                                  &flow_tcf_mask_supported.ipv6,
2043                                  &flow_tcf_mask_empty.ipv6,
2044                                  sizeof(flow_tcf_mask_supported.ipv6),
2045                                  error);
2046                         if (!mask.ipv6)
2047                                 return -rte_errno;
2048                         if (mask.ipv6->hdr.proto &&
2049                             mask.ipv6->hdr.proto != 0xff)
2050                                 return rte_flow_error_set
2051                                         (error, ENOTSUP,
2052                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2053                                          mask.ipv6,
2054                                          "no support for partial mask on"
2055                                          " \"hdr.proto\" field");
2056                         else if (mask.ipv6->hdr.proto)
2057                                 next_protocol =
2058                                         ((const struct rte_flow_item_ipv6 *)
2059                                          (items->spec))->hdr.proto;
2060                         break;
2061                 case RTE_FLOW_ITEM_TYPE_UDP:
2062                         ret = mlx5_flow_validate_item_udp(items, item_flags,
2063                                                           next_protocol, error);
2064                         if (ret < 0)
2065                                 return ret;
2066                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2067                                       MLX5_FLOW_LAYER_INNER_L4_UDP :
2068                                       MLX5_FLOW_LAYER_OUTER_L4_UDP;
2069                         mask.udp = flow_tcf_item_mask
2070                                 (items, &rte_flow_item_udp_mask,
2071                                  &flow_tcf_mask_supported.udp,
2072                                  &flow_tcf_mask_empty.udp,
2073                                  sizeof(flow_tcf_mask_supported.udp),
2074                                  error);
2075                         if (!mask.udp)
2076                                 return -rte_errno;
2077                         /*
2078                          * Save the presumed outer UDP item for extra check
2079                          * if the tunnel item will be found later in the list.
2080                          */
2081                         if (!(item_flags & MLX5_FLOW_LAYER_TUNNEL))
2082                                 outer_udp = items;
2083                         break;
2084                 case RTE_FLOW_ITEM_TYPE_TCP:
2085                         ret = mlx5_flow_validate_item_tcp
2086                                              (items, item_flags,
2087                                               next_protocol,
2088                                               &flow_tcf_mask_supported.tcp,
2089                                               error);
2090                         if (ret < 0)
2091                                 return ret;
2092                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
2093                                       MLX5_FLOW_LAYER_INNER_L4_TCP :
2094                                       MLX5_FLOW_LAYER_OUTER_L4_TCP;
2095                         mask.tcp = flow_tcf_item_mask
2096                                 (items, &rte_flow_item_tcp_mask,
2097                                  &flow_tcf_mask_supported.tcp,
2098                                  &flow_tcf_mask_empty.tcp,
2099                                  sizeof(flow_tcf_mask_supported.tcp),
2100                                  error);
2101                         if (!mask.tcp)
2102                                 return -rte_errno;
2103                         break;
2104                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2105                         if (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)
2106                                 return rte_flow_error_set
2107                                         (error, ENOTSUP,
2108                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
2109                                          "vxlan tunnel over vlan"
2110                                          " is not supported");
2111                         ret = mlx5_flow_validate_item_vxlan(items,
2112                                                             item_flags, error);
2113                         if (ret < 0)
2114                                 return ret;
2115                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
2116                         mask.vxlan = flow_tcf_item_mask
2117                                 (items, &rte_flow_item_vxlan_mask,
2118                                  &flow_tcf_mask_supported.vxlan,
2119                                  &flow_tcf_mask_empty.vxlan,
2120                                  sizeof(flow_tcf_mask_supported.vxlan), error);
2121                         if (!mask.vxlan)
2122                                 return -rte_errno;
2123                         if (mask.vxlan->vni[0] != 0xff ||
2124                             mask.vxlan->vni[1] != 0xff ||
2125                             mask.vxlan->vni[2] != 0xff)
2126                                 return rte_flow_error_set
2127                                         (error, ENOTSUP,
2128                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2129                                          mask.vxlan,
2130                                          "no support for partial or "
2131                                          "empty mask on \"vxlan.vni\" field");
2132                         /*
2133                          * The VNI item assumes the VXLAN tunnel, it requires
2134                          * at least the outer destination UDP port must be
2135                          * specified without wildcards to allow kernel select
2136                          * the virtual VXLAN device by port. Also outer IPv4
2137                          * or IPv6 item must be specified (wilcards or even
2138                          * zero mask are allowed) to let driver know the tunnel
2139                          * IP version and process UDP traffic correctly.
2140                          */
2141                         if (!(item_flags &
2142                              (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2143                               MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2144                                 return rte_flow_error_set
2145                                                  (error, EINVAL,
2146                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2147                                                   NULL,
2148                                                   "no outer IP pattern found"
2149                                                   " for vxlan tunnel");
2150                         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2151                                 return rte_flow_error_set
2152                                                  (error, EINVAL,
2153                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2154                                                   NULL,
2155                                                   "no outer UDP pattern found"
2156                                                   " for vxlan tunnel");
2157                         /*
2158                          * All items preceding the tunnel item become outer
2159                          * ones and we should do extra validation for them
2160                          * due to tc limitations for tunnel outer parameters.
2161                          * Currently only outer UDP item requres extra check,
2162                          * use the saved pointer instead of item list rescan.
2163                          */
2164                         assert(outer_udp);
2165                         ret = flow_tcf_validate_vxlan_decap_udp
2166                                                 (outer_udp, error);
2167                         if (ret < 0)
2168                                 return ret;
2169                         /* Reset L4 protocol for inner parameters. */
2170                         next_protocol = 0xff;
2171                         break;
2172                 default:
2173                         return rte_flow_error_set(error, ENOTSUP,
2174                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2175                                                   items, "item not supported");
2176                 }
2177         }
2178         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2179             (action_flags & MLX5_FLOW_ACTION_DROP))
2180                 return rte_flow_error_set(error, ENOTSUP,
2181                                           RTE_FLOW_ERROR_TYPE_ACTION,
2182                                           actions,
2183                                           "set action is not compatible with "
2184                                           "drop action");
2185         if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
2186             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2187                 return rte_flow_error_set(error, ENOTSUP,
2188                                           RTE_FLOW_ERROR_TYPE_ACTION,
2189                                           actions,
2190                                           "set action must be followed by "
2191                                           "port_id action");
2192         if (action_flags &
2193            (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST)) {
2194                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4))
2195                         return rte_flow_error_set(error, EINVAL,
2196                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2197                                                   actions,
2198                                                   "no ipv4 item found in"
2199                                                   " pattern");
2200         }
2201         if (action_flags &
2202            (MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST)) {
2203                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6))
2204                         return rte_flow_error_set(error, EINVAL,
2205                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2206                                                   actions,
2207                                                   "no ipv6 item found in"
2208                                                   " pattern");
2209         }
2210         if (action_flags &
2211            (MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST)) {
2212                 if (!(item_flags &
2213                      (MLX5_FLOW_LAYER_OUTER_L4_UDP |
2214                       MLX5_FLOW_LAYER_OUTER_L4_TCP)))
2215                         return rte_flow_error_set(error, EINVAL,
2216                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2217                                                   actions,
2218                                                   "no TCP/UDP item found in"
2219                                                   " pattern");
2220         }
2221         /*
2222          * FW syndrome (0xA9C090):
2223          *     set_flow_table_entry: push vlan action fte in fdb can ONLY be
2224          *     forward to the uplink.
2225          */
2226         if ((action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2227             (action_flags & MLX5_FLOW_ACTION_PORT_ID) &&
2228             ((struct priv *)port_id_dev->data->dev_private)->representor)
2229                 return rte_flow_error_set(error, ENOTSUP,
2230                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2231                                           "vlan push can only be applied"
2232                                           " when forwarding to uplink port");
2233         /*
2234          * FW syndrome (0x294609):
2235          *     set_flow_table_entry: modify/pop/push actions in fdb flow table
2236          *     are supported only while forwarding to vport.
2237          */
2238         if ((action_flags & MLX5_TCF_VLAN_ACTIONS) &&
2239             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2240                 return rte_flow_error_set(error, ENOTSUP,
2241                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2242                                           "vlan actions are supported"
2243                                           " only with port_id action");
2244         if ((action_flags & MLX5_TCF_VXLAN_ACTIONS) &&
2245             !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
2246                 return rte_flow_error_set(error, ENOTSUP,
2247                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2248                                           "vxlan actions are supported"
2249                                           " only with port_id action");
2250         if (!(action_flags & MLX5_TCF_FATE_ACTIONS))
2251                 return rte_flow_error_set(error, EINVAL,
2252                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
2253                                           "no fate action is found");
2254         if (action_flags &
2255            (MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL)) {
2256                 if (!(item_flags &
2257                      (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
2258                       MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2259                         return rte_flow_error_set(error, EINVAL,
2260                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2261                                                   actions,
2262                                                   "no IP found in pattern");
2263         }
2264         if (action_flags &
2265             (MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)) {
2266                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L2))
2267                         return rte_flow_error_set(error, ENOTSUP,
2268                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2269                                                   actions,
2270                                                   "no ethernet found in"
2271                                                   " pattern");
2272         }
2273         if ((action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) &&
2274             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
2275                 return rte_flow_error_set(error, EINVAL,
2276                                           RTE_FLOW_ERROR_TYPE_ACTION,
2277                                           NULL,
2278                                           "no VNI pattern found"
2279                                           " for vxlan decap action");
2280         if ((action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) &&
2281             (item_flags & MLX5_FLOW_LAYER_TUNNEL))
2282                 return rte_flow_error_set(error, EINVAL,
2283                                           RTE_FLOW_ERROR_TYPE_ACTION,
2284                                           NULL,
2285                                           "vxlan encap not supported"
2286                                           " for tunneled traffic");
2287         return 0;
2288 }
2289
2290 /**
2291  * Calculate maximum size of memory for flow items of Linux TC flower.
2292  *
2293  * @param[in] attr
2294  *   Pointer to the flow attributes.
2295  * @param[in] items
2296  *   Pointer to the list of items.
2297  * @param[out] action_flags
2298  *   Pointer to the detected actions.
2299  *
2300  * @return
2301  *   Maximum size of memory for items.
2302  */
2303 static int
2304 flow_tcf_get_items_size(const struct rte_flow_attr *attr,
2305                         const struct rte_flow_item items[],
2306                         uint64_t *action_flags)
2307 {
2308         int size = 0;
2309
2310         size += SZ_NLATTR_STRZ_OF("flower") +
2311                 SZ_NLATTR_NEST + /* TCA_OPTIONS. */
2312                 SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CLS_FLAGS_SKIP_SW. */
2313         if (attr->group > 0)
2314                 size += SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CHAIN. */
2315         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2316                 switch (items->type) {
2317                 case RTE_FLOW_ITEM_TYPE_VOID:
2318                         break;
2319                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2320                         break;
2321                 case RTE_FLOW_ITEM_TYPE_ETH:
2322                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2323                                 SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) * 4;
2324                                 /* dst/src MAC addr and mask. */
2325                         break;
2326                 case RTE_FLOW_ITEM_TYPE_VLAN:
2327                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2328                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2329                                 /* VLAN Ether type. */
2330                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* VLAN prio. */
2331                                 SZ_NLATTR_TYPE_OF(uint16_t); /* VLAN ID. */
2332                         break;
2333                 case RTE_FLOW_ITEM_TYPE_IPV4:
2334                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2335                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2336                                 SZ_NLATTR_TYPE_OF(uint32_t) * 4;
2337                                 /* dst/src IP addr and mask. */
2338                         break;
2339                 case RTE_FLOW_ITEM_TYPE_IPV6:
2340                         size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
2341                                 SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2342                                 SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 4;
2343                                 /* dst/src IP addr and mask. */
2344                         break;
2345                 case RTE_FLOW_ITEM_TYPE_UDP:
2346                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2347                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2348                                 /* dst/src port and mask. */
2349                         break;
2350                 case RTE_FLOW_ITEM_TYPE_TCP:
2351                         size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
2352                                 SZ_NLATTR_TYPE_OF(uint16_t) * 4;
2353                                 /* dst/src port and mask. */
2354                         break;
2355                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2356                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2357                         /*
2358                          * There might be no VXLAN decap action in the action
2359                          * list, nonetheless the VXLAN tunnel flow requires
2360                          * the decap structure to be correctly applied to
2361                          * VXLAN device, set the flag to create the structure.
2362                          * Translation routine will not put the decap action
2363                          * in tne Netlink message if there is no actual action
2364                          * in the list.
2365                          */
2366                         *action_flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2367                         break;
2368                 default:
2369                         DRV_LOG(WARNING,
2370                                 "unsupported item %p type %d,"
2371                                 " items must be validated before flow creation",
2372                                 (const void *)items, items->type);
2373                         break;
2374                 }
2375         }
2376         return size;
2377 }
2378
2379 /**
2380  * Calculate size of memory to store the VXLAN encapsultion
2381  * related items in the Netlink message buffer. Items list
2382  * is specified by RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action.
2383  * The item list should be validated.
2384  *
2385  * @param[in] action
2386  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2387  *   List of pattern items to scan data from.
2388  *
2389  * @return
2390  *   The size the part of Netlink message buffer to store the
2391  *   VXLAN encapsulation item attributes.
2392  */
2393 static int
2394 flow_tcf_vxlan_encap_size(const struct rte_flow_action *action)
2395 {
2396         const struct rte_flow_item *items;
2397         int size = 0;
2398
2399         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2400         assert(action->conf);
2401
2402         items = ((const struct rte_flow_action_vxlan_encap *)
2403                                         action->conf)->definition;
2404         assert(items);
2405         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2406                 switch (items->type) {
2407                 case RTE_FLOW_ITEM_TYPE_VOID:
2408                         break;
2409                 case RTE_FLOW_ITEM_TYPE_ETH:
2410                         /* This item does not require message buffer. */
2411                         break;
2412                 case RTE_FLOW_ITEM_TYPE_IPV4:
2413                         size += SZ_NLATTR_DATA_OF(IPV4_ADDR_LEN) * 2;
2414                         break;
2415                 case RTE_FLOW_ITEM_TYPE_IPV6:
2416                         size += SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN) * 2;
2417                         break;
2418                 case RTE_FLOW_ITEM_TYPE_UDP: {
2419                         const struct rte_flow_item_udp *udp = items->mask;
2420
2421                         size += SZ_NLATTR_TYPE_OF(uint16_t);
2422                         if (!udp || udp->hdr.src_port != RTE_BE16(0x0000))
2423                                 size += SZ_NLATTR_TYPE_OF(uint16_t);
2424                         break;
2425                 }
2426                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2427                         size += SZ_NLATTR_TYPE_OF(uint32_t);
2428                         break;
2429                 default:
2430                         assert(false);
2431                         DRV_LOG(WARNING,
2432                                 "unsupported item %p type %d,"
2433                                 " items must be validated"
2434                                 " before flow creation",
2435                                 (const void *)items, items->type);
2436                         return 0;
2437                 }
2438         }
2439         return size;
2440 }
2441
2442 /**
2443  * Calculate maximum size of memory for flow actions of Linux TC flower and
2444  * extract specified actions.
2445  *
2446  * @param[in] actions
2447  *   Pointer to the list of actions.
2448  * @param[out] action_flags
2449  *   Pointer to the detected actions.
2450  *
2451  * @return
2452  *   Maximum size of memory for actions.
2453  */
2454 static int
2455 flow_tcf_get_actions_and_size(const struct rte_flow_action actions[],
2456                               uint64_t *action_flags)
2457 {
2458         int size = 0;
2459         uint64_t flags = 0;
2460
2461         size += SZ_NLATTR_NEST; /* TCA_FLOWER_ACT. */
2462         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2463                 switch (actions->type) {
2464                 case RTE_FLOW_ACTION_TYPE_VOID:
2465                         break;
2466                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
2467                         size += SZ_NLATTR_NEST + /* na_act_index. */
2468                                 SZ_NLATTR_STRZ_OF("mirred") +
2469                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2470                                 SZ_NLATTR_TYPE_OF(struct tc_mirred);
2471                         flags |= MLX5_FLOW_ACTION_PORT_ID;
2472                         break;
2473                 case RTE_FLOW_ACTION_TYPE_JUMP:
2474                         size += SZ_NLATTR_NEST + /* na_act_index. */
2475                                 SZ_NLATTR_STRZ_OF("gact") +
2476                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2477                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2478                         flags |= MLX5_FLOW_ACTION_JUMP;
2479                         break;
2480                 case RTE_FLOW_ACTION_TYPE_DROP:
2481                         size += SZ_NLATTR_NEST + /* na_act_index. */
2482                                 SZ_NLATTR_STRZ_OF("gact") +
2483                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2484                                 SZ_NLATTR_TYPE_OF(struct tc_gact);
2485                         flags |= MLX5_FLOW_ACTION_DROP;
2486                         break;
2487                 case RTE_FLOW_ACTION_TYPE_COUNT:
2488                         break;
2489                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
2490                         flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
2491                         goto action_of_vlan;
2492                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
2493                         flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
2494                         goto action_of_vlan;
2495                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
2496                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
2497                         goto action_of_vlan;
2498                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
2499                         flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
2500                         goto action_of_vlan;
2501 action_of_vlan:
2502                         size += SZ_NLATTR_NEST + /* na_act_index. */
2503                                 SZ_NLATTR_STRZ_OF("vlan") +
2504                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2505                                 SZ_NLATTR_TYPE_OF(struct tc_vlan) +
2506                                 SZ_NLATTR_TYPE_OF(uint16_t) +
2507                                 /* VLAN protocol. */
2508                                 SZ_NLATTR_TYPE_OF(uint16_t) + /* VLAN ID. */
2509                                 SZ_NLATTR_TYPE_OF(uint8_t); /* VLAN prio. */
2510                         break;
2511                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2512                         size += SZ_NLATTR_NEST + /* na_act_index. */
2513                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2514                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2515                                 SZ_NLATTR_TYPE_OF(uint8_t);
2516                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2517                         size += flow_tcf_vxlan_encap_size(actions) +
2518                                 RTE_ALIGN_CEIL /* preceding encap params. */
2519                                 (sizeof(struct flow_tcf_vxlan_encap),
2520                                 MNL_ALIGNTO);
2521                         flags |= MLX5_FLOW_ACTION_VXLAN_ENCAP;
2522                         break;
2523                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
2524                         size += SZ_NLATTR_NEST + /* na_act_index. */
2525                                 SZ_NLATTR_STRZ_OF("tunnel_key") +
2526                                 SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
2527                                 SZ_NLATTR_TYPE_OF(uint8_t);
2528                         size += SZ_NLATTR_TYPE_OF(struct tc_tunnel_key);
2529                         size += RTE_ALIGN_CEIL /* preceding decap params. */
2530                                 (sizeof(struct flow_tcf_vxlan_decap),
2531                                 MNL_ALIGNTO);
2532                         flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
2533                         break;
2534                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
2535                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
2536                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
2537                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
2538                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
2539                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
2540                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
2541                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
2542                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
2543                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
2544                         size += flow_tcf_get_pedit_actions_size(&actions,
2545                                                                 &flags);
2546                         break;
2547                 default:
2548                         DRV_LOG(WARNING,
2549                                 "unsupported action %p type %d,"
2550                                 " items must be validated before flow creation",
2551                                 (const void *)actions, actions->type);
2552                         break;
2553                 }
2554         }
2555         *action_flags = flags;
2556         return size;
2557 }
2558
2559 /**
2560  * Brand rtnetlink buffer with unique handle.
2561  *
2562  * This handle should be unique for a given network interface to avoid
2563  * collisions.
2564  *
2565  * @param nlh
2566  *   Pointer to Netlink message.
2567  * @param handle
2568  *   Unique 32-bit handle to use.
2569  */
2570 static void
2571 flow_tcf_nl_brand(struct nlmsghdr *nlh, uint32_t handle)
2572 {
2573         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
2574
2575         tcm->tcm_handle = handle;
2576         DRV_LOG(DEBUG, "Netlink msg %p is branded with handle %x",
2577                 (void *)nlh, handle);
2578 }
2579
2580 /**
2581  * Prepare a flow object for Linux TC flower. It calculates the maximum size of
2582  * memory required, allocates the memory, initializes Netlink message headers
2583  * and set unique TC message handle.
2584  *
2585  * @param[in] attr
2586  *   Pointer to the flow attributes.
2587  * @param[in] items
2588  *   Pointer to the list of items.
2589  * @param[in] actions
2590  *   Pointer to the list of actions.
2591  * @param[out] error
2592  *   Pointer to the error structure.
2593  *
2594  * @return
2595  *   Pointer to mlx5_flow object on success,
2596  *   otherwise NULL and rte_ernno is set.
2597  */
2598 static struct mlx5_flow *
2599 flow_tcf_prepare(const struct rte_flow_attr *attr,
2600                  const struct rte_flow_item items[],
2601                  const struct rte_flow_action actions[],
2602                  struct rte_flow_error *error)
2603 {
2604         size_t size = RTE_ALIGN_CEIL
2605                         (sizeof(struct mlx5_flow),
2606                          alignof(struct flow_tcf_tunnel_hdr)) +
2607                       MNL_ALIGN(sizeof(struct nlmsghdr)) +
2608                       MNL_ALIGN(sizeof(struct tcmsg));
2609         struct mlx5_flow *dev_flow;
2610         uint64_t action_flags = 0;
2611         struct nlmsghdr *nlh;
2612         struct tcmsg *tcm;
2613         uint8_t *sp, *tun = NULL;
2614
2615         size += flow_tcf_get_items_size(attr, items, &action_flags);
2616         size += flow_tcf_get_actions_and_size(actions, &action_flags);
2617         dev_flow = rte_zmalloc(__func__, size, MNL_ALIGNTO);
2618         if (!dev_flow) {
2619                 rte_flow_error_set(error, ENOMEM,
2620                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2621                                    "not enough memory to create E-Switch flow");
2622                 return NULL;
2623         }
2624         sp = (uint8_t *)(dev_flow + 1);
2625         if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP) {
2626                 sp = RTE_PTR_ALIGN
2627                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2628                 tun = sp;
2629                 sp += RTE_ALIGN_CEIL
2630                         (sizeof(struct flow_tcf_vxlan_encap),
2631                         MNL_ALIGNTO);
2632 #ifndef NDEBUG
2633                 size -= RTE_ALIGN_CEIL
2634                         (sizeof(struct flow_tcf_vxlan_encap),
2635                         MNL_ALIGNTO);
2636 #endif
2637         } else if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP) {
2638                 sp = RTE_PTR_ALIGN
2639                         (sp, alignof(struct flow_tcf_tunnel_hdr));
2640                 tun = sp;
2641                 sp += RTE_ALIGN_CEIL
2642                         (sizeof(struct flow_tcf_vxlan_decap),
2643                         MNL_ALIGNTO);
2644 #ifndef NDEBUG
2645                 size -= RTE_ALIGN_CEIL
2646                         (sizeof(struct flow_tcf_vxlan_decap),
2647                         MNL_ALIGNTO);
2648 #endif
2649         } else {
2650                 sp = RTE_PTR_ALIGN(sp, MNL_ALIGNTO);
2651         }
2652         nlh = mnl_nlmsg_put_header(sp);
2653         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
2654         *dev_flow = (struct mlx5_flow){
2655                 .tcf = (struct mlx5_flow_tcf){
2656 #ifndef NDEBUG
2657                         .nlsize = size - RTE_ALIGN_CEIL
2658                                 (sizeof(struct mlx5_flow),
2659                                  alignof(struct flow_tcf_tunnel_hdr)),
2660 #endif
2661                         .tunnel = (struct flow_tcf_tunnel_hdr *)tun,
2662                         .nlh = nlh,
2663                         .tcm = tcm,
2664                 },
2665         };
2666         if (action_flags & MLX5_FLOW_ACTION_VXLAN_DECAP)
2667                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_DECAP;
2668         else if (action_flags & MLX5_FLOW_ACTION_VXLAN_ENCAP)
2669                 dev_flow->tcf.tunnel->type = FLOW_TCF_TUNACT_VXLAN_ENCAP;
2670         /*
2671          * Generate a reasonably unique handle based on the address of the
2672          * target buffer.
2673          *
2674          * This is straightforward on 32-bit systems where the flow pointer can
2675          * be used directly. Otherwise, its least significant part is taken
2676          * after shifting it by the previous power of two of the pointed buffer
2677          * size.
2678          */
2679         if (sizeof(dev_flow) <= 4)
2680                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow);
2681         else
2682                 flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow >>
2683                                        rte_log2_u32(rte_align32prevpow2(size)));
2684         return dev_flow;
2685 }
2686
2687 /**
2688  * Make adjustments for supporting count actions.
2689  *
2690  * @param[in] dev
2691  *   Pointer to the Ethernet device structure.
2692  * @param[in] dev_flow
2693  *   Pointer to mlx5_flow.
2694  * @param[out] error
2695  *   Pointer to error structure.
2696  *
2697  * @return
2698  *   0 On success else a negative errno value is returned and rte_errno is set.
2699  */
2700 static int
2701 flow_tcf_translate_action_count(struct rte_eth_dev *dev __rte_unused,
2702                                   struct mlx5_flow *dev_flow,
2703                                   struct rte_flow_error *error)
2704 {
2705         struct rte_flow *flow = dev_flow->flow;
2706
2707         if (!flow->counter) {
2708                 flow->counter = flow_tcf_counter_new();
2709                 if (!flow->counter)
2710                         return rte_flow_error_set(error, rte_errno,
2711                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2712                                                   NULL,
2713                                                   "cannot get counter"
2714                                                   " context.");
2715         }
2716         return 0;
2717 }
2718
2719 /**
2720  * Convert VXLAN VNI to 32-bit integer.
2721  *
2722  * @param[in] vni
2723  *   VXLAN VNI in 24-bit wire format.
2724  *
2725  * @return
2726  *   VXLAN VNI as a 32-bit integer value in network endian.
2727  */
2728 static inline rte_be32_t
2729 vxlan_vni_as_be32(const uint8_t vni[3])
2730 {
2731         union {
2732                 uint8_t vni[4];
2733                 rte_be32_t dword;
2734         } ret = {
2735                 .vni = { 0, vni[0], vni[1], vni[2] },
2736         };
2737         return ret.dword;
2738 }
2739
2740 /**
2741  * Helper function to process RTE_FLOW_ITEM_TYPE_ETH entry in configuration
2742  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the MAC address fields
2743  * in the encapsulation parameters structure. The item must be prevalidated,
2744  * no any validation checks performed by function.
2745  *
2746  * @param[in] spec
2747  *   RTE_FLOW_ITEM_TYPE_ETH entry specification.
2748  * @param[in] mask
2749  *   RTE_FLOW_ITEM_TYPE_ETH entry mask.
2750  * @param[out] encap
2751  *   Structure to fill the gathered MAC address data.
2752  */
2753 static void
2754 flow_tcf_parse_vxlan_encap_eth(const struct rte_flow_item_eth *spec,
2755                                const struct rte_flow_item_eth *mask,
2756                                struct flow_tcf_vxlan_encap *encap)
2757 {
2758         /* Item must be validated before. No redundant checks. */
2759         assert(spec);
2760         if (!mask || !memcmp(&mask->dst,
2761                              &rte_flow_item_eth_mask.dst,
2762                              sizeof(rte_flow_item_eth_mask.dst))) {
2763                 /*
2764                  * Ethernet addresses are not supported by
2765                  * tc as tunnel_key parameters. Destination
2766                  * address is needed to form encap packet
2767                  * header and retrieved by kernel from
2768                  * implicit sources (ARP table, etc),
2769                  * address masks are not supported at all.
2770                  */
2771                 encap->eth.dst = spec->dst;
2772                 encap->mask |= FLOW_TCF_ENCAP_ETH_DST;
2773         }
2774         if (!mask || !memcmp(&mask->src,
2775                              &rte_flow_item_eth_mask.src,
2776                              sizeof(rte_flow_item_eth_mask.src))) {
2777                 /*
2778                  * Ethernet addresses are not supported by
2779                  * tc as tunnel_key parameters. Source ethernet
2780                  * address is ignored anyway.
2781                  */
2782                 encap->eth.src = spec->src;
2783                 encap->mask |= FLOW_TCF_ENCAP_ETH_SRC;
2784         }
2785 }
2786
2787 /**
2788  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV4 entry in configuration
2789  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV4 address fields
2790  * in the encapsulation parameters structure. The item must be prevalidated,
2791  * no any validation checks performed by function.
2792  *
2793  * @param[in] spec
2794  *   RTE_FLOW_ITEM_TYPE_IPV4 entry specification.
2795  * @param[out] encap
2796  *   Structure to fill the gathered IPV4 address data.
2797  */
2798 static void
2799 flow_tcf_parse_vxlan_encap_ipv4(const struct rte_flow_item_ipv4 *spec,
2800                                 struct flow_tcf_vxlan_encap *encap)
2801 {
2802         /* Item must be validated before. No redundant checks. */
2803         assert(spec);
2804         encap->ipv4.dst = spec->hdr.dst_addr;
2805         encap->ipv4.src = spec->hdr.src_addr;
2806         encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC |
2807                        FLOW_TCF_ENCAP_IPV4_DST;
2808 }
2809
2810 /**
2811  * Helper function to process RTE_FLOW_ITEM_TYPE_IPV6 entry in configuration
2812  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the IPV6 address fields
2813  * in the encapsulation parameters structure. The item must be prevalidated,
2814  * no any validation checks performed by function.
2815  *
2816  * @param[in] spec
2817  *   RTE_FLOW_ITEM_TYPE_IPV6 entry specification.
2818  * @param[out] encap
2819  *   Structure to fill the gathered IPV6 address data.
2820  */
2821 static void
2822 flow_tcf_parse_vxlan_encap_ipv6(const struct rte_flow_item_ipv6 *spec,
2823                                 struct flow_tcf_vxlan_encap *encap)
2824 {
2825         /* Item must be validated before. No redundant checks. */
2826         assert(spec);
2827         memcpy(encap->ipv6.dst, spec->hdr.dst_addr, IPV6_ADDR_LEN);
2828         memcpy(encap->ipv6.src, spec->hdr.src_addr, IPV6_ADDR_LEN);
2829         encap->mask |= FLOW_TCF_ENCAP_IPV6_SRC |
2830                        FLOW_TCF_ENCAP_IPV6_DST;
2831 }
2832
2833 /**
2834  * Helper function to process RTE_FLOW_ITEM_TYPE_UDP entry in configuration
2835  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the UDP port fields
2836  * in the encapsulation parameters structure. The item must be prevalidated,
2837  * no any validation checks performed by function.
2838  *
2839  * @param[in] spec
2840  *   RTE_FLOW_ITEM_TYPE_UDP entry specification.
2841  * @param[in] mask
2842  *   RTE_FLOW_ITEM_TYPE_UDP entry mask.
2843  * @param[out] encap
2844  *   Structure to fill the gathered UDP port data.
2845  */
2846 static void
2847 flow_tcf_parse_vxlan_encap_udp(const struct rte_flow_item_udp *spec,
2848                                const struct rte_flow_item_udp *mask,
2849                                struct flow_tcf_vxlan_encap *encap)
2850 {
2851         assert(spec);
2852         encap->udp.dst = spec->hdr.dst_port;
2853         encap->mask |= FLOW_TCF_ENCAP_UDP_DST;
2854         if (!mask || mask->hdr.src_port != RTE_BE16(0x0000)) {
2855                 encap->udp.src = spec->hdr.src_port;
2856                 encap->mask |= FLOW_TCF_ENCAP_IPV4_SRC;
2857         }
2858 }
2859
2860 /**
2861  * Helper function to process RTE_FLOW_ITEM_TYPE_VXLAN entry in configuration
2862  * of action RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. Fills the VNI fields
2863  * in the encapsulation parameters structure. The item must be prevalidated,
2864  * no any validation checks performed by function.
2865  *
2866  * @param[in] spec
2867  *   RTE_FLOW_ITEM_TYPE_VXLAN entry specification.
2868  * @param[out] encap
2869  *   Structure to fill the gathered VNI address data.
2870  */
2871 static void
2872 flow_tcf_parse_vxlan_encap_vni(const struct rte_flow_item_vxlan *spec,
2873                                struct flow_tcf_vxlan_encap *encap)
2874 {
2875         /* Item must be validated before. Do not redundant checks. */
2876         assert(spec);
2877         memcpy(encap->vxlan.vni, spec->vni, sizeof(encap->vxlan.vni));
2878         encap->mask |= FLOW_TCF_ENCAP_VXLAN_VNI;
2879 }
2880
2881 /**
2882  * Populate consolidated encapsulation object from list of pattern items.
2883  *
2884  * Helper function to process configuration of action such as
2885  * RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP. The item list should be
2886  * validated, there is no way to return an meaningful error.
2887  *
2888  * @param[in] action
2889  *   RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP action object.
2890  *   List of pattern items to gather data from.
2891  * @param[out] src
2892  *   Structure to fill gathered data.
2893  */
2894 static void
2895 flow_tcf_vxlan_encap_parse(const struct rte_flow_action *action,
2896                            struct flow_tcf_vxlan_encap *encap)
2897 {
2898         union {
2899                 const struct rte_flow_item_eth *eth;
2900                 const struct rte_flow_item_ipv4 *ipv4;
2901                 const struct rte_flow_item_ipv6 *ipv6;
2902                 const struct rte_flow_item_udp *udp;
2903                 const struct rte_flow_item_vxlan *vxlan;
2904         } spec, mask;
2905         const struct rte_flow_item *items;
2906
2907         assert(action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP);
2908         assert(action->conf);
2909
2910         items = ((const struct rte_flow_action_vxlan_encap *)
2911                                         action->conf)->definition;
2912         assert(items);
2913         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2914                 switch (items->type) {
2915                 case RTE_FLOW_ITEM_TYPE_VOID:
2916                         break;
2917                 case RTE_FLOW_ITEM_TYPE_ETH:
2918                         mask.eth = items->mask;
2919                         spec.eth = items->spec;
2920                         flow_tcf_parse_vxlan_encap_eth(spec.eth, mask.eth,
2921                                                        encap);
2922                         break;
2923                 case RTE_FLOW_ITEM_TYPE_IPV4:
2924                         spec.ipv4 = items->spec;
2925                         flow_tcf_parse_vxlan_encap_ipv4(spec.ipv4, encap);
2926                         break;
2927                 case RTE_FLOW_ITEM_TYPE_IPV6:
2928                         spec.ipv6 = items->spec;
2929                         flow_tcf_parse_vxlan_encap_ipv6(spec.ipv6, encap);
2930                         break;
2931                 case RTE_FLOW_ITEM_TYPE_UDP:
2932                         mask.udp = items->mask;
2933                         spec.udp = items->spec;
2934                         flow_tcf_parse_vxlan_encap_udp(spec.udp, mask.udp,
2935                                                        encap);
2936                         break;
2937                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2938                         spec.vxlan = items->spec;
2939                         flow_tcf_parse_vxlan_encap_vni(spec.vxlan, encap);
2940                         break;
2941                 default:
2942                         assert(false);
2943                         DRV_LOG(WARNING,
2944                                 "unsupported item %p type %d,"
2945                                 " items must be validated"
2946                                 " before flow creation",
2947                                 (const void *)items, items->type);
2948                         encap->mask = 0;
2949                         return;
2950                 }
2951         }
2952 }
2953
2954 /**
2955  * Translate flow for Linux TC flower and construct Netlink message.
2956  *
2957  * @param[in] priv
2958  *   Pointer to the priv structure.
2959  * @param[in, out] flow
2960  *   Pointer to the sub flow.
2961  * @param[in] attr
2962  *   Pointer to the flow attributes.
2963  * @param[in] items
2964  *   Pointer to the list of items.
2965  * @param[in] actions
2966  *   Pointer to the list of actions.
2967  * @param[out] error
2968  *   Pointer to the error structure.
2969  *
2970  * @return
2971  *   0 on success, a negative errno value otherwise and rte_ernno is set.
2972  */
2973 static int
2974 flow_tcf_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
2975                    const struct rte_flow_attr *attr,
2976                    const struct rte_flow_item items[],
2977                    const struct rte_flow_action actions[],
2978                    struct rte_flow_error *error)
2979 {
2980         union {
2981                 const struct rte_flow_item_port_id *port_id;
2982                 const struct rte_flow_item_eth *eth;
2983                 const struct rte_flow_item_vlan *vlan;
2984                 const struct rte_flow_item_ipv4 *ipv4;
2985                 const struct rte_flow_item_ipv6 *ipv6;
2986                 const struct rte_flow_item_tcp *tcp;
2987                 const struct rte_flow_item_udp *udp;
2988                 const struct rte_flow_item_vxlan *vxlan;
2989         } spec, mask;
2990         union {
2991                 const struct rte_flow_action_port_id *port_id;
2992                 const struct rte_flow_action_jump *jump;
2993                 const struct rte_flow_action_of_push_vlan *of_push_vlan;
2994                 const struct rte_flow_action_of_set_vlan_vid *
2995                         of_set_vlan_vid;
2996                 const struct rte_flow_action_of_set_vlan_pcp *
2997                         of_set_vlan_pcp;
2998         } conf;
2999         union {
3000                 struct flow_tcf_tunnel_hdr *hdr;
3001                 struct flow_tcf_vxlan_decap *vxlan;
3002         } decap = {
3003                 .hdr = NULL,
3004         };
3005         union {
3006                 struct flow_tcf_tunnel_hdr *hdr;
3007                 struct flow_tcf_vxlan_encap *vxlan;
3008         } encap = {
3009                 .hdr = NULL,
3010         };
3011         struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
3012         struct nlmsghdr *nlh = dev_flow->tcf.nlh;
3013         struct tcmsg *tcm = dev_flow->tcf.tcm;
3014         uint32_t na_act_index_cur;
3015         bool eth_type_set = 0;
3016         bool vlan_present = 0;
3017         bool vlan_eth_type_set = 0;
3018         bool ip_proto_set = 0;
3019         bool tunnel_outer = 0;
3020         struct nlattr *na_flower;
3021         struct nlattr *na_flower_act;
3022         struct nlattr *na_vlan_id = NULL;
3023         struct nlattr *na_vlan_priority = NULL;
3024         uint64_t item_flags = 0;
3025         int ret;
3026
3027         claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
3028                                                 PTOI_TABLE_SZ_MAX(dev)));
3029         if (dev_flow->tcf.tunnel) {
3030                 switch (dev_flow->tcf.tunnel->type) {
3031                 case FLOW_TCF_TUNACT_VXLAN_DECAP:
3032                         decap.vxlan = dev_flow->tcf.vxlan_decap;
3033                         tunnel_outer = 1;
3034                         break;
3035                 case FLOW_TCF_TUNACT_VXLAN_ENCAP:
3036                         encap.vxlan = dev_flow->tcf.vxlan_encap;
3037                         break;
3038                 /* New tunnel actions can be added here. */
3039                 default:
3040                         assert(false);
3041                         break;
3042                 }
3043         }
3044         nlh = dev_flow->tcf.nlh;
3045         tcm = dev_flow->tcf.tcm;
3046         /* Prepare API must have been called beforehand. */
3047         assert(nlh != NULL && tcm != NULL);
3048         tcm->tcm_family = AF_UNSPEC;
3049         tcm->tcm_ifindex = ptoi[0].ifindex;
3050         tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
3051         /*
3052          * Priority cannot be zero to prevent the kernel from picking one
3053          * automatically.
3054          */
3055         tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
3056                                   RTE_BE16(ETH_P_ALL));
3057         if (attr->group > 0)
3058                 mnl_attr_put_u32(nlh, TCA_CHAIN, attr->group);
3059         mnl_attr_put_strz(nlh, TCA_KIND, "flower");
3060         na_flower = mnl_attr_nest_start(nlh, TCA_OPTIONS);
3061         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3062                 unsigned int i;
3063
3064                 switch (items->type) {
3065                 case RTE_FLOW_ITEM_TYPE_VOID:
3066                         break;
3067                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3068                         mask.port_id = flow_tcf_item_mask
3069                                 (items, &rte_flow_item_port_id_mask,
3070                                  &flow_tcf_mask_supported.port_id,
3071                                  &flow_tcf_mask_empty.port_id,
3072                                  sizeof(flow_tcf_mask_supported.port_id),
3073                                  error);
3074                         assert(mask.port_id);
3075                         if (mask.port_id == &flow_tcf_mask_empty.port_id)
3076                                 break;
3077                         spec.port_id = items->spec;
3078                         if (!mask.port_id->id)
3079                                 i = 0;
3080                         else
3081                                 for (i = 0; ptoi[i].ifindex; ++i)
3082                                         if (ptoi[i].port_id == spec.port_id->id)
3083                                                 break;
3084                         assert(ptoi[i].ifindex);
3085                         tcm->tcm_ifindex = ptoi[i].ifindex;
3086                         break;
3087                 case RTE_FLOW_ITEM_TYPE_ETH:
3088                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3089                                       MLX5_FLOW_LAYER_INNER_L2 :
3090                                       MLX5_FLOW_LAYER_OUTER_L2;
3091                         mask.eth = flow_tcf_item_mask
3092                                 (items, &rte_flow_item_eth_mask,
3093                                  &flow_tcf_mask_supported.eth,
3094                                  &flow_tcf_mask_empty.eth,
3095                                  sizeof(flow_tcf_mask_supported.eth),
3096                                  error);
3097                         assert(mask.eth);
3098                         if (mask.eth == &flow_tcf_mask_empty.eth)
3099                                 break;
3100                         spec.eth = items->spec;
3101                         if (tunnel_outer) {
3102                                 DRV_LOG(WARNING,
3103                                         "outer L2 addresses cannot be"
3104                                         " forced is outer ones for tunnel,"
3105                                         " parameter is ignored");
3106                                 break;
3107                         }
3108                         if (mask.eth->type) {
3109                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3110                                                  spec.eth->type);
3111                                 eth_type_set = 1;
3112                         }
3113                         if (!is_zero_ether_addr(&mask.eth->dst)) {
3114                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST,
3115                                              ETHER_ADDR_LEN,
3116                                              spec.eth->dst.addr_bytes);
3117                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST_MASK,
3118                                              ETHER_ADDR_LEN,
3119                                              mask.eth->dst.addr_bytes);
3120                         }
3121                         if (!is_zero_ether_addr(&mask.eth->src)) {
3122                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC,
3123                                              ETHER_ADDR_LEN,
3124                                              spec.eth->src.addr_bytes);
3125                                 mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC_MASK,
3126                                              ETHER_ADDR_LEN,
3127                                              mask.eth->src.addr_bytes);
3128                         }
3129                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3130                         break;
3131                 case RTE_FLOW_ITEM_TYPE_VLAN:
3132                         assert(!encap.hdr);
3133                         assert(!decap.hdr);
3134                         assert(!tunnel_outer);
3135                         item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
3136                         mask.vlan = flow_tcf_item_mask
3137                                 (items, &rte_flow_item_vlan_mask,
3138                                  &flow_tcf_mask_supported.vlan,
3139                                  &flow_tcf_mask_empty.vlan,
3140                                  sizeof(flow_tcf_mask_supported.vlan),
3141                                  error);
3142                         assert(mask.vlan);
3143                         if (!eth_type_set)
3144                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
3145                                                  RTE_BE16(ETH_P_8021Q));
3146                         eth_type_set = 1;
3147                         vlan_present = 1;
3148                         if (mask.vlan == &flow_tcf_mask_empty.vlan)
3149                                 break;
3150                         spec.vlan = items->spec;
3151                         if (mask.vlan->inner_type) {
3152                                 mnl_attr_put_u16(nlh,
3153                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE,
3154                                                  spec.vlan->inner_type);
3155                                 vlan_eth_type_set = 1;
3156                         }
3157                         if (mask.vlan->tci & RTE_BE16(0xe000))
3158                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_VLAN_PRIO,
3159                                                 (rte_be_to_cpu_16
3160                                                  (spec.vlan->tci) >> 13) & 0x7);
3161                         if (mask.vlan->tci & RTE_BE16(0x0fff))
3162                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_VLAN_ID,
3163                                                  rte_be_to_cpu_16
3164                                                  (spec.vlan->tci &
3165                                                   RTE_BE16(0x0fff)));
3166                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3167                         break;
3168                 case RTE_FLOW_ITEM_TYPE_IPV4:
3169                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3170                                       MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3171                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3172                         mask.ipv4 = flow_tcf_item_mask
3173                                 (items, &rte_flow_item_ipv4_mask,
3174                                  &flow_tcf_mask_supported.ipv4,
3175                                  &flow_tcf_mask_empty.ipv4,
3176                                  sizeof(flow_tcf_mask_supported.ipv4),
3177                                  error);
3178                         assert(mask.ipv4);
3179                         spec.ipv4 = items->spec;
3180                         if (!tunnel_outer) {
3181                                 if (!eth_type_set ||
3182                                     (!vlan_eth_type_set && vlan_present))
3183                                         mnl_attr_put_u16
3184                                                 (nlh,
3185                                                  vlan_present ?
3186                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3187                                                  TCA_FLOWER_KEY_ETH_TYPE,
3188                                                  RTE_BE16(ETH_P_IP));
3189                                 eth_type_set = 1;
3190                                 vlan_eth_type_set = 1;
3191                         }
3192                         if (!tunnel_outer && mask.ipv4->hdr.next_proto_id) {
3193                                 /*
3194                                  * No way to set IP protocol for outer tunnel
3195                                  * layers. Usually it is fixed, for example,
3196                                  * to UDP for VXLAN/GPE.
3197                                  */
3198                                 assert(spec.ipv4); /* Mask is not empty. */
3199                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3200                                                 spec.ipv4->hdr.next_proto_id);
3201                                 ip_proto_set = 1;
3202                         }
3203                         if (mask.ipv4 == &flow_tcf_mask_empty.ipv4 ||
3204                              (!mask.ipv4->hdr.src_addr &&
3205                               !mask.ipv4->hdr.dst_addr)) {
3206                                 if (!tunnel_outer)
3207                                         break;
3208                                 /*
3209                                  * For tunnel outer we must set outer IP key
3210                                  * anyway, even if the specification/mask is
3211                                  * empty. There is no another way to tell
3212                                  * kernel about he outer layer protocol.
3213                                  */
3214                                 mnl_attr_put_u32
3215                                         (nlh, TCA_FLOWER_KEY_ENC_IPV4_SRC,
3216                                          mask.ipv4->hdr.src_addr);
3217                                 mnl_attr_put_u32
3218                                         (nlh, TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
3219                                          mask.ipv4->hdr.src_addr);
3220                                 assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3221                                 break;
3222                         }
3223                         if (mask.ipv4->hdr.src_addr) {
3224                                 mnl_attr_put_u32
3225                                         (nlh, tunnel_outer ?
3226                                          TCA_FLOWER_KEY_ENC_IPV4_SRC :
3227                                          TCA_FLOWER_KEY_IPV4_SRC,
3228                                          spec.ipv4->hdr.src_addr);
3229                                 mnl_attr_put_u32
3230                                         (nlh, tunnel_outer ?
3231                                          TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK :
3232                                          TCA_FLOWER_KEY_IPV4_SRC_MASK,
3233                                          mask.ipv4->hdr.src_addr);
3234                         }
3235                         if (mask.ipv4->hdr.dst_addr) {
3236                                 mnl_attr_put_u32
3237                                         (nlh, tunnel_outer ?
3238                                          TCA_FLOWER_KEY_ENC_IPV4_DST :
3239                                          TCA_FLOWER_KEY_IPV4_DST,
3240                                          spec.ipv4->hdr.dst_addr);
3241                                 mnl_attr_put_u32
3242                                         (nlh, tunnel_outer ?
3243                                          TCA_FLOWER_KEY_ENC_IPV4_DST_MASK :
3244                                          TCA_FLOWER_KEY_IPV4_DST_MASK,
3245                                          mask.ipv4->hdr.dst_addr);
3246                         }
3247                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3248                         break;
3249                 case RTE_FLOW_ITEM_TYPE_IPV6: {
3250                         bool ipv6_src, ipv6_dst;
3251
3252                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3253                                       MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3254                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3255                         mask.ipv6 = flow_tcf_item_mask
3256                                 (items, &rte_flow_item_ipv6_mask,
3257                                  &flow_tcf_mask_supported.ipv6,
3258                                  &flow_tcf_mask_empty.ipv6,
3259                                  sizeof(flow_tcf_mask_supported.ipv6),
3260                                  error);
3261                         assert(mask.ipv6);
3262                         spec.ipv6 = items->spec;
3263                         if (!tunnel_outer) {
3264                                 if (!eth_type_set ||
3265                                     (!vlan_eth_type_set && vlan_present))
3266                                         mnl_attr_put_u16
3267                                                 (nlh,
3268                                                  vlan_present ?
3269                                                  TCA_FLOWER_KEY_VLAN_ETH_TYPE :
3270                                                  TCA_FLOWER_KEY_ETH_TYPE,
3271                                                  RTE_BE16(ETH_P_IPV6));
3272                                 eth_type_set = 1;
3273                                 vlan_eth_type_set = 1;
3274                         }
3275                         if (!tunnel_outer && mask.ipv6->hdr.proto) {
3276                                 /*
3277                                  * No way to set IP protocol for outer tunnel
3278                                  * layers. Usually it is fixed, for example,
3279                                  * to UDP for VXLAN/GPE.
3280                                  */
3281                                 assert(spec.ipv6); /* Mask is not empty. */
3282                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3283                                                 spec.ipv6->hdr.proto);
3284                                 ip_proto_set = 1;
3285                         }
3286                         ipv6_dst = !IN6_IS_ADDR_UNSPECIFIED
3287                                                 (mask.ipv6->hdr.dst_addr);
3288                         ipv6_src = !IN6_IS_ADDR_UNSPECIFIED
3289                                                 (mask.ipv6->hdr.src_addr);
3290                         if (mask.ipv6 == &flow_tcf_mask_empty.ipv6 ||
3291                              (!ipv6_dst && !ipv6_src)) {
3292                                 if (!tunnel_outer)
3293                                         break;
3294                                 /*
3295                                  * For tunnel outer we must set outer IP key
3296                                  * anyway, even if the specification/mask is
3297                                  * empty. There is no another way to tell
3298                                  * kernel about he outer layer protocol.
3299                                  */
3300                                 mnl_attr_put(nlh,
3301                                              TCA_FLOWER_KEY_ENC_IPV6_SRC,
3302                                              IPV6_ADDR_LEN,
3303                                              mask.ipv6->hdr.src_addr);
3304                                 mnl_attr_put(nlh,
3305                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
3306                                              IPV6_ADDR_LEN,
3307                                              mask.ipv6->hdr.src_addr);
3308                                 assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3309                                 break;
3310                         }
3311                         if (ipv6_src) {
3312                                 mnl_attr_put(nlh, tunnel_outer ?
3313                                              TCA_FLOWER_KEY_ENC_IPV6_SRC :
3314                                              TCA_FLOWER_KEY_IPV6_SRC,
3315                                              IPV6_ADDR_LEN,
3316                                              spec.ipv6->hdr.src_addr);
3317                                 mnl_attr_put(nlh, tunnel_outer ?
3318                                              TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK :
3319                                              TCA_FLOWER_KEY_IPV6_SRC_MASK,
3320                                              IPV6_ADDR_LEN,
3321                                              mask.ipv6->hdr.src_addr);
3322                         }
3323                         if (ipv6_dst) {
3324                                 mnl_attr_put(nlh, tunnel_outer ?
3325                                              TCA_FLOWER_KEY_ENC_IPV6_DST :
3326                                              TCA_FLOWER_KEY_IPV6_DST,
3327                                              IPV6_ADDR_LEN,
3328                                              spec.ipv6->hdr.dst_addr);
3329                                 mnl_attr_put(nlh, tunnel_outer ?
3330                                              TCA_FLOWER_KEY_ENC_IPV6_DST_MASK :
3331                                              TCA_FLOWER_KEY_IPV6_DST_MASK,
3332                                              IPV6_ADDR_LEN,
3333                                              mask.ipv6->hdr.dst_addr);
3334                         }
3335                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3336                         break;
3337                 }
3338                 case RTE_FLOW_ITEM_TYPE_UDP:
3339                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3340                                       MLX5_FLOW_LAYER_INNER_L4_UDP :
3341                                       MLX5_FLOW_LAYER_OUTER_L4_UDP;
3342                         mask.udp = flow_tcf_item_mask
3343                                 (items, &rte_flow_item_udp_mask,
3344                                  &flow_tcf_mask_supported.udp,
3345                                  &flow_tcf_mask_empty.udp,
3346                                  sizeof(flow_tcf_mask_supported.udp),
3347                                  error);
3348                         assert(mask.udp);
3349                         spec.udp = items->spec;
3350                         if (!tunnel_outer) {
3351                                 if (!ip_proto_set)
3352                                         mnl_attr_put_u8
3353                                                 (nlh, TCA_FLOWER_KEY_IP_PROTO,
3354                                                 IPPROTO_UDP);
3355                                 if (mask.udp == &flow_tcf_mask_empty.udp)
3356                                         break;
3357                         } else {
3358                                 assert(mask.udp != &flow_tcf_mask_empty.udp);
3359                                 decap.vxlan->udp_port =
3360                                         rte_be_to_cpu_16
3361                                                 (spec.udp->hdr.dst_port);
3362                         }
3363                         if (mask.udp->hdr.src_port) {
3364                                 mnl_attr_put_u16
3365                                         (nlh, tunnel_outer ?
3366                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT :
3367                                          TCA_FLOWER_KEY_UDP_SRC,
3368                                          spec.udp->hdr.src_port);
3369                                 mnl_attr_put_u16
3370                                         (nlh, tunnel_outer ?
3371                                          TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK :
3372                                          TCA_FLOWER_KEY_UDP_SRC_MASK,
3373                                          mask.udp->hdr.src_port);
3374                         }
3375                         if (mask.udp->hdr.dst_port) {
3376                                 mnl_attr_put_u16
3377                                         (nlh, tunnel_outer ?
3378                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT :
3379                                          TCA_FLOWER_KEY_UDP_DST,
3380                                          spec.udp->hdr.dst_port);
3381                                 mnl_attr_put_u16
3382                                         (nlh, tunnel_outer ?
3383                                          TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK :
3384                                          TCA_FLOWER_KEY_UDP_DST_MASK,
3385                                          mask.udp->hdr.dst_port);
3386                         }
3387                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3388                         break;
3389                 case RTE_FLOW_ITEM_TYPE_TCP:
3390                         item_flags |= (item_flags & MLX5_FLOW_LAYER_TUNNEL) ?
3391                                       MLX5_FLOW_LAYER_INNER_L4_TCP :
3392                                       MLX5_FLOW_LAYER_OUTER_L4_TCP;
3393                         mask.tcp = flow_tcf_item_mask
3394                                 (items, &rte_flow_item_tcp_mask,
3395                                  &flow_tcf_mask_supported.tcp,
3396                                  &flow_tcf_mask_empty.tcp,
3397                                  sizeof(flow_tcf_mask_supported.tcp),
3398                                  error);
3399                         assert(mask.tcp);
3400                         if (!ip_proto_set)
3401                                 mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
3402                                                 IPPROTO_TCP);
3403                         if (mask.tcp == &flow_tcf_mask_empty.tcp)
3404                                 break;
3405                         spec.tcp = items->spec;
3406                         if (mask.tcp->hdr.src_port) {
3407                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_SRC,
3408                                                  spec.tcp->hdr.src_port);
3409                                 mnl_attr_put_u16(nlh,
3410                                                  TCA_FLOWER_KEY_TCP_SRC_MASK,
3411                                                  mask.tcp->hdr.src_port);
3412                         }
3413                         if (mask.tcp->hdr.dst_port) {
3414                                 mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_DST,
3415                                                  spec.tcp->hdr.dst_port);
3416                                 mnl_attr_put_u16(nlh,
3417                                                  TCA_FLOWER_KEY_TCP_DST_MASK,
3418                                                  mask.tcp->hdr.dst_port);
3419                         }
3420                         if (mask.tcp->hdr.tcp_flags) {
3421                                 mnl_attr_put_u16
3422                                         (nlh,
3423                                          TCA_FLOWER_KEY_TCP_FLAGS,
3424                                          rte_cpu_to_be_16
3425                                                 (spec.tcp->hdr.tcp_flags));
3426                                 mnl_attr_put_u16
3427                                         (nlh,
3428                                          TCA_FLOWER_KEY_TCP_FLAGS_MASK,
3429                                          rte_cpu_to_be_16
3430                                                 (mask.tcp->hdr.tcp_flags));
3431                         }
3432                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3433                         break;
3434                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3435                         assert(decap.vxlan);
3436                         tunnel_outer = 0;
3437                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
3438                         spec.vxlan = items->spec;
3439                         mnl_attr_put_u32(nlh,
3440                                          TCA_FLOWER_KEY_ENC_KEY_ID,
3441                                          vxlan_vni_as_be32(spec.vxlan->vni));
3442                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3443                         break;
3444                 default:
3445                         return rte_flow_error_set(error, ENOTSUP,
3446                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3447                                                   NULL, "item not supported");
3448                 }
3449         }
3450         na_flower_act = mnl_attr_nest_start(nlh, TCA_FLOWER_ACT);
3451         na_act_index_cur = 1;
3452         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3453                 struct nlattr *na_act_index;
3454                 struct nlattr *na_act;
3455                 unsigned int vlan_act;
3456                 unsigned int i;
3457
3458                 switch (actions->type) {
3459                 case RTE_FLOW_ACTION_TYPE_VOID:
3460                         break;
3461                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3462                         conf.port_id = actions->conf;
3463                         if (conf.port_id->original)
3464                                 i = 0;
3465                         else
3466                                 for (i = 0; ptoi[i].ifindex; ++i)
3467                                         if (ptoi[i].port_id == conf.port_id->id)
3468                                                 break;
3469                         assert(ptoi[i].ifindex);
3470                         na_act_index =
3471                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3472                         assert(na_act_index);
3473                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "mirred");
3474                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3475                         assert(na_act);
3476                         if (encap.hdr) {
3477                                 assert(dev_flow->tcf.tunnel);
3478                                 dev_flow->tcf.tunnel->ifindex_ptr =
3479                                         &((struct tc_mirred *)
3480                                         mnl_attr_get_payload
3481                                         (mnl_nlmsg_get_payload_tail
3482                                                 (nlh)))->ifindex;
3483                         }
3484                         mnl_attr_put(nlh, TCA_MIRRED_PARMS,
3485                                      sizeof(struct tc_mirred),
3486                                      &(struct tc_mirred){
3487                                         .action = TC_ACT_STOLEN,
3488                                         .eaction = TCA_EGRESS_REDIR,
3489                                         .ifindex = ptoi[i].ifindex,
3490                                      });
3491                         mnl_attr_nest_end(nlh, na_act);
3492                         mnl_attr_nest_end(nlh, na_act_index);
3493                         break;
3494                 case RTE_FLOW_ACTION_TYPE_JUMP:
3495                         conf.jump = actions->conf;
3496                         na_act_index =
3497                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3498                         assert(na_act_index);
3499                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3500                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3501                         assert(na_act);
3502                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3503                                      sizeof(struct tc_gact),
3504                                      &(struct tc_gact){
3505                                         .action = TC_ACT_GOTO_CHAIN |
3506                                                   conf.jump->group,
3507                                      });
3508                         mnl_attr_nest_end(nlh, na_act);
3509                         mnl_attr_nest_end(nlh, na_act_index);
3510                         break;
3511                 case RTE_FLOW_ACTION_TYPE_DROP:
3512                         na_act_index =
3513                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3514                         assert(na_act_index);
3515                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
3516                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3517                         assert(na_act);
3518                         mnl_attr_put(nlh, TCA_GACT_PARMS,
3519                                      sizeof(struct tc_gact),
3520                                      &(struct tc_gact){
3521                                         .action = TC_ACT_SHOT,
3522                                      });
3523                         mnl_attr_nest_end(nlh, na_act);
3524                         mnl_attr_nest_end(nlh, na_act_index);
3525                         break;
3526                 case RTE_FLOW_ACTION_TYPE_COUNT:
3527                         /*
3528                          * Driver adds the count action implicitly for
3529                          * each rule it creates.
3530                          */
3531                         ret = flow_tcf_translate_action_count(dev,
3532                                                               dev_flow, error);
3533                         if (ret < 0)
3534                                 return ret;
3535                         break;
3536                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3537                         conf.of_push_vlan = NULL;
3538                         vlan_act = TCA_VLAN_ACT_POP;
3539                         goto action_of_vlan;
3540                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3541                         conf.of_push_vlan = actions->conf;
3542                         vlan_act = TCA_VLAN_ACT_PUSH;
3543                         goto action_of_vlan;
3544                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3545                         conf.of_set_vlan_vid = actions->conf;
3546                         if (na_vlan_id)
3547                                 goto override_na_vlan_id;
3548                         vlan_act = TCA_VLAN_ACT_MODIFY;
3549                         goto action_of_vlan;
3550                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3551                         conf.of_set_vlan_pcp = actions->conf;
3552                         if (na_vlan_priority)
3553                                 goto override_na_vlan_priority;
3554                         vlan_act = TCA_VLAN_ACT_MODIFY;
3555                         goto action_of_vlan;
3556 action_of_vlan:
3557                         na_act_index =
3558                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3559                         assert(na_act_index);
3560                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "vlan");
3561                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3562                         assert(na_act);
3563                         mnl_attr_put(nlh, TCA_VLAN_PARMS,
3564                                      sizeof(struct tc_vlan),
3565                                      &(struct tc_vlan){
3566                                         .action = TC_ACT_PIPE,
3567                                         .v_action = vlan_act,
3568                                      });
3569                         if (vlan_act == TCA_VLAN_ACT_POP) {
3570                                 mnl_attr_nest_end(nlh, na_act);
3571                                 mnl_attr_nest_end(nlh, na_act_index);
3572                                 break;
3573                         }
3574                         if (vlan_act == TCA_VLAN_ACT_PUSH)
3575                                 mnl_attr_put_u16(nlh,
3576                                                  TCA_VLAN_PUSH_VLAN_PROTOCOL,
3577                                                  conf.of_push_vlan->ethertype);
3578                         na_vlan_id = mnl_nlmsg_get_payload_tail(nlh);
3579                         mnl_attr_put_u16(nlh, TCA_VLAN_PAD, 0);
3580                         na_vlan_priority = mnl_nlmsg_get_payload_tail(nlh);
3581                         mnl_attr_put_u8(nlh, TCA_VLAN_PAD, 0);
3582                         mnl_attr_nest_end(nlh, na_act);
3583                         mnl_attr_nest_end(nlh, na_act_index);
3584                         if (actions->type ==
3585                             RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
3586 override_na_vlan_id:
3587                                 na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
3588                                 *(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
3589                                         rte_be_to_cpu_16
3590                                         (conf.of_set_vlan_vid->vlan_vid);
3591                         } else if (actions->type ==
3592                                    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
3593 override_na_vlan_priority:
3594                                 na_vlan_priority->nla_type =
3595                                         TCA_VLAN_PUSH_VLAN_PRIORITY;
3596                                 *(uint8_t *)mnl_attr_get_payload
3597                                         (na_vlan_priority) =
3598                                         conf.of_set_vlan_pcp->vlan_pcp;
3599                         }
3600                         break;
3601                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3602                         assert(decap.vxlan);
3603                         assert(dev_flow->tcf.tunnel);
3604                         dev_flow->tcf.tunnel->ifindex_ptr =
3605                                 (unsigned int *)&tcm->tcm_ifindex;
3606                         na_act_index =
3607                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3608                         assert(na_act_index);
3609                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3610                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3611                         assert(na_act);
3612                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3613                                 sizeof(struct tc_tunnel_key),
3614                                 &(struct tc_tunnel_key){
3615                                         .action = TC_ACT_PIPE,
3616                                         .t_action = TCA_TUNNEL_KEY_ACT_RELEASE,
3617                                         });
3618                         mnl_attr_nest_end(nlh, na_act);
3619                         mnl_attr_nest_end(nlh, na_act_index);
3620                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3621                         break;
3622                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3623                         assert(encap.vxlan);
3624                         flow_tcf_vxlan_encap_parse(actions, encap.vxlan);
3625                         na_act_index =
3626                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3627                         assert(na_act_index);
3628                         mnl_attr_put_strz(nlh, TCA_ACT_KIND, "tunnel_key");
3629                         na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
3630                         assert(na_act);
3631                         mnl_attr_put(nlh, TCA_TUNNEL_KEY_PARMS,
3632                                 sizeof(struct tc_tunnel_key),
3633                                 &(struct tc_tunnel_key){
3634                                         .action = TC_ACT_PIPE,
3635                                         .t_action = TCA_TUNNEL_KEY_ACT_SET,
3636                                         });
3637                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_UDP_DST)
3638                                 mnl_attr_put_u16(nlh,
3639                                          TCA_TUNNEL_KEY_ENC_DST_PORT,
3640                                          encap.vxlan->udp.dst);
3641                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_SRC)
3642                                 mnl_attr_put_u32(nlh,
3643                                          TCA_TUNNEL_KEY_ENC_IPV4_SRC,
3644                                          encap.vxlan->ipv4.src);
3645                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV4_DST)
3646                                 mnl_attr_put_u32(nlh,
3647                                          TCA_TUNNEL_KEY_ENC_IPV4_DST,
3648                                          encap.vxlan->ipv4.dst);
3649                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_SRC)
3650                                 mnl_attr_put(nlh,
3651                                          TCA_TUNNEL_KEY_ENC_IPV6_SRC,
3652                                          sizeof(encap.vxlan->ipv6.src),
3653                                          &encap.vxlan->ipv6.src);
3654                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_IPV6_DST)
3655                                 mnl_attr_put(nlh,
3656                                          TCA_TUNNEL_KEY_ENC_IPV6_DST,
3657                                          sizeof(encap.vxlan->ipv6.dst),
3658                                          &encap.vxlan->ipv6.dst);
3659                         if (encap.vxlan->mask & FLOW_TCF_ENCAP_VXLAN_VNI)
3660                                 mnl_attr_put_u32(nlh,
3661                                          TCA_TUNNEL_KEY_ENC_KEY_ID,
3662                                          vxlan_vni_as_be32
3663                                                 (encap.vxlan->vxlan.vni));
3664                         mnl_attr_put_u8(nlh, TCA_TUNNEL_KEY_NO_CSUM, 0);
3665                         mnl_attr_nest_end(nlh, na_act);
3666                         mnl_attr_nest_end(nlh, na_act_index);
3667                         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3668                         break;
3669                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3670                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3671                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3672                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3673                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3674                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3675                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3676                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3677                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3678                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3679                         na_act_index =
3680                                 mnl_attr_nest_start(nlh, na_act_index_cur++);
3681                         flow_tcf_create_pedit_mnl_msg(nlh,
3682                                                       &actions, item_flags);
3683                         mnl_attr_nest_end(nlh, na_act_index);
3684                         break;
3685                 default:
3686                         return rte_flow_error_set(error, ENOTSUP,
3687                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3688                                                   actions,
3689                                                   "action not supported");
3690                 }
3691         }
3692         assert(na_flower);
3693         assert(na_flower_act);
3694         mnl_attr_nest_end(nlh, na_flower_act);
3695         dev_flow->tcf.ptc_flags = mnl_attr_get_payload
3696                                         (mnl_nlmsg_get_payload_tail(nlh));
3697         mnl_attr_put_u32(nlh, TCA_FLOWER_FLAGS, decap.vxlan ?
3698                                                 0 : TCA_CLS_FLAGS_SKIP_SW);
3699         mnl_attr_nest_end(nlh, na_flower);
3700         if (dev_flow->tcf.tunnel && dev_flow->tcf.tunnel->ifindex_ptr)
3701                 dev_flow->tcf.tunnel->ifindex_org =
3702                         *dev_flow->tcf.tunnel->ifindex_ptr;
3703         assert(dev_flow->tcf.nlsize >= nlh->nlmsg_len);
3704         return 0;
3705 }
3706
3707 /**
3708  * Send Netlink message with acknowledgment.
3709  *
3710  * @param tcf
3711  *   Flow context to use.
3712  * @param nlh
3713  *   Message to send. This function always raises the NLM_F_ACK flag before
3714  *   sending.
3715  * @param[in] cb
3716  *   Callback handler for received message.
3717  * @param[in] arg
3718  *   Context pointer for callback handler.
3719  *
3720  * @return
3721  *   0 on success, a negative errno value otherwise and rte_errno is set.
3722  */
3723 static int
3724 flow_tcf_nl_ack(struct mlx5_flow_tcf_context *tcf,
3725                 struct nlmsghdr *nlh,
3726                 mnl_cb_t cb, void *arg)
3727 {
3728         unsigned int portid = mnl_socket_get_portid(tcf->nl);
3729         uint32_t seq = tcf->seq++;
3730         int ret, err = 0;
3731
3732         assert(tcf->nl);
3733         assert(tcf->buf);
3734         if (!seq) {
3735                 /* seq 0 is reserved for kernel event-driven notifications. */
3736                 seq = tcf->seq++;
3737         }
3738         nlh->nlmsg_seq = seq;
3739         nlh->nlmsg_flags |= NLM_F_ACK;
3740         ret = mnl_socket_sendto(tcf->nl, nlh, nlh->nlmsg_len);
3741         if (ret <= 0) {
3742                 /* Message send error occurres. */
3743                 rte_errno = errno;
3744                 return -rte_errno;
3745         }
3746         nlh = (struct nlmsghdr *)(tcf->buf);
3747         /*
3748          * The following loop postpones non-fatal errors until multipart
3749          * messages are complete.
3750          */
3751         while (true) {
3752                 ret = mnl_socket_recvfrom(tcf->nl, tcf->buf, tcf->buf_size);
3753                 if (ret < 0) {
3754                         err = errno;
3755                         /*
3756                          * In case of overflow Will receive till
3757                          * end of multipart message. We may lost part
3758                          * of reply messages but mark and return an error.
3759                          */
3760                         if (err != ENOSPC ||
3761                             !(nlh->nlmsg_flags & NLM_F_MULTI) ||
3762                             nlh->nlmsg_type == NLMSG_DONE)
3763                                 break;
3764                 } else {
3765                         ret = mnl_cb_run(nlh, ret, seq, portid, cb, arg);
3766                         if (!ret) {
3767                                 /*
3768                                  * libmnl returns 0 if DONE or
3769                                  * success ACK message found.
3770                                  */
3771                                 break;
3772                         }
3773                         if (ret < 0) {
3774                                 /*
3775                                  * ACK message with error found
3776                                  * or some error occurred.
3777                                  */
3778                                 err = errno;
3779                                 break;
3780                         }
3781                         /* We should continue receiving. */
3782                 }
3783         }
3784         if (!err)
3785                 return 0;
3786         rte_errno = err;
3787         return -err;
3788 }
3789
3790 #define MNL_BUF_EXTRA_SPACE 16
3791 #define MNL_REQUEST_SIZE_MIN 256
3792 #define MNL_REQUEST_SIZE_MAX 2048
3793 #define MNL_REQUEST_SIZE RTE_MIN(RTE_MAX(sysconf(_SC_PAGESIZE), \
3794                                  MNL_REQUEST_SIZE_MIN), MNL_REQUEST_SIZE_MAX)
3795
3796 /* Data structures used by flow_tcf_xxx_cb() routines. */
3797 struct tcf_nlcb_buf {
3798         LIST_ENTRY(tcf_nlcb_buf) next;
3799         uint32_t size;
3800         alignas(struct nlmsghdr)
3801         uint8_t msg[]; /**< Netlink message data. */
3802 };
3803
3804 struct tcf_nlcb_context {
3805         unsigned int ifindex; /**< Base interface index. */
3806         uint32_t bufsize;
3807         LIST_HEAD(, tcf_nlcb_buf) nlbuf;
3808 };
3809
3810 /**
3811  * Allocate space for netlink command in buffer list
3812  *
3813  * @param[in, out] ctx
3814  *   Pointer to callback context with command buffers list.
3815  * @param[in] size
3816  *   Required size of data buffer to be allocated.
3817  *
3818  * @return
3819  *   Pointer to allocated memory, aligned as message header.
3820  *   NULL if some error occurred.
3821  */
3822 static struct nlmsghdr *
3823 flow_tcf_alloc_nlcmd(struct tcf_nlcb_context *ctx, uint32_t size)
3824 {
3825         struct tcf_nlcb_buf *buf;
3826         struct nlmsghdr *nlh;
3827
3828         size = NLMSG_ALIGN(size);
3829         buf = LIST_FIRST(&ctx->nlbuf);
3830         if (buf && (buf->size + size) <= ctx->bufsize) {
3831                 nlh = (struct nlmsghdr *)&buf->msg[buf->size];
3832                 buf->size += size;
3833                 return nlh;
3834         }
3835         if (size > ctx->bufsize) {
3836                 DRV_LOG(WARNING, "netlink: too long command buffer requested");
3837                 return NULL;
3838         }
3839         buf = rte_malloc(__func__,
3840                         ctx->bufsize + sizeof(struct tcf_nlcb_buf),
3841                         alignof(struct tcf_nlcb_buf));
3842         if (!buf) {
3843                 DRV_LOG(WARNING, "netlink: no memory for command buffer");
3844                 return NULL;
3845         }
3846         LIST_INSERT_HEAD(&ctx->nlbuf, buf, next);
3847         buf->size = size;
3848         nlh = (struct nlmsghdr *)&buf->msg[0];
3849         return nlh;
3850 }
3851
3852 /**
3853  * Send the buffers with prepared netlink commands. Scans the list and
3854  * sends all found buffers. Buffers are sent and freed anyway in order
3855  * to prevent memory leakage if some every message in received packet.
3856  *
3857  * @param[in] tcf
3858  *   Context object initialized by mlx5_flow_tcf_context_create().
3859  * @param[in, out] ctx
3860  *   Pointer to callback context with command buffers list.
3861  *
3862  * @return
3863  *   Zero value on success, negative errno value otherwise
3864  *   and rte_errno is set.
3865  */
3866 static int
3867 flow_tcf_send_nlcmd(struct mlx5_flow_tcf_context *tcf,
3868                     struct tcf_nlcb_context *ctx)
3869 {
3870         struct tcf_nlcb_buf *bc = LIST_FIRST(&ctx->nlbuf);
3871         int ret = 0;
3872
3873         while (bc) {
3874                 struct tcf_nlcb_buf *bn = LIST_NEXT(bc, next);
3875                 struct nlmsghdr *nlh;
3876                 uint32_t msg = 0;
3877                 int rc;
3878
3879                 while (msg < bc->size) {
3880                         /*
3881                          * Send Netlink commands from buffer in one by one
3882                          * fashion. If we send multiple rule deletion commands
3883                          * in one Netlink message and some error occurs it may
3884                          * cause multiple ACK error messages and break sequence
3885                          * numbers of Netlink communication, because we expect
3886                          * the only one ACK reply.
3887                          */
3888                         assert((bc->size - msg) >= sizeof(struct nlmsghdr));
3889                         nlh = (struct nlmsghdr *)&bc->msg[msg];
3890                         assert((bc->size - msg) >= nlh->nlmsg_len);
3891                         msg += nlh->nlmsg_len;
3892                         rc = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
3893                         if (rc) {
3894                                 DRV_LOG(WARNING,
3895                                         "netlink: cleanup error %d", rc);
3896                                 if (!ret)
3897                                         ret = rc;
3898                         }
3899                 }
3900                 rte_free(bc);
3901                 bc = bn;
3902         }
3903         LIST_INIT(&ctx->nlbuf);
3904         return ret;
3905 }
3906
3907 /**
3908  * Collect local IP address rules with scope link attribute  on specified
3909  * network device. This is callback routine called by libmnl mnl_cb_run()
3910  * in loop for every message in received packet.
3911  *
3912  * @param[in] nlh
3913  *   Pointer to reply header.
3914  * @param[in, out] arg
3915  *   Opaque data pointer for this callback.
3916  *
3917  * @return
3918  *   A positive, nonzero value on success, negative errno value otherwise
3919  *   and rte_errno is set.
3920  */
3921 static int
3922 flow_tcf_collect_local_cb(const struct nlmsghdr *nlh, void *arg)
3923 {
3924         struct tcf_nlcb_context *ctx = arg;
3925         struct nlmsghdr *cmd;
3926         struct ifaddrmsg *ifa;
3927         struct nlattr *na;
3928         struct nlattr *na_local = NULL;
3929         struct nlattr *na_peer = NULL;
3930         unsigned char family;
3931         uint32_t size;
3932
3933         if (nlh->nlmsg_type != RTM_NEWADDR) {
3934                 rte_errno = EINVAL;
3935                 return -rte_errno;
3936         }
3937         ifa = mnl_nlmsg_get_payload(nlh);
3938         family = ifa->ifa_family;
3939         if (ifa->ifa_index != ctx->ifindex ||
3940             ifa->ifa_scope != RT_SCOPE_LINK ||
3941             !(ifa->ifa_flags & IFA_F_PERMANENT) ||
3942             (family != AF_INET && family != AF_INET6))
3943                 return 1;
3944         mnl_attr_for_each(na, nlh, sizeof(*ifa)) {
3945                 switch (mnl_attr_get_type(na)) {
3946                 case IFA_LOCAL:
3947                         na_local = na;
3948                         break;
3949                 case IFA_ADDRESS:
3950                         na_peer = na;
3951                         break;
3952                 }
3953                 if (na_local && na_peer)
3954                         break;
3955         }
3956         if (!na_local || !na_peer)
3957                 return 1;
3958         /* Local rule found with scope link, permanent and assigned peer. */
3959         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
3960                MNL_ALIGN(sizeof(struct ifaddrmsg)) +
3961                (family == AF_INET6 ? 2 * SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
3962                                    : 2 * SZ_NLATTR_TYPE_OF(uint32_t));
3963         cmd = flow_tcf_alloc_nlcmd(ctx, size);
3964         if (!cmd) {
3965                 rte_errno = ENOMEM;
3966                 return -rte_errno;
3967         }
3968         cmd = mnl_nlmsg_put_header(cmd);
3969         cmd->nlmsg_type = RTM_DELADDR;
3970         cmd->nlmsg_flags = NLM_F_REQUEST;
3971         ifa = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifa));
3972         ifa->ifa_flags = IFA_F_PERMANENT;
3973         ifa->ifa_scope = RT_SCOPE_LINK;
3974         ifa->ifa_index = ctx->ifindex;
3975         if (family == AF_INET) {
3976                 ifa->ifa_family = AF_INET;
3977                 ifa->ifa_prefixlen = 32;
3978                 mnl_attr_put_u32(cmd, IFA_LOCAL, mnl_attr_get_u32(na_local));
3979                 mnl_attr_put_u32(cmd, IFA_ADDRESS, mnl_attr_get_u32(na_peer));
3980         } else {
3981                 ifa->ifa_family = AF_INET6;
3982                 ifa->ifa_prefixlen = 128;
3983                 mnl_attr_put(cmd, IFA_LOCAL, IPV6_ADDR_LEN,
3984                         mnl_attr_get_payload(na_local));
3985                 mnl_attr_put(cmd, IFA_ADDRESS, IPV6_ADDR_LEN,
3986                         mnl_attr_get_payload(na_peer));
3987         }
3988         assert(size == cmd->nlmsg_len);
3989         return 1;
3990 }
3991
3992 /**
3993  * Cleanup the local IP addresses on outer interface.
3994  *
3995  * @param[in] tcf
3996  *   Context object initialized by mlx5_flow_tcf_context_create().
3997  * @param[in] ifindex
3998  *   Network inferface index to perform cleanup.
3999  */
4000 static void
4001 flow_tcf_encap_local_cleanup(struct mlx5_flow_tcf_context *tcf,
4002                             unsigned int ifindex)
4003 {
4004         struct nlmsghdr *nlh;
4005         struct ifaddrmsg *ifa;
4006         struct tcf_nlcb_context ctx = {
4007                 .ifindex = ifindex,
4008                 .bufsize = MNL_REQUEST_SIZE,
4009                 .nlbuf = LIST_HEAD_INITIALIZER(),
4010         };
4011         int ret;
4012
4013         assert(ifindex);
4014         /*
4015          * Seek and destroy leftovers of local IP addresses with
4016          * matching properties "scope link".
4017          */
4018         nlh = mnl_nlmsg_put_header(tcf->buf);
4019         nlh->nlmsg_type = RTM_GETADDR;
4020         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4021         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4022         ifa->ifa_family = AF_UNSPEC;
4023         ifa->ifa_index = ifindex;
4024         ifa->ifa_scope = RT_SCOPE_LINK;
4025         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_local_cb, &ctx);
4026         if (ret)
4027                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4028         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4029         if (ret)
4030                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4031 }
4032
4033 /**
4034  * Collect neigh permament rules on specified network device.
4035  * This is callback routine called by libmnl mnl_cb_run() in loop for
4036  * every message in received packet.
4037  *
4038  * @param[in] nlh
4039  *   Pointer to reply header.
4040  * @param[in, out] arg
4041  *   Opaque data pointer for this callback.
4042  *
4043  * @return
4044  *   A positive, nonzero value on success, negative errno value otherwise
4045  *   and rte_errno is set.
4046  */
4047 static int
4048 flow_tcf_collect_neigh_cb(const struct nlmsghdr *nlh, void *arg)
4049 {
4050         struct tcf_nlcb_context *ctx = arg;
4051         struct nlmsghdr *cmd;
4052         struct ndmsg *ndm;
4053         struct nlattr *na;
4054         struct nlattr *na_ip = NULL;
4055         struct nlattr *na_mac = NULL;
4056         unsigned char family;
4057         uint32_t size;
4058
4059         if (nlh->nlmsg_type != RTM_NEWNEIGH) {
4060                 rte_errno = EINVAL;
4061                 return -rte_errno;
4062         }
4063         ndm = mnl_nlmsg_get_payload(nlh);
4064         family = ndm->ndm_family;
4065         if (ndm->ndm_ifindex != (int)ctx->ifindex ||
4066            !(ndm->ndm_state & NUD_PERMANENT) ||
4067            (family != AF_INET && family != AF_INET6))
4068                 return 1;
4069         mnl_attr_for_each(na, nlh, sizeof(*ndm)) {
4070                 switch (mnl_attr_get_type(na)) {
4071                 case NDA_DST:
4072                         na_ip = na;
4073                         break;
4074                 case NDA_LLADDR:
4075                         na_mac = na;
4076                         break;
4077                 }
4078                 if (na_mac && na_ip)
4079                         break;
4080         }
4081         if (!na_mac || !na_ip)
4082                 return 1;
4083         /* Neigh rule with permenent attribute found. */
4084         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4085                MNL_ALIGN(sizeof(struct ndmsg)) +
4086                SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) +
4087                (family == AF_INET6 ? SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
4088                                    : SZ_NLATTR_TYPE_OF(uint32_t));
4089         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4090         if (!cmd) {
4091                 rte_errno = ENOMEM;
4092                 return -rte_errno;
4093         }
4094         cmd = mnl_nlmsg_put_header(cmd);
4095         cmd->nlmsg_type = RTM_DELNEIGH;
4096         cmd->nlmsg_flags = NLM_F_REQUEST;
4097         ndm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ndm));
4098         ndm->ndm_ifindex = ctx->ifindex;
4099         ndm->ndm_state = NUD_PERMANENT;
4100         ndm->ndm_flags = 0;
4101         ndm->ndm_type = 0;
4102         if (family == AF_INET) {
4103                 ndm->ndm_family = AF_INET;
4104                 mnl_attr_put_u32(cmd, NDA_DST, mnl_attr_get_u32(na_ip));
4105         } else {
4106                 ndm->ndm_family = AF_INET6;
4107                 mnl_attr_put(cmd, NDA_DST, IPV6_ADDR_LEN,
4108                              mnl_attr_get_payload(na_ip));
4109         }
4110         mnl_attr_put(cmd, NDA_LLADDR, ETHER_ADDR_LEN,
4111                      mnl_attr_get_payload(na_mac));
4112         assert(size == cmd->nlmsg_len);
4113         return 1;
4114 }
4115
4116 /**
4117  * Cleanup the neigh rules on outer interface.
4118  *
4119  * @param[in] tcf
4120  *   Context object initialized by mlx5_flow_tcf_context_create().
4121  * @param[in] ifindex
4122  *   Network inferface index to perform cleanup.
4123  */
4124 static void
4125 flow_tcf_encap_neigh_cleanup(struct mlx5_flow_tcf_context *tcf,
4126                             unsigned int ifindex)
4127 {
4128         struct nlmsghdr *nlh;
4129         struct ndmsg *ndm;
4130         struct tcf_nlcb_context ctx = {
4131                 .ifindex = ifindex,
4132                 .bufsize = MNL_REQUEST_SIZE,
4133                 .nlbuf = LIST_HEAD_INITIALIZER(),
4134         };
4135         int ret;
4136
4137         assert(ifindex);
4138         /* Seek and destroy leftovers of neigh rules. */
4139         nlh = mnl_nlmsg_put_header(tcf->buf);
4140         nlh->nlmsg_type = RTM_GETNEIGH;
4141         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4142         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4143         ndm->ndm_family = AF_UNSPEC;
4144         ndm->ndm_ifindex = ifindex;
4145         ndm->ndm_state = NUD_PERMANENT;
4146         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_neigh_cb, &ctx);
4147         if (ret)
4148                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4149         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4150         if (ret)
4151                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4152 }
4153
4154 /**
4155  * Collect indices of VXLAN encap/decap interfaces associated with device.
4156  * This is callback routine called by libmnl mnl_cb_run() in loop for
4157  * every message in received packet.
4158  *
4159  * @param[in] nlh
4160  *   Pointer to reply header.
4161  * @param[in, out] arg
4162  *   Opaque data pointer for this callback.
4163  *
4164  * @return
4165  *   A positive, nonzero value on success, negative errno value otherwise
4166  *   and rte_errno is set.
4167  */
4168 static int
4169 flow_tcf_collect_vxlan_cb(const struct nlmsghdr *nlh, void *arg)
4170 {
4171         struct tcf_nlcb_context *ctx = arg;
4172         struct nlmsghdr *cmd;
4173         struct ifinfomsg *ifm;
4174         struct nlattr *na;
4175         struct nlattr *na_info = NULL;
4176         struct nlattr *na_vxlan = NULL;
4177         bool found = false;
4178         unsigned int vxindex;
4179         uint32_t size;
4180
4181         if (nlh->nlmsg_type != RTM_NEWLINK) {
4182                 rte_errno = EINVAL;
4183                 return -rte_errno;
4184         }
4185         ifm = mnl_nlmsg_get_payload(nlh);
4186         if (!ifm->ifi_index) {
4187                 rte_errno = EINVAL;
4188                 return -rte_errno;
4189         }
4190         mnl_attr_for_each(na, nlh, sizeof(*ifm))
4191                 if (mnl_attr_get_type(na) == IFLA_LINKINFO) {
4192                         na_info = na;
4193                         break;
4194                 }
4195         if (!na_info)
4196                 return 1;
4197         mnl_attr_for_each_nested(na, na_info) {
4198                 switch (mnl_attr_get_type(na)) {
4199                 case IFLA_INFO_KIND:
4200                         if (!strncmp("vxlan", mnl_attr_get_str(na),
4201                                      mnl_attr_get_len(na)))
4202                                 found = true;
4203                         break;
4204                 case IFLA_INFO_DATA:
4205                         na_vxlan = na;
4206                         break;
4207                 }
4208                 if (found && na_vxlan)
4209                         break;
4210         }
4211         if (!found || !na_vxlan)
4212                 return 1;
4213         found = false;
4214         mnl_attr_for_each_nested(na, na_vxlan) {
4215                 if (mnl_attr_get_type(na) == IFLA_VXLAN_LINK &&
4216                     mnl_attr_get_u32(na) == ctx->ifindex) {
4217                         found = true;
4218                         break;
4219                 }
4220         }
4221         if (!found)
4222                 return 1;
4223         /* Attached VXLAN device found, store the command to delete. */
4224         vxindex = ifm->ifi_index;
4225         size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
4226                MNL_ALIGN(sizeof(struct ifinfomsg));
4227         cmd = flow_tcf_alloc_nlcmd(ctx, size);
4228         if (!cmd) {
4229                 rte_errno = ENOMEM;
4230                 return -rte_errno;
4231         }
4232         cmd = mnl_nlmsg_put_header(cmd);
4233         cmd->nlmsg_type = RTM_DELLINK;
4234         cmd->nlmsg_flags = NLM_F_REQUEST;
4235         ifm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifm));
4236         ifm->ifi_family = AF_UNSPEC;
4237         ifm->ifi_index = vxindex;
4238         assert(size == cmd->nlmsg_len);
4239         return 1;
4240 }
4241
4242 /**
4243  * Cleanup the outer interface. Removes all found vxlan devices
4244  * attached to specified index, flushes the meigh and local IP
4245  * datavase.
4246  *
4247  * @param[in] tcf
4248  *   Context object initialized by mlx5_flow_tcf_context_create().
4249  * @param[in] ifindex
4250  *   Network inferface index to perform cleanup.
4251  */
4252 static void
4253 flow_tcf_encap_iface_cleanup(struct mlx5_flow_tcf_context *tcf,
4254                             unsigned int ifindex)
4255 {
4256         struct nlmsghdr *nlh;
4257         struct ifinfomsg *ifm;
4258         struct tcf_nlcb_context ctx = {
4259                 .ifindex = ifindex,
4260                 .bufsize = MNL_REQUEST_SIZE,
4261                 .nlbuf = LIST_HEAD_INITIALIZER(),
4262         };
4263         int ret;
4264
4265         assert(ifindex);
4266         /*
4267          * Seek and destroy leftover VXLAN encap/decap interfaces with
4268          * matching properties.
4269          */
4270         nlh = mnl_nlmsg_put_header(tcf->buf);
4271         nlh->nlmsg_type = RTM_GETLINK;
4272         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
4273         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4274         ifm->ifi_family = AF_UNSPEC;
4275         ret = flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_vxlan_cb, &ctx);
4276         if (ret)
4277                 DRV_LOG(WARNING, "netlink: query device list error %d", ret);
4278         ret = flow_tcf_send_nlcmd(tcf, &ctx);
4279         if (ret)
4280                 DRV_LOG(WARNING, "netlink: device delete error %d", ret);
4281 }
4282
4283 /**
4284  * Emit Netlink message to add/remove local address to the outer device.
4285  * The address being added is visible within the link only (scope link).
4286  *
4287  * Note that an implicit route is maintained by the kernel due to the
4288  * presence of a peer address (IFA_ADDRESS).
4289  *
4290  * These rules are used for encapsultion only and allow to assign
4291  * the outer tunnel source IP address.
4292  *
4293  * @param[in] tcf
4294  *   Libmnl socket context object.
4295  * @param[in] encap
4296  *   Encapsulation properties (source address and its peer).
4297  * @param[in] ifindex
4298  *   Network interface to apply rule.
4299  * @param[in] enable
4300  *   Toggle between add and remove.
4301  * @param[out] error
4302  *   Perform verbose error reporting if not NULL.
4303  *
4304  * @return
4305  *   0 on success, a negative errno value otherwise and rte_errno is set.
4306  */
4307 static int
4308 flow_tcf_rule_local(struct mlx5_flow_tcf_context *tcf,
4309                     const struct flow_tcf_vxlan_encap *encap,
4310                     unsigned int ifindex,
4311                     bool enable,
4312                     struct rte_flow_error *error)
4313 {
4314         struct nlmsghdr *nlh;
4315         struct ifaddrmsg *ifa;
4316         alignas(struct nlmsghdr)
4317         uint8_t buf[mnl_nlmsg_size(sizeof(*ifa) + 128)];
4318
4319         nlh = mnl_nlmsg_put_header(buf);
4320         nlh->nlmsg_type = enable ? RTM_NEWADDR : RTM_DELADDR;
4321         nlh->nlmsg_flags =
4322                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4323         nlh->nlmsg_seq = 0;
4324         ifa = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifa));
4325         ifa->ifa_flags = IFA_F_PERMANENT;
4326         ifa->ifa_scope = RT_SCOPE_LINK;
4327         ifa->ifa_index = ifindex;
4328         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4329                 ifa->ifa_family = AF_INET;
4330                 ifa->ifa_prefixlen = 32;
4331                 mnl_attr_put_u32(nlh, IFA_LOCAL, encap->ipv4.src);
4332                 if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST)
4333                         mnl_attr_put_u32(nlh, IFA_ADDRESS,
4334                                               encap->ipv4.dst);
4335         } else {
4336                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4337                 ifa->ifa_family = AF_INET6;
4338                 ifa->ifa_prefixlen = 128;
4339                 mnl_attr_put(nlh, IFA_LOCAL,
4340                                   sizeof(encap->ipv6.src),
4341                                   &encap->ipv6.src);
4342                 if (encap->mask & FLOW_TCF_ENCAP_IPV6_DST)
4343                         mnl_attr_put(nlh, IFA_ADDRESS,
4344                                           sizeof(encap->ipv6.dst),
4345                                           &encap->ipv6.dst);
4346         }
4347         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4348                 return 0;
4349         return rte_flow_error_set(error, rte_errno,
4350                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4351                                   "netlink: cannot complete IFA request"
4352                                   " (ip addr add)");
4353 }
4354
4355 /**
4356  * Emit Netlink message to add/remove neighbor.
4357  *
4358  * @param[in] tcf
4359  *   Libmnl socket context object.
4360  * @param[in] encap
4361  *   Encapsulation properties (destination address).
4362  * @param[in] ifindex
4363  *   Network interface.
4364  * @param[in] enable
4365  *   Toggle between add and remove.
4366  * @param[out] error
4367  *   Perform verbose error reporting if not NULL.
4368  *
4369  * @return
4370  *   0 on success, a negative errno value otherwise and rte_errno is set.
4371  */
4372 static int
4373 flow_tcf_rule_neigh(struct mlx5_flow_tcf_context *tcf,
4374                      const struct flow_tcf_vxlan_encap *encap,
4375                      unsigned int ifindex,
4376                      bool enable,
4377                      struct rte_flow_error *error)
4378 {
4379         struct nlmsghdr *nlh;
4380         struct ndmsg *ndm;
4381         alignas(struct nlmsghdr)
4382         uint8_t buf[mnl_nlmsg_size(sizeof(*ndm) + 128)];
4383
4384         nlh = mnl_nlmsg_put_header(buf);
4385         nlh->nlmsg_type = enable ? RTM_NEWNEIGH : RTM_DELNEIGH;
4386         nlh->nlmsg_flags =
4387                 NLM_F_REQUEST | (enable ? NLM_F_CREATE | NLM_F_REPLACE : 0);
4388         nlh->nlmsg_seq = 0;
4389         ndm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ndm));
4390         ndm->ndm_ifindex = ifindex;
4391         ndm->ndm_state = NUD_PERMANENT;
4392         ndm->ndm_flags = 0;
4393         ndm->ndm_type = 0;
4394         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4395                 ndm->ndm_family = AF_INET;
4396                 mnl_attr_put_u32(nlh, NDA_DST, encap->ipv4.dst);
4397         } else {
4398                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4399                 ndm->ndm_family = AF_INET6;
4400                 mnl_attr_put(nlh, NDA_DST, sizeof(encap->ipv6.dst),
4401                                                  &encap->ipv6.dst);
4402         }
4403         if (encap->mask & FLOW_TCF_ENCAP_ETH_SRC && enable)
4404                 DRV_LOG(WARNING,
4405                         "outer ethernet source address cannot be "
4406                         "forced for VXLAN encapsulation");
4407         if (encap->mask & FLOW_TCF_ENCAP_ETH_DST)
4408                 mnl_attr_put(nlh, NDA_LLADDR, sizeof(encap->eth.dst),
4409                                                     &encap->eth.dst);
4410         if (!flow_tcf_nl_ack(tcf, nlh, NULL, NULL))
4411                 return 0;
4412         return rte_flow_error_set(error, rte_errno,
4413                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4414                                   "netlink: cannot complete ND request"
4415                                   " (ip neigh)");
4416 }
4417
4418 /**
4419  * Manage the local IP addresses and their peers IP addresses on the
4420  * outer interface for encapsulation purposes. The kernel searches the
4421  * appropriate device for tunnel egress traffic using the outer source
4422  * IP, this IP should be assigned to the outer network device, otherwise
4423  * kernel rejects the rule.
4424  *
4425  * Adds or removes the addresses using the Netlink command like this:
4426  *   ip addr add <src_ip> peer <dst_ip> scope link dev <ifouter>
4427  *
4428  * The addresses are local to the netdev ("scope link"), this reduces
4429  * the risk of conflicts. Note that an implicit route is maintained by
4430  * the kernel due to the presence of a peer address (IFA_ADDRESS).
4431  *
4432  * @param[in] tcf
4433  *   Libmnl socket context object.
4434  * @param[in] vtep
4435  *   VTEP object, contains rule database and ifouter index.
4436  * @param[in] dev_flow
4437  *   Flow object, contains the tunnel parameters (for encap only).
4438  * @param[in] enable
4439  *   Toggle between add and remove.
4440  * @param[out] error
4441  *   Perform verbose error reporting if not NULL.
4442  *
4443  * @return
4444  *   0 on success, a negative errno value otherwise and rte_errno is set.
4445  */
4446 static int
4447 flow_tcf_encap_local(struct mlx5_flow_tcf_context *tcf,
4448                      struct tcf_vtep *vtep,
4449                      struct mlx5_flow *dev_flow,
4450                      bool enable,
4451                      struct rte_flow_error *error)
4452 {
4453         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4454         struct tcf_local_rule *rule;
4455         bool found = false;
4456         int ret;
4457
4458         assert(encap);
4459         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4460         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4461                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_DST);
4462                 LIST_FOREACH(rule, &vtep->local, next) {
4463                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_SRC &&
4464                             encap->ipv4.src == rule->ipv4.src &&
4465                             encap->ipv4.dst == rule->ipv4.dst) {
4466                                 found = true;
4467                                 break;
4468                         }
4469                 }
4470         } else {
4471                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4472                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4473                 LIST_FOREACH(rule, &vtep->local, next) {
4474                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_SRC &&
4475                             !memcmp(&encap->ipv6.src, &rule->ipv6.src,
4476                                             sizeof(encap->ipv6.src)) &&
4477                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4478                                             sizeof(encap->ipv6.dst))) {
4479                                 found = true;
4480                                 break;
4481                         }
4482                 }
4483         }
4484         if (found) {
4485                 if (enable) {
4486                         rule->refcnt++;
4487                         return 0;
4488                 }
4489                 if (!rule->refcnt || !--rule->refcnt) {
4490                         LIST_REMOVE(rule, next);
4491                         return flow_tcf_rule_local(tcf, encap,
4492                                         vtep->ifouter, false, error);
4493                 }
4494                 return 0;
4495         }
4496         if (!enable) {
4497                 DRV_LOG(WARNING, "disabling not existing local rule");
4498                 rte_flow_error_set(error, ENOENT,
4499                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4500                                    "disabling not existing local rule");
4501                 return -ENOENT;
4502         }
4503         rule = rte_zmalloc(__func__, sizeof(struct tcf_local_rule),
4504                                 alignof(struct tcf_local_rule));
4505         if (!rule) {
4506                 rte_flow_error_set(error, ENOMEM,
4507                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4508                                    "unable to allocate memory for local rule");
4509                 return -rte_errno;
4510         }
4511         *rule = (struct tcf_local_rule){.refcnt = 0,
4512                                         .mask = 0,
4513                                         };
4514         if (encap->mask & FLOW_TCF_ENCAP_IPV4_SRC) {
4515                 rule->mask = FLOW_TCF_ENCAP_IPV4_SRC
4516                            | FLOW_TCF_ENCAP_IPV4_DST;
4517                 rule->ipv4.src = encap->ipv4.src;
4518                 rule->ipv4.dst = encap->ipv4.dst;
4519         } else {
4520                 rule->mask = FLOW_TCF_ENCAP_IPV6_SRC
4521                            | FLOW_TCF_ENCAP_IPV6_DST;
4522                 memcpy(&rule->ipv6.src, &encap->ipv6.src, IPV6_ADDR_LEN);
4523                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4524         }
4525         ret = flow_tcf_rule_local(tcf, encap, vtep->ifouter, true, error);
4526         if (ret) {
4527                 rte_free(rule);
4528                 return ret;
4529         }
4530         rule->refcnt++;
4531         LIST_INSERT_HEAD(&vtep->local, rule, next);
4532         return 0;
4533 }
4534
4535 /**
4536  * Manage the destination MAC/IP addresses neigh database, kernel uses
4537  * this one to determine the destination MAC address within encapsulation
4538  * header. Adds or removes the entries using the Netlink command like this:
4539  *   ip neigh add dev <ifouter> lladdr <dst_mac> to <dst_ip> nud permanent
4540  *
4541  * @param[in] tcf
4542  *   Libmnl socket context object.
4543  * @param[in] vtep
4544  *   VTEP object, contains rule database and ifouter index.
4545  * @param[in] dev_flow
4546  *   Flow object, contains the tunnel parameters (for encap only).
4547  * @param[in] enable
4548  *   Toggle between add and remove.
4549  * @param[out] error
4550  *   Perform verbose error reporting if not NULL.
4551  *
4552  * @return
4553  *   0 on success, a negative errno value otherwise and rte_errno is set.
4554  */
4555 static int
4556 flow_tcf_encap_neigh(struct mlx5_flow_tcf_context *tcf,
4557                      struct tcf_vtep *vtep,
4558                      struct mlx5_flow *dev_flow,
4559                      bool enable,
4560                      struct rte_flow_error *error)
4561 {
4562         const struct flow_tcf_vxlan_encap *encap = dev_flow->tcf.vxlan_encap;
4563         struct tcf_neigh_rule *rule;
4564         bool found = false;
4565         int ret;
4566
4567         assert(encap);
4568         assert(encap->hdr.type == FLOW_TCF_TUNACT_VXLAN_ENCAP);
4569         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4570                 assert(encap->mask & FLOW_TCF_ENCAP_IPV4_SRC);
4571                 LIST_FOREACH(rule, &vtep->neigh, next) {
4572                         if (rule->mask & FLOW_TCF_ENCAP_IPV4_DST &&
4573                             encap->ipv4.dst == rule->ipv4.dst) {
4574                                 found = true;
4575                                 break;
4576                         }
4577                 }
4578         } else {
4579                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_SRC);
4580                 assert(encap->mask & FLOW_TCF_ENCAP_IPV6_DST);
4581                 LIST_FOREACH(rule, &vtep->neigh, next) {
4582                         if (rule->mask & FLOW_TCF_ENCAP_IPV6_DST &&
4583                             !memcmp(&encap->ipv6.dst, &rule->ipv6.dst,
4584                                                 sizeof(encap->ipv6.dst))) {
4585                                 found = true;
4586                                 break;
4587                         }
4588                 }
4589         }
4590         if (found) {
4591                 if (memcmp(&encap->eth.dst, &rule->eth,
4592                            sizeof(encap->eth.dst))) {
4593                         DRV_LOG(WARNING, "Destination MAC differs"
4594                                          " in neigh rule");
4595                         rte_flow_error_set(error, EEXIST,
4596                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4597                                            NULL, "Different MAC address"
4598                                            " neigh rule for the same"
4599                                            " destination IP");
4600                                         return -EEXIST;
4601                 }
4602                 if (enable) {
4603                         rule->refcnt++;
4604                         return 0;
4605                 }
4606                 if (!rule->refcnt || !--rule->refcnt) {
4607                         LIST_REMOVE(rule, next);
4608                         return flow_tcf_rule_neigh(tcf, encap,
4609                                                    vtep->ifouter,
4610                                                    false, error);
4611                 }
4612                 return 0;
4613         }
4614         if (!enable) {
4615                 DRV_LOG(WARNING, "Disabling not existing neigh rule");
4616                 rte_flow_error_set(error, ENOENT,
4617                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4618                                    "unable to allocate memory for neigh rule");
4619                 return -ENOENT;
4620         }
4621         rule = rte_zmalloc(__func__, sizeof(struct tcf_neigh_rule),
4622                                 alignof(struct tcf_neigh_rule));
4623         if (!rule) {
4624                 rte_flow_error_set(error, ENOMEM,
4625                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4626                                    "unable to allocate memory for neigh rule");
4627                 return -rte_errno;
4628         }
4629         *rule = (struct tcf_neigh_rule){.refcnt = 0,
4630                                         .mask = 0,
4631                                         };
4632         if (encap->mask & FLOW_TCF_ENCAP_IPV4_DST) {
4633                 rule->mask = FLOW_TCF_ENCAP_IPV4_DST;
4634                 rule->ipv4.dst = encap->ipv4.dst;
4635         } else {
4636                 rule->mask = FLOW_TCF_ENCAP_IPV6_DST;
4637                 memcpy(&rule->ipv6.dst, &encap->ipv6.dst, IPV6_ADDR_LEN);
4638         }
4639         memcpy(&rule->eth, &encap->eth.dst, sizeof(rule->eth));
4640         ret = flow_tcf_rule_neigh(tcf, encap, vtep->ifouter, true, error);
4641         if (ret) {
4642                 rte_free(rule);
4643                 return ret;
4644         }
4645         rule->refcnt++;
4646         LIST_INSERT_HEAD(&vtep->neigh, rule, next);
4647         return 0;
4648 }
4649
4650 /* VTEP device list is shared between PMD port instances. */
4651 static LIST_HEAD(, tcf_vtep) vtep_list_vxlan = LIST_HEAD_INITIALIZER();
4652 static pthread_mutex_t vtep_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4653
4654 /**
4655  * Deletes VTEP network device.
4656  *
4657  * @param[in] tcf
4658  *   Context object initialized by mlx5_flow_tcf_context_create().
4659  * @param[in] vtep
4660  *   Object represinting the network device to delete. Memory
4661  *   allocated for this object is freed by routine.
4662  */
4663 static void
4664 flow_tcf_vtep_delete(struct mlx5_flow_tcf_context *tcf,
4665                      struct tcf_vtep *vtep)
4666 {
4667         struct nlmsghdr *nlh;
4668         struct ifinfomsg *ifm;
4669         alignas(struct nlmsghdr)
4670         uint8_t buf[mnl_nlmsg_size(MNL_ALIGN(sizeof(*ifm))) +
4671                     MNL_BUF_EXTRA_SPACE];
4672         int ret;
4673
4674         assert(!vtep->refcnt);
4675         /* Delete only ifaces those we actually created. */
4676         if (vtep->created && vtep->ifindex) {
4677                 DRV_LOG(INFO, "VTEP delete (%d)", vtep->ifindex);
4678                 nlh = mnl_nlmsg_put_header(buf);
4679                 nlh->nlmsg_type = RTM_DELLINK;
4680                 nlh->nlmsg_flags = NLM_F_REQUEST;
4681                 ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4682                 ifm->ifi_family = AF_UNSPEC;
4683                 ifm->ifi_index = vtep->ifindex;
4684                 assert(sizeof(buf) >= nlh->nlmsg_len);
4685                 ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4686                 if (ret)
4687                         DRV_LOG(WARNING, "netlink: error deleting vxlan"
4688                                          " encap/decap ifindex %u",
4689                                          ifm->ifi_index);
4690         }
4691         rte_free(vtep);
4692 }
4693
4694 /**
4695  * Creates VTEP network device.
4696  *
4697  * @param[in] tcf
4698  *   Context object initialized by mlx5_flow_tcf_context_create().
4699  * @param[in] ifouter
4700  *   Outer interface to attach new-created VXLAN device
4701  *   If zero the VXLAN device will not be attached to any device.
4702  *   These VTEPs are used for decapsulation and can be precreated
4703  *   and shared between processes.
4704  * @param[in] port
4705  *   UDP port of created VTEP device.
4706  * @param[out] error
4707  *   Perform verbose error reporting if not NULL.
4708  *
4709  * @return
4710  * Pointer to created device structure on success,
4711  * NULL otherwise and rte_errno is set.
4712  */
4713 #ifdef HAVE_IFLA_VXLAN_COLLECT_METADATA
4714 static struct tcf_vtep*
4715 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf,
4716                      unsigned int ifouter,
4717                      uint16_t port, struct rte_flow_error *error)
4718 {
4719         struct tcf_vtep *vtep;
4720         struct nlmsghdr *nlh;
4721         struct ifinfomsg *ifm;
4722         char name[sizeof(MLX5_VXLAN_DEVICE_PFX) + 24];
4723         alignas(struct nlmsghdr)
4724         uint8_t buf[mnl_nlmsg_size(sizeof(*ifm)) +
4725                     SZ_NLATTR_DATA_OF(sizeof(name)) +
4726                     SZ_NLATTR_NEST * 2 +
4727                     SZ_NLATTR_STRZ_OF("vxlan") +
4728                     SZ_NLATTR_DATA_OF(sizeof(uint32_t)) +
4729                     SZ_NLATTR_DATA_OF(sizeof(uint16_t)) +
4730                     SZ_NLATTR_DATA_OF(sizeof(uint8_t)) * 3 +
4731                     MNL_BUF_EXTRA_SPACE];
4732         struct nlattr *na_info;
4733         struct nlattr *na_vxlan;
4734         rte_be16_t vxlan_port = rte_cpu_to_be_16(port);
4735         int ret;
4736
4737         vtep = rte_zmalloc(__func__, sizeof(*vtep), alignof(struct tcf_vtep));
4738         if (!vtep) {
4739                 rte_flow_error_set(error, ENOMEM,
4740                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4741                                    "unable to allocate memory for VTEP");
4742                 return NULL;
4743         }
4744         *vtep = (struct tcf_vtep){
4745                         .port = port,
4746                         .local = LIST_HEAD_INITIALIZER(),
4747                         .neigh = LIST_HEAD_INITIALIZER(),
4748         };
4749         memset(buf, 0, sizeof(buf));
4750         nlh = mnl_nlmsg_put_header(buf);
4751         nlh->nlmsg_type = RTM_NEWLINK;
4752         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE  | NLM_F_EXCL;
4753         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4754         ifm->ifi_family = AF_UNSPEC;
4755         ifm->ifi_type = 0;
4756         ifm->ifi_index = 0;
4757         ifm->ifi_flags = IFF_UP;
4758         ifm->ifi_change = 0xffffffff;
4759         snprintf(name, sizeof(name), "%s%u", MLX5_VXLAN_DEVICE_PFX, port);
4760         mnl_attr_put_strz(nlh, IFLA_IFNAME, name);
4761         na_info = mnl_attr_nest_start(nlh, IFLA_LINKINFO);
4762         assert(na_info);
4763         mnl_attr_put_strz(nlh, IFLA_INFO_KIND, "vxlan");
4764         na_vxlan = mnl_attr_nest_start(nlh, IFLA_INFO_DATA);
4765         if (ifouter)
4766                 mnl_attr_put_u32(nlh, IFLA_VXLAN_LINK, ifouter);
4767         assert(na_vxlan);
4768         mnl_attr_put_u8(nlh, IFLA_VXLAN_COLLECT_METADATA, 1);
4769         mnl_attr_put_u8(nlh, IFLA_VXLAN_UDP_ZERO_CSUM6_RX, 1);
4770         mnl_attr_put_u8(nlh, IFLA_VXLAN_LEARNING, 0);
4771         mnl_attr_put_u16(nlh, IFLA_VXLAN_PORT, vxlan_port);
4772         mnl_attr_nest_end(nlh, na_vxlan);
4773         mnl_attr_nest_end(nlh, na_info);
4774         assert(sizeof(buf) >= nlh->nlmsg_len);
4775         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4776         if (ret) {
4777                 DRV_LOG(WARNING,
4778                         "netlink: VTEP %s create failure (%d)",
4779                         name, rte_errno);
4780                 if (rte_errno != EEXIST || ifouter)
4781                         /*
4782                          * Some unhandled error occurred or device is
4783                          * for encapsulation and cannot be shared.
4784                          */
4785                         goto error;
4786         } else {
4787                 /*
4788                  * Mark device we actually created.
4789                  * We should explicitly delete
4790                  * when we do not need it anymore.
4791                  */
4792                 vtep->created = 1;
4793         }
4794         /* Try to get ifindex of created of pre-existing device. */
4795         ret = if_nametoindex(name);
4796         if (!ret) {
4797                 DRV_LOG(WARNING,
4798                         "VTEP %s failed to get index (%d)", name, errno);
4799                 rte_flow_error_set
4800                         (error, -errno,
4801                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4802                          "netlink: failed to retrieve VTEP ifindex");
4803                 goto error;
4804         }
4805         vtep->ifindex = ret;
4806         vtep->ifouter = ifouter;
4807         memset(buf, 0, sizeof(buf));
4808         nlh = mnl_nlmsg_put_header(buf);
4809         nlh->nlmsg_type = RTM_NEWLINK;
4810         nlh->nlmsg_flags = NLM_F_REQUEST;
4811         ifm = mnl_nlmsg_put_extra_header(nlh, sizeof(*ifm));
4812         ifm->ifi_family = AF_UNSPEC;
4813         ifm->ifi_type = 0;
4814         ifm->ifi_index = vtep->ifindex;
4815         ifm->ifi_flags = IFF_UP;
4816         ifm->ifi_change = IFF_UP;
4817         ret = flow_tcf_nl_ack(tcf, nlh, NULL, NULL);
4818         if (ret) {
4819                 rte_flow_error_set(error, -errno,
4820                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4821                                    "netlink: failed to set VTEP link up");
4822                 DRV_LOG(WARNING, "netlink: VTEP %s set link up failure (%d)",
4823                         name, rte_errno);
4824                 goto clean;
4825         }
4826         ret = mlx5_flow_tcf_init(tcf, vtep->ifindex, error);
4827         if (ret) {
4828                 DRV_LOG(WARNING, "VTEP %s init failure (%d)", name, rte_errno);
4829                 goto clean;
4830         }
4831         DRV_LOG(INFO, "VTEP create (%d, %d)", vtep->port, vtep->ifindex);
4832         vtep->refcnt = 1;
4833         return vtep;
4834 clean:
4835         flow_tcf_vtep_delete(tcf, vtep);
4836         return NULL;
4837 error:
4838         rte_free(vtep);
4839         return NULL;
4840 }
4841 #else
4842 static struct tcf_vtep*
4843 flow_tcf_vtep_create(struct mlx5_flow_tcf_context *tcf __rte_unused,
4844                      unsigned int ifouter __rte_unused,
4845                      uint16_t port __rte_unused,
4846                      struct rte_flow_error *error)
4847 {
4848         rte_flow_error_set(error, ENOTSUP,
4849                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4850                            "netlink: failed to create VTEP, "
4851                            "vxlan metadata are not supported by kernel");
4852         return NULL;
4853 }
4854 #endif /* HAVE_IFLA_VXLAN_COLLECT_METADATA */
4855
4856 /**
4857  * Acquire target interface index for VXLAN tunneling decapsulation.
4858  * In order to share the UDP port within the other interfaces the
4859  * VXLAN device created as not attached to any interface (if created).
4860  *
4861  * @param[in] tcf
4862  *   Context object initialized by mlx5_flow_tcf_context_create().
4863  * @param[in] dev_flow
4864  *   Flow tcf object with tunnel structure pointer set.
4865  * @param[out] error
4866  *   Perform verbose error reporting if not NULL.
4867  * @return
4868  *   Interface descriptor pointer on success,
4869  *   NULL otherwise and rte_errno is set.
4870  */
4871 static struct tcf_vtep*
4872 flow_tcf_decap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4873                             struct mlx5_flow *dev_flow,
4874                             struct rte_flow_error *error)
4875 {
4876         struct tcf_vtep *vtep;
4877         uint16_t port = dev_flow->tcf.vxlan_decap->udp_port;
4878
4879         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4880                 if (vtep->port == port)
4881                         break;
4882         }
4883         if (vtep && vtep->ifouter) {
4884                 rte_flow_error_set(error, -errno,
4885                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4886                                    "Failed to create decap VTEP with specified"
4887                                    " UDP port, atatched device exists");
4888                 return NULL;
4889         }
4890         if (vtep) {
4891                 /* Device exists, just increment the reference counter. */
4892                 vtep->refcnt++;
4893                 assert(vtep->ifindex);
4894                 return vtep;
4895         }
4896         /* No decapsulation device exists, try to create the new one. */
4897         vtep = flow_tcf_vtep_create(tcf, 0, port, error);
4898         if (vtep)
4899                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
4900         return vtep;
4901 }
4902
4903 /**
4904  * Aqcuire target interface index for VXLAN tunneling encapsulation.
4905  *
4906  * @param[in] tcf
4907  *   Context object initialized by mlx5_flow_tcf_context_create().
4908  * @param[in] ifouter
4909  *   Network interface index to attach VXLAN encap device to.
4910  * @param[in] dev_flow
4911  *   Flow tcf object with tunnel structure pointer set.
4912  * @param[out] error
4913  *   Perform verbose error reporting if not NULL.
4914  * @return
4915  *   Interface descriptor pointer on success,
4916  *   NULL otherwise and rte_errno is set.
4917  */
4918 static struct tcf_vtep*
4919 flow_tcf_encap_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
4920                             unsigned int ifouter,
4921                             struct mlx5_flow *dev_flow __rte_unused,
4922                             struct rte_flow_error *error)
4923 {
4924         static uint16_t encap_port = MLX5_VXLAN_PORT_MIN - 1;
4925         struct tcf_vtep *vtep;
4926         int ret;
4927
4928         assert(ifouter);
4929         /* Look whether the attached VTEP for encap is created. */
4930         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4931                 if (vtep->ifouter == ifouter)
4932                         break;
4933         }
4934         if (vtep) {
4935                 /* VTEP already exists, just increment the reference. */
4936                 vtep->refcnt++;
4937         } else {
4938                 uint16_t pcnt;
4939
4940                 /* Not found, we should create the new attached VTEP. */
4941                 flow_tcf_encap_iface_cleanup(tcf, ifouter);
4942                 flow_tcf_encap_local_cleanup(tcf, ifouter);
4943                 flow_tcf_encap_neigh_cleanup(tcf, ifouter);
4944                 for (pcnt = 0; pcnt <= (MLX5_VXLAN_PORT_MAX
4945                                      - MLX5_VXLAN_PORT_MIN); pcnt++) {
4946                         encap_port++;
4947                         /* Wraparound the UDP port index. */
4948                         if (encap_port < MLX5_VXLAN_PORT_MIN ||
4949                             encap_port > MLX5_VXLAN_PORT_MAX)
4950                                 encap_port = MLX5_VXLAN_PORT_MIN;
4951                         /* Check whether UDP port is in already in use. */
4952                         LIST_FOREACH(vtep, &vtep_list_vxlan, next) {
4953                                 if (vtep->port == encap_port)
4954                                         break;
4955                         }
4956                         if (vtep) {
4957                                 /* Port is in use, try the next one. */
4958                                 vtep = NULL;
4959                                 continue;
4960                         }
4961                         vtep = flow_tcf_vtep_create(tcf, ifouter,
4962                                                     encap_port, error);
4963                         if (vtep) {
4964                                 LIST_INSERT_HEAD(&vtep_list_vxlan, vtep, next);
4965                                 break;
4966                         }
4967                         if (rte_errno != EEXIST)
4968                                 break;
4969                 }
4970                 if (!vtep)
4971                         return NULL;
4972         }
4973         assert(vtep->ifouter == ifouter);
4974         assert(vtep->ifindex);
4975         /* Create local ipaddr with peer to specify the outer IPs. */
4976         ret = flow_tcf_encap_local(tcf, vtep, dev_flow, true, error);
4977         if (!ret) {
4978                 /* Create neigh rule to specify outer destination MAC. */
4979                 ret = flow_tcf_encap_neigh(tcf, vtep, dev_flow, true, error);
4980                 if (ret)
4981                         flow_tcf_encap_local(tcf, vtep,
4982                                              dev_flow, false, error);
4983         }
4984         if (ret) {
4985                 if (--vtep->refcnt == 0)
4986                         flow_tcf_vtep_delete(tcf, vtep);
4987                 return NULL;
4988         }
4989         return vtep;
4990 }
4991
4992 /**
4993  * Acquires target interface index for tunneling of any type.
4994  * Creates the new VTEP if needed.
4995  *
4996  * @param[in] tcf
4997  *   Context object initialized by mlx5_flow_tcf_context_create().
4998  * @param[in] ifouter
4999  *   Network interface index to attach VXLAN encap device to.
5000  * @param[in] dev_flow
5001  *   Flow tcf object with tunnel structure pointer set.
5002  * @param[out] error
5003  *   Perform verbose error reporting if not NULL.
5004  * @return
5005  *   Interface descriptor pointer on success,
5006  *   NULL otherwise and rte_errno is set.
5007  */
5008 static struct tcf_vtep*
5009 flow_tcf_vtep_acquire(struct mlx5_flow_tcf_context *tcf,
5010                       unsigned int ifouter,
5011                       struct mlx5_flow *dev_flow,
5012                       struct rte_flow_error *error)
5013 {
5014         struct tcf_vtep *vtep = NULL;
5015
5016         assert(dev_flow->tcf.tunnel);
5017         pthread_mutex_lock(&vtep_list_mutex);
5018         switch (dev_flow->tcf.tunnel->type) {
5019         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5020                 vtep = flow_tcf_encap_vtep_acquire(tcf, ifouter,
5021                                                   dev_flow, error);
5022                 break;
5023         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5024                 vtep = flow_tcf_decap_vtep_acquire(tcf, dev_flow, error);
5025                 break;
5026         default:
5027                 rte_flow_error_set(error, ENOTSUP,
5028                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5029                                    "unsupported tunnel type");
5030                 break;
5031         }
5032         pthread_mutex_unlock(&vtep_list_mutex);
5033         return vtep;
5034 }
5035
5036 /**
5037  * Release tunneling interface by ifindex. Decrements reference
5038  * counter and actually removes the device if counter is zero.
5039  *
5040  * @param[in] tcf
5041  *   Context object initialized by mlx5_flow_tcf_context_create().
5042  * @param[in] vtep
5043  *   VTEP device descriptor structure.
5044  * @param[in] dev_flow
5045  *   Flow tcf object with tunnel structure pointer set.
5046  */
5047 static void
5048 flow_tcf_vtep_release(struct mlx5_flow_tcf_context *tcf,
5049                       struct tcf_vtep *vtep,
5050                       struct mlx5_flow *dev_flow)
5051 {
5052         assert(dev_flow->tcf.tunnel);
5053         pthread_mutex_lock(&vtep_list_mutex);
5054         switch (dev_flow->tcf.tunnel->type) {
5055         case FLOW_TCF_TUNACT_VXLAN_DECAP:
5056                 break;
5057         case FLOW_TCF_TUNACT_VXLAN_ENCAP:
5058                 /* Remove the encap ancillary rules first. */
5059                 flow_tcf_encap_neigh(tcf, vtep, dev_flow, false, NULL);
5060                 flow_tcf_encap_local(tcf, vtep, dev_flow, false, NULL);
5061                 break;
5062         default:
5063                 assert(false);
5064                 DRV_LOG(WARNING, "Unsupported tunnel type");
5065                 break;
5066         }
5067         assert(vtep->refcnt);
5068         if (--vtep->refcnt == 0) {
5069                 LIST_REMOVE(vtep, next);
5070                 flow_tcf_vtep_delete(tcf, vtep);
5071         }
5072         pthread_mutex_unlock(&vtep_list_mutex);
5073 }
5074
5075 struct tcf_nlcb_query {
5076         uint32_t handle;
5077         uint32_t tc_flags;
5078         uint32_t flags_valid:1;
5079 };
5080
5081 /**
5082  * Collect queried rule attributes. This is callback routine called by
5083  * libmnl mnl_cb_run() in loop for every message in received packet.
5084  * Current implementation collects the flower flags only.
5085  *
5086  * @param[in] nlh
5087  *   Pointer to reply header.
5088  * @param[in, out] arg
5089  *   Context pointer for this callback.
5090  *
5091  * @return
5092  *   A positive, nonzero value on success (required by libmnl
5093  *   to continue messages processing).
5094  */
5095 static int
5096 flow_tcf_collect_query_cb(const struct nlmsghdr *nlh, void *arg)
5097 {
5098         struct tcf_nlcb_query *query = arg;
5099         struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
5100         struct nlattr *na, *na_opt;
5101         bool flower = false;
5102
5103         if (nlh->nlmsg_type != RTM_NEWTFILTER ||
5104             tcm->tcm_handle != query->handle)
5105                 return 1;
5106         mnl_attr_for_each(na, nlh, sizeof(*tcm)) {
5107                 switch (mnl_attr_get_type(na)) {
5108                 case TCA_KIND:
5109                         if (strcmp(mnl_attr_get_payload(na), "flower")) {
5110                                 /* Not flower filter, drop entire message. */
5111                                 return 1;
5112                         }
5113                         flower = true;
5114                         break;
5115                 case TCA_OPTIONS:
5116                         if (!flower) {
5117                                 /* Not flower options, drop entire message. */
5118                                 return 1;
5119                         }
5120                         /* Check nested flower options. */
5121                         mnl_attr_for_each_nested(na_opt, na) {
5122                                 switch (mnl_attr_get_type(na_opt)) {
5123                                 case TCA_FLOWER_FLAGS:
5124                                         query->flags_valid = 1;
5125                                         query->tc_flags =
5126                                                 mnl_attr_get_u32(na_opt);
5127                                         break;
5128                                 }
5129                         }
5130                         break;
5131                 }
5132         }
5133         return 1;
5134 }
5135
5136 /**
5137  * Query a TC flower rule flags via netlink.
5138  *
5139  * @param[in] tcf
5140  *   Context object initialized by mlx5_flow_tcf_context_create().
5141  * @param[in] dev_flow
5142  *   Pointer to the flow.
5143  * @param[out] pflags
5144  *   pointer to the data retrieved by the query.
5145  *
5146  * @return
5147  *   0 on success, a negative errno value otherwise.
5148  */
5149 static int
5150 flow_tcf_query_flags(struct mlx5_flow_tcf_context *tcf,
5151                      struct mlx5_flow *dev_flow,
5152                      uint32_t *pflags)
5153 {
5154         struct nlmsghdr *nlh;
5155         struct tcmsg *tcm;
5156         struct tcf_nlcb_query query = {
5157                 .handle = dev_flow->tcf.tcm->tcm_handle,
5158         };
5159
5160         nlh = mnl_nlmsg_put_header(tcf->buf);
5161         nlh->nlmsg_type = RTM_GETTFILTER;
5162         nlh->nlmsg_flags = NLM_F_REQUEST;
5163         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5164         memcpy(tcm, dev_flow->tcf.tcm, sizeof(*tcm));
5165         /*
5166          * Ignore Netlink error for filter query operations.
5167          * The reply length is sent by kernel as errno.
5168          * Just check we got the flags option.
5169          */
5170         flow_tcf_nl_ack(tcf, nlh, flow_tcf_collect_query_cb, &query);
5171         if (!query.flags_valid) {
5172                 *pflags = 0;
5173                 return -ENOENT;
5174         }
5175         *pflags = query.tc_flags;
5176         return 0;
5177 }
5178
5179 /**
5180  * Query and check the in_hw set for specified rule.
5181  *
5182  * @param[in] tcf
5183  *   Context object initialized by mlx5_flow_tcf_context_create().
5184  * @param[in] dev_flow
5185  *   Pointer to the flow to check.
5186  *
5187  * @return
5188  *   0 on success, a negative errno value otherwise.
5189  */
5190 static int
5191 flow_tcf_check_inhw(struct mlx5_flow_tcf_context *tcf,
5192                     struct mlx5_flow *dev_flow)
5193 {
5194         uint32_t flags;
5195         int ret;
5196
5197         ret = flow_tcf_query_flags(tcf, dev_flow, &flags);
5198         if (ret)
5199                 return ret;
5200         return  (flags & TCA_CLS_FLAGS_IN_HW) ? 0 : -ENOENT;
5201 }
5202
5203 /**
5204  * Remove flow from E-Switch by sending Netlink message.
5205  *
5206  * @param[in] dev
5207  *   Pointer to Ethernet device.
5208  * @param[in, out] flow
5209  *   Pointer to the sub flow.
5210  */
5211 static void
5212 flow_tcf_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5213 {
5214         struct priv *priv = dev->data->dev_private;
5215         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5216         struct mlx5_flow *dev_flow;
5217         struct nlmsghdr *nlh;
5218
5219         if (!flow)
5220                 return;
5221         dev_flow = LIST_FIRST(&flow->dev_flows);
5222         if (!dev_flow)
5223                 return;
5224         /* E-Switch flow can't be expanded. */
5225         assert(!LIST_NEXT(dev_flow, next));
5226         if (dev_flow->tcf.applied) {
5227                 nlh = dev_flow->tcf.nlh;
5228                 nlh->nlmsg_type = RTM_DELTFILTER;
5229                 nlh->nlmsg_flags = NLM_F_REQUEST;
5230                 flow_tcf_nl_ack(ctx, nlh, NULL, NULL);
5231                 if (dev_flow->tcf.tunnel) {
5232                         assert(dev_flow->tcf.tunnel->vtep);
5233                         flow_tcf_vtep_release(ctx,
5234                                 dev_flow->tcf.tunnel->vtep,
5235                                 dev_flow);
5236                         dev_flow->tcf.tunnel->vtep = NULL;
5237                 }
5238                 dev_flow->tcf.applied = 0;
5239         }
5240 }
5241
5242 /**
5243  * Apply flow to E-Switch by sending Netlink message.
5244  *
5245  * @param[in] dev
5246  *   Pointer to Ethernet device.
5247  * @param[in, out] flow
5248  *   Pointer to the sub flow.
5249  * @param[out] error
5250  *   Pointer to the error structure.
5251  *
5252  * @return
5253  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5254  */
5255 static int
5256 flow_tcf_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5257                struct rte_flow_error *error)
5258 {
5259         struct priv *priv = dev->data->dev_private;
5260         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5261         struct mlx5_flow *dev_flow;
5262         struct nlmsghdr *nlh;
5263
5264         dev_flow = LIST_FIRST(&flow->dev_flows);
5265         /* E-Switch flow can't be expanded. */
5266         assert(!LIST_NEXT(dev_flow, next));
5267         if (dev_flow->tcf.applied)
5268                 return 0;
5269         nlh = dev_flow->tcf.nlh;
5270         nlh->nlmsg_type = RTM_NEWTFILTER;
5271         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5272         if (dev_flow->tcf.tunnel) {
5273                 /*
5274                  * Replace the interface index, target for
5275                  * encapsulation, source for decapsulation.
5276                  */
5277                 assert(!dev_flow->tcf.tunnel->vtep);
5278                 assert(dev_flow->tcf.tunnel->ifindex_ptr);
5279                 /* Acquire actual VTEP device when rule is being applied. */
5280                 dev_flow->tcf.tunnel->vtep =
5281                         flow_tcf_vtep_acquire(ctx,
5282                                         dev_flow->tcf.tunnel->ifindex_org,
5283                                         dev_flow, error);
5284                 if (!dev_flow->tcf.tunnel->vtep)
5285                         return -rte_errno;
5286                 DRV_LOG(INFO, "Replace ifindex: %d->%d",
5287                                 dev_flow->tcf.tunnel->vtep->ifindex,
5288                                 dev_flow->tcf.tunnel->ifindex_org);
5289                 *dev_flow->tcf.tunnel->ifindex_ptr =
5290                         dev_flow->tcf.tunnel->vtep->ifindex;
5291         }
5292         if (!flow_tcf_nl_ack(ctx, nlh, NULL, NULL)) {
5293                 dev_flow->tcf.applied = 1;
5294                 if (*dev_flow->tcf.ptc_flags & TCA_CLS_FLAGS_SKIP_SW)
5295                         return 0;
5296                 /*
5297                  * Rule was applied without skip_sw flag set.
5298                  * We should check whether the rule was acctually
5299                  * accepted by hardware (have look at in_hw flag).
5300                  */
5301                 if (flow_tcf_check_inhw(ctx, dev_flow)) {
5302                         flow_tcf_remove(dev, flow);
5303                         return rte_flow_error_set
5304                                 (error, ENOENT,
5305                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5306                                  "netlink: rule has no in_hw flag set");
5307                 }
5308                 return 0;
5309         }
5310         if (dev_flow->tcf.tunnel) {
5311                 /* Rollback the VTEP configuration if rule apply failed. */
5312                 assert(dev_flow->tcf.tunnel->vtep);
5313                 flow_tcf_vtep_release(ctx, dev_flow->tcf.tunnel->vtep,
5314                                       dev_flow);
5315                 dev_flow->tcf.tunnel->vtep = NULL;
5316         }
5317         return rte_flow_error_set(error, rte_errno,
5318                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5319                                   "netlink: failed to create TC flow rule");
5320 }
5321
5322 /**
5323  * Remove flow from E-Switch and release resources of the device flow.
5324  *
5325  * @param[in] dev
5326  *   Pointer to Ethernet device.
5327  * @param[in, out] flow
5328  *   Pointer to the sub flow.
5329  */
5330 static void
5331 flow_tcf_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5332 {
5333         struct mlx5_flow *dev_flow;
5334
5335         if (!flow)
5336                 return;
5337         flow_tcf_remove(dev, flow);
5338         if (flow->counter) {
5339                 if (--flow->counter->ref_cnt == 0) {
5340                         rte_free(flow->counter);
5341                         flow->counter = NULL;
5342                 }
5343         }
5344         dev_flow = LIST_FIRST(&flow->dev_flows);
5345         if (!dev_flow)
5346                 return;
5347         /* E-Switch flow can't be expanded. */
5348         assert(!LIST_NEXT(dev_flow, next));
5349         LIST_REMOVE(dev_flow, next);
5350         rte_free(dev_flow);
5351 }
5352
5353 /**
5354  * Helper routine for figuring the space size required for a parse buffer.
5355  *
5356  * @param array
5357  *   array of values to use.
5358  * @param idx
5359  *   Current location in array.
5360  * @param value
5361  *   Value to compare with.
5362  *
5363  * @return
5364  *   The maximum between the given value and the array value on index.
5365  */
5366 static uint16_t
5367 flow_tcf_arr_val_max(uint16_t array[], int idx, uint16_t value)
5368 {
5369         return idx < 0 ? (value) : RTE_MAX((array)[idx], value);
5370 }
5371
5372 /**
5373  * Parse rtnetlink message attributes filling the attribute table with the info
5374  * retrieved.
5375  *
5376  * @param tb
5377  *   Attribute table to be filled.
5378  * @param[out] max
5379  *   Maxinum entry in the attribute table.
5380  * @param rte
5381  *   The attributes section in the message to be parsed.
5382  * @param len
5383  *   The length of the attributes section in the message.
5384  */
5385 static void
5386 flow_tcf_nl_parse_rtattr(struct rtattr *tb[], int max,
5387                          struct rtattr *rta, int len)
5388 {
5389         unsigned short type;
5390         memset(tb, 0, sizeof(struct rtattr *) * (max + 1));
5391         while (RTA_OK(rta, len)) {
5392                 type = rta->rta_type;
5393                 if (type <= max && !tb[type])
5394                         tb[type] = rta;
5395                 rta = RTA_NEXT(rta, len);
5396         }
5397 }
5398
5399 /**
5400  * Extract flow counters from flower action.
5401  *
5402  * @param rta
5403  *   flower action stats properties in the Netlink message received.
5404  * @param rta_type
5405  *   The backward sequence of rta_types, as written in the attribute table,
5406  *   we need to traverse in order to get to the requested object.
5407  * @param idx
5408  *   Current location in rta_type table.
5409  * @param[out] data
5410  *   data holding the count statistics of the rte_flow retrieved from
5411  *   the message.
5412  *
5413  * @return
5414  *   0 if data was found and retrieved, -1 otherwise.
5415  */
5416 static int
5417 flow_tcf_nl_action_stats_parse_and_get(struct rtattr *rta,
5418                                        uint16_t rta_type[], int idx,
5419                                        struct gnet_stats_basic *data)
5420 {
5421         int tca_stats_max = flow_tcf_arr_val_max(rta_type, idx,
5422                                                  TCA_STATS_BASIC);
5423         struct rtattr *tbs[tca_stats_max + 1];
5424
5425         if (rta == NULL || idx < 0)
5426                 return -1;
5427         flow_tcf_nl_parse_rtattr(tbs, tca_stats_max,
5428                                  RTA_DATA(rta), RTA_PAYLOAD(rta));
5429         switch (rta_type[idx]) {
5430         case TCA_STATS_BASIC:
5431                 if (tbs[TCA_STATS_BASIC]) {
5432                         memcpy(data, RTA_DATA(tbs[TCA_STATS_BASIC]),
5433                                RTE_MIN(RTA_PAYLOAD(tbs[TCA_STATS_BASIC]),
5434                                sizeof(*data)));
5435                         return 0;
5436                 }
5437                 break;
5438         default:
5439                 break;
5440         }
5441         return -1;
5442 }
5443
5444 /**
5445  * Parse flower single action retrieving the requested action attribute,
5446  * if found.
5447  *
5448  * @param arg
5449  *   flower action properties in the Netlink message received.
5450  * @param rta_type
5451  *   The backward sequence of rta_types, as written in the attribute table,
5452  *   we need to traverse in order to get to the requested object.
5453  * @param idx
5454  *   Current location in rta_type table.
5455  * @param[out] data
5456  *   Count statistics retrieved from the message query.
5457  *
5458  * @return
5459  *   0 if data was found and retrieved, -1 otherwise.
5460  */
5461 static int
5462 flow_tcf_nl_parse_one_action_and_get(struct rtattr *arg,
5463                                      uint16_t rta_type[], int idx, void *data)
5464 {
5465         int tca_act_max = flow_tcf_arr_val_max(rta_type, idx, TCA_ACT_STATS);
5466         struct rtattr *tb[tca_act_max + 1];
5467
5468         if (arg == NULL || idx < 0)
5469                 return -1;
5470         flow_tcf_nl_parse_rtattr(tb, tca_act_max,
5471                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5472         if (tb[TCA_ACT_KIND] == NULL)
5473                 return -1;
5474         switch (rta_type[idx]) {
5475         case TCA_ACT_STATS:
5476                 if (tb[TCA_ACT_STATS])
5477                         return flow_tcf_nl_action_stats_parse_and_get
5478                                         (tb[TCA_ACT_STATS],
5479                                          rta_type, --idx,
5480                                          (struct gnet_stats_basic *)data);
5481                 break;
5482         default:
5483                 break;
5484         }
5485         return -1;
5486 }
5487
5488 /**
5489  * Parse flower action section in the message retrieving the requested
5490  * attribute from the first action that provides it.
5491  *
5492  * @param opt
5493  *   flower section in the Netlink message received.
5494  * @param rta_type
5495  *   The backward sequence of rta_types, as written in the attribute table,
5496  *   we need to traverse in order to get to the requested object.
5497  * @param idx
5498  *   Current location in rta_type table.
5499  * @param[out] data
5500  *   data retrieved from the message query.
5501  *
5502  * @return
5503  *   0 if data was found and retrieved, -1 otherwise.
5504  */
5505 static int
5506 flow_tcf_nl_action_parse_and_get(struct rtattr *arg,
5507                                  uint16_t rta_type[], int idx, void *data)
5508 {
5509         struct rtattr *tb[TCA_ACT_MAX_PRIO + 1];
5510         int i;
5511
5512         if (arg == NULL || idx < 0)
5513                 return -1;
5514         flow_tcf_nl_parse_rtattr(tb, TCA_ACT_MAX_PRIO,
5515                                  RTA_DATA(arg), RTA_PAYLOAD(arg));
5516         switch (rta_type[idx]) {
5517         /*
5518          * flow counters are stored in the actions defined by the flow
5519          * and not in the flow itself, therefore we need to traverse the
5520          * flower chain of actions in search for them.
5521          *
5522          * Note that the index is not decremented here.
5523          */
5524         case TCA_ACT_STATS:
5525                 for (i = 0; i <= TCA_ACT_MAX_PRIO; i++) {
5526                         if (tb[i] &&
5527                         !flow_tcf_nl_parse_one_action_and_get(tb[i],
5528                                                               rta_type,
5529                                                               idx, data))
5530                                 return 0;
5531                 }
5532                 break;
5533         default:
5534                 break;
5535         }
5536         return -1;
5537 }
5538
5539 /**
5540  * Parse flower classifier options in the message, retrieving the requested
5541  * attribute if found.
5542  *
5543  * @param opt
5544  *   flower section in the Netlink message received.
5545  * @param rta_type
5546  *   The backward sequence of rta_types, as written in the attribute table,
5547  *   we need to traverse in order to get to the requested object.
5548  * @param idx
5549  *   Current location in rta_type table.
5550  * @param[out] data
5551  *   data retrieved from the message query.
5552  *
5553  * @return
5554  *   0 if data was found and retrieved, -1 otherwise.
5555  */
5556 static int
5557 flow_tcf_nl_opts_parse_and_get(struct rtattr *opt,
5558                                uint16_t rta_type[], int idx, void *data)
5559 {
5560         int tca_flower_max = flow_tcf_arr_val_max(rta_type, idx,
5561                                                   TCA_FLOWER_ACT);
5562         struct rtattr *tb[tca_flower_max + 1];
5563
5564         if (!opt || idx < 0)
5565                 return -1;
5566         flow_tcf_nl_parse_rtattr(tb, tca_flower_max,
5567                                  RTA_DATA(opt), RTA_PAYLOAD(opt));
5568         switch (rta_type[idx]) {
5569         case TCA_FLOWER_ACT:
5570                 if (tb[TCA_FLOWER_ACT])
5571                         return flow_tcf_nl_action_parse_and_get
5572                                                         (tb[TCA_FLOWER_ACT],
5573                                                          rta_type, --idx, data);
5574                 break;
5575         default:
5576                 break;
5577         }
5578         return -1;
5579 }
5580
5581 /**
5582  * Parse Netlink reply on filter query, retrieving the flow counters.
5583  *
5584  * @param nlh
5585  *   Message received from Netlink.
5586  * @param rta_type
5587  *   The backward sequence of rta_types, as written in the attribute table,
5588  *   we need to traverse in order to get to the requested object.
5589  * @param idx
5590  *   Current location in rta_type table.
5591  * @param[out] data
5592  *   data retrieved from the message query.
5593  *
5594  * @return
5595  *   0 if data was found and retrieved, -1 otherwise.
5596  */
5597 static int
5598 flow_tcf_nl_filter_parse_and_get(struct nlmsghdr *cnlh,
5599                                  uint16_t rta_type[], int idx, void *data)
5600 {
5601         struct nlmsghdr *nlh = cnlh;
5602         struct tcmsg *t = NLMSG_DATA(nlh);
5603         int len = nlh->nlmsg_len;
5604         int tca_max = flow_tcf_arr_val_max(rta_type, idx, TCA_OPTIONS);
5605         struct rtattr *tb[tca_max + 1];
5606
5607         if (idx < 0)
5608                 return -1;
5609         if (nlh->nlmsg_type != RTM_NEWTFILTER &&
5610             nlh->nlmsg_type != RTM_GETTFILTER &&
5611             nlh->nlmsg_type != RTM_DELTFILTER)
5612                 return -1;
5613         len -= NLMSG_LENGTH(sizeof(*t));
5614         if (len < 0)
5615                 return -1;
5616         flow_tcf_nl_parse_rtattr(tb, tca_max, TCA_RTA(t), len);
5617         /* Not a TC flower flow - bail out */
5618         if (!tb[TCA_KIND] ||
5619             strcmp(RTA_DATA(tb[TCA_KIND]), "flower"))
5620                 return -1;
5621         switch (rta_type[idx]) {
5622         case TCA_OPTIONS:
5623                 if (tb[TCA_OPTIONS])
5624                         return flow_tcf_nl_opts_parse_and_get(tb[TCA_OPTIONS],
5625                                                               rta_type,
5626                                                               --idx, data);
5627                 break;
5628         default:
5629                 break;
5630         }
5631         return -1;
5632 }
5633
5634 /**
5635  * A callback to parse Netlink reply on TC flower query.
5636  *
5637  * @param nlh
5638  *   Message received from Netlink.
5639  * @param[out] data
5640  *   Pointer to data area to be filled by the parsing routine.
5641  *   assumed to be a pointer to struct flow_tcf_stats_basic.
5642  *
5643  * @return
5644  *   MNL_CB_OK value.
5645  */
5646 static int
5647 flow_tcf_nl_message_get_stats_basic(const struct nlmsghdr *nlh, void *data)
5648 {
5649         /*
5650          * The backward sequence of rta_types to pass in order to get
5651          *  to the counters.
5652          */
5653         uint16_t rta_type[] = { TCA_STATS_BASIC, TCA_ACT_STATS,
5654                                 TCA_FLOWER_ACT, TCA_OPTIONS };
5655         struct flow_tcf_stats_basic *sb_data = data;
5656         union {
5657                 const struct nlmsghdr *c;
5658                 struct nlmsghdr *nc;
5659         } tnlh = { .c = nlh };
5660
5661         if (!flow_tcf_nl_filter_parse_and_get(tnlh.nc, rta_type,
5662                                               RTE_DIM(rta_type) - 1,
5663                                               (void *)&sb_data->counters))
5664                 sb_data->valid = true;
5665         return MNL_CB_OK;
5666 }
5667
5668 /**
5669  * Query a TC flower rule for its statistics via netlink.
5670  *
5671  * @param[in] dev
5672  *   Pointer to Ethernet device.
5673  * @param[in] flow
5674  *   Pointer to the sub flow.
5675  * @param[out] data
5676  *   data retrieved by the query.
5677  * @param[out] error
5678  *   Perform verbose error reporting if not NULL.
5679  *
5680  * @return
5681  *   0 on success, a negative errno value otherwise and rte_errno is set.
5682  */
5683 static int
5684 flow_tcf_query_count(struct rte_eth_dev *dev,
5685                           struct rte_flow *flow,
5686                           void *data,
5687                           struct rte_flow_error *error)
5688 {
5689         struct flow_tcf_stats_basic sb_data;
5690         struct rte_flow_query_count *qc = data;
5691         struct priv *priv = dev->data->dev_private;
5692         struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
5693         struct mnl_socket *nl = ctx->nl;
5694         struct mlx5_flow *dev_flow;
5695         struct nlmsghdr *nlh;
5696         uint32_t seq = priv->tcf_context->seq++;
5697         ssize_t ret;
5698         assert(qc);
5699
5700         memset(&sb_data, 0, sizeof(sb_data));
5701         dev_flow = LIST_FIRST(&flow->dev_flows);
5702         /* E-Switch flow can't be expanded. */
5703         assert(!LIST_NEXT(dev_flow, next));
5704         if (!dev_flow->flow->counter)
5705                 goto notsup_exit;
5706         nlh = dev_flow->tcf.nlh;
5707         nlh->nlmsg_type = RTM_GETTFILTER;
5708         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_ECHO;
5709         nlh->nlmsg_seq = seq;
5710         if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) == -1)
5711                 goto error_exit;
5712         do {
5713                 ret = mnl_socket_recvfrom(nl, ctx->buf, ctx->buf_size);
5714                 if (ret <= 0)
5715                         break;
5716                 ret = mnl_cb_run(ctx->buf, ret, seq,
5717                                  mnl_socket_get_portid(nl),
5718                                  flow_tcf_nl_message_get_stats_basic,
5719                                  (void *)&sb_data);
5720         } while (ret > 0);
5721         /* Return the delta from last reset. */
5722         if (sb_data.valid) {
5723                 /* Return the delta from last reset. */
5724                 qc->hits_set = 1;
5725                 qc->bytes_set = 1;
5726                 qc->hits = sb_data.counters.packets - flow->counter->hits;
5727                 qc->bytes = sb_data.counters.bytes - flow->counter->bytes;
5728                 if (qc->reset) {
5729                         flow->counter->hits = sb_data.counters.packets;
5730                         flow->counter->bytes = sb_data.counters.bytes;
5731                 }
5732                 return 0;
5733         }
5734         return rte_flow_error_set(error, EINVAL,
5735                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5736                                   NULL,
5737                                   "flow does not have counter");
5738 error_exit:
5739         return rte_flow_error_set
5740                         (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5741                          NULL, "netlink: failed to read flow rule counters");
5742 notsup_exit:
5743         return rte_flow_error_set
5744                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5745                          NULL, "counters are not available.");
5746 }
5747
5748 /**
5749  * Query a flow.
5750  *
5751  * @see rte_flow_query()
5752  * @see rte_flow_ops
5753  */
5754 static int
5755 flow_tcf_query(struct rte_eth_dev *dev,
5756                struct rte_flow *flow,
5757                const struct rte_flow_action *actions,
5758                void *data,
5759                struct rte_flow_error *error)
5760 {
5761         int ret = -EINVAL;
5762
5763         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5764                 switch (actions->type) {
5765                 case RTE_FLOW_ACTION_TYPE_VOID:
5766                         break;
5767                 case RTE_FLOW_ACTION_TYPE_COUNT:
5768                         ret = flow_tcf_query_count(dev, flow, data, error);
5769                         break;
5770                 default:
5771                         return rte_flow_error_set(error, ENOTSUP,
5772                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5773                                                   actions,
5774                                                   "action not supported");
5775                 }
5776         }
5777         return ret;
5778 }
5779
5780 const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops = {
5781         .validate = flow_tcf_validate,
5782         .prepare = flow_tcf_prepare,
5783         .translate = flow_tcf_translate,
5784         .apply = flow_tcf_apply,
5785         .remove = flow_tcf_remove,
5786         .destroy = flow_tcf_destroy,
5787         .query = flow_tcf_query,
5788 };
5789
5790 /**
5791  * Create and configure a libmnl socket for Netlink flow rules.
5792  *
5793  * @return
5794  *   A valid libmnl socket object pointer on success, NULL otherwise and
5795  *   rte_errno is set.
5796  */
5797 static struct mnl_socket *
5798 flow_tcf_mnl_socket_create(void)
5799 {
5800         struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
5801
5802         if (nl) {
5803                 mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
5804                                       sizeof(int));
5805                 if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
5806                         return nl;
5807         }
5808         rte_errno = errno;
5809         if (nl)
5810                 mnl_socket_close(nl);
5811         return NULL;
5812 }
5813
5814 /**
5815  * Destroy a libmnl socket.
5816  *
5817  * @param nl
5818  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5819  */
5820 static void
5821 flow_tcf_mnl_socket_destroy(struct mnl_socket *nl)
5822 {
5823         if (nl)
5824                 mnl_socket_close(nl);
5825 }
5826
5827 /**
5828  * Initialize ingress qdisc of a given network interface.
5829  *
5830  * @param ctx
5831  *   Pointer to tc-flower context to use.
5832  * @param ifindex
5833  *   Index of network interface to initialize.
5834  * @param[out] error
5835  *   Perform verbose error reporting if not NULL.
5836  *
5837  * @return
5838  *   0 on success, a negative errno value otherwise and rte_errno is set.
5839  */
5840 int
5841 mlx5_flow_tcf_init(struct mlx5_flow_tcf_context *ctx,
5842                    unsigned int ifindex, struct rte_flow_error *error)
5843 {
5844         struct nlmsghdr *nlh;
5845         struct tcmsg *tcm;
5846         alignas(struct nlmsghdr)
5847         uint8_t buf[mnl_nlmsg_size(sizeof(*tcm)) +
5848                     SZ_NLATTR_STRZ_OF("ingress") +
5849                     MNL_BUF_EXTRA_SPACE];
5850
5851         /* Destroy existing ingress qdisc and everything attached to it. */
5852         nlh = mnl_nlmsg_put_header(buf);
5853         nlh->nlmsg_type = RTM_DELQDISC;
5854         nlh->nlmsg_flags = NLM_F_REQUEST;
5855         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5856         tcm->tcm_family = AF_UNSPEC;
5857         tcm->tcm_ifindex = ifindex;
5858         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5859         tcm->tcm_parent = TC_H_INGRESS;
5860         assert(sizeof(buf) >= nlh->nlmsg_len);
5861         /* Ignore errors when qdisc is already absent. */
5862         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL) &&
5863             rte_errno != EINVAL && rte_errno != ENOENT)
5864                 return rte_flow_error_set(error, rte_errno,
5865                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5866                                           "netlink: failed to remove ingress"
5867                                           " qdisc");
5868         /* Create fresh ingress qdisc. */
5869         nlh = mnl_nlmsg_put_header(buf);
5870         nlh->nlmsg_type = RTM_NEWQDISC;
5871         nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
5872         tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
5873         tcm->tcm_family = AF_UNSPEC;
5874         tcm->tcm_ifindex = ifindex;
5875         tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
5876         tcm->tcm_parent = TC_H_INGRESS;
5877         mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
5878         assert(sizeof(buf) >= nlh->nlmsg_len);
5879         if (flow_tcf_nl_ack(ctx, nlh, NULL, NULL))
5880                 return rte_flow_error_set(error, rte_errno,
5881                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5882                                           "netlink: failed to create ingress"
5883                                           " qdisc");
5884         return 0;
5885 }
5886
5887 /**
5888  * Create libmnl context for Netlink flow rules.
5889  *
5890  * @return
5891  *   A valid libmnl socket object pointer on success, NULL otherwise and
5892  *   rte_errno is set.
5893  */
5894 struct mlx5_flow_tcf_context *
5895 mlx5_flow_tcf_context_create(void)
5896 {
5897         struct mlx5_flow_tcf_context *ctx = rte_zmalloc(__func__,
5898                                                         sizeof(*ctx),
5899                                                         sizeof(uint32_t));
5900         if (!ctx)
5901                 goto error;
5902         ctx->nl = flow_tcf_mnl_socket_create();
5903         if (!ctx->nl)
5904                 goto error;
5905         ctx->buf_size = MNL_SOCKET_BUFFER_SIZE;
5906         ctx->buf = rte_zmalloc(__func__,
5907                                ctx->buf_size, sizeof(uint32_t));
5908         if (!ctx->buf)
5909                 goto error;
5910         ctx->seq = random();
5911         return ctx;
5912 error:
5913         mlx5_flow_tcf_context_destroy(ctx);
5914         return NULL;
5915 }
5916
5917 /**
5918  * Destroy a libmnl context.
5919  *
5920  * @param ctx
5921  *   Libmnl socket of the @p NETLINK_ROUTE kind.
5922  */
5923 void
5924 mlx5_flow_tcf_context_destroy(struct mlx5_flow_tcf_context *ctx)
5925 {
5926         if (!ctx)
5927                 return;
5928         flow_tcf_mnl_socket_destroy(ctx->nl);
5929         rte_free(ctx->buf);
5930         rte_free(ctx);
5931 }