lib/ethdev/rte_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stddef.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_errno.h>
  13 #include <rte_branch_prediction.h>
  14 #include <rte_string_fns.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_mbuf_dyn.h>
  17 #include "rte_ethdev.h"
  18 #include "rte_flow_driver.h"
  19 #include "rte_flow.h"
  20
  21 /* Mbuf dynamic field name for metadata. */
  22 int32_t rte_flow_dynf_metadata_offs = -1;
  23
  24 /* Mbuf dynamic field flag bit number for metadata. */
  25 uint64_t rte_flow_dynf_metadata_mask;
  26
  27 /**
  28  * Flow elements description tables.
  29  */
  30 struct rte_flow_desc_data {
  31         const char *name;
  32         size_t size;
  33 };
  34
  35 /** Generate flow_item[] entry. */
  36 #define MK_FLOW_ITEM(t, s) \
  37         [RTE_FLOW_ITEM_TYPE_ ## t] = { \
  38                 .name = # t, \
  39                 .size = s, \
  40         }
  41
  42 /** Information about known flow pattern items. */
  43 static const struct rte_flow_desc_data rte_flow_desc_item[] = {
  44         MK_FLOW_ITEM(END, 0),
  45         MK_FLOW_ITEM(VOID, 0),
  46         MK_FLOW_ITEM(INVERT, 0),
  47         MK_FLOW_ITEM(ANY, sizeof(struct rte_flow_item_any)),
  48         MK_FLOW_ITEM(PF, 0),
  49         MK_FLOW_ITEM(VF, sizeof(struct rte_flow_item_vf)),
  50         MK_FLOW_ITEM(PHY_PORT, sizeof(struct rte_flow_item_phy_port)),
  51         MK_FLOW_ITEM(PORT_ID, sizeof(struct rte_flow_item_port_id)),
  52         MK_FLOW_ITEM(RAW, sizeof(struct rte_flow_item_raw)),
  53         MK_FLOW_ITEM(ETH, sizeof(struct rte_flow_item_eth)),
  54         MK_FLOW_ITEM(VLAN, sizeof(struct rte_flow_item_vlan)),
  55         MK_FLOW_ITEM(IPV4, sizeof(struct rte_flow_item_ipv4)),
  56         MK_FLOW_ITEM(IPV6, sizeof(struct rte_flow_item_ipv6)),
  57         MK_FLOW_ITEM(ICMP, sizeof(struct rte_flow_item_icmp)),
  58         MK_FLOW_ITEM(UDP, sizeof(struct rte_flow_item_udp)),
  59         MK_FLOW_ITEM(TCP, sizeof(struct rte_flow_item_tcp)),
  60         MK_FLOW_ITEM(SCTP, sizeof(struct rte_flow_item_sctp)),
  61         MK_FLOW_ITEM(VXLAN, sizeof(struct rte_flow_item_vxlan)),
  62         MK_FLOW_ITEM(E_TAG, sizeof(struct rte_flow_item_e_tag)),
  63         MK_FLOW_ITEM(NVGRE, sizeof(struct rte_flow_item_nvgre)),
  64         MK_FLOW_ITEM(MPLS, sizeof(struct rte_flow_item_mpls)),
  65         MK_FLOW_ITEM(GRE, sizeof(struct rte_flow_item_gre)),
  66         MK_FLOW_ITEM(FUZZY, sizeof(struct rte_flow_item_fuzzy)),
  67         MK_FLOW_ITEM(GTP, sizeof(struct rte_flow_item_gtp)),
  68         MK_FLOW_ITEM(GTPC, sizeof(struct rte_flow_item_gtp)),
  69         MK_FLOW_ITEM(GTPU, sizeof(struct rte_flow_item_gtp)),
  70         MK_FLOW_ITEM(ESP, sizeof(struct rte_flow_item_esp)),
  71         MK_FLOW_ITEM(GENEVE, sizeof(struct rte_flow_item_geneve)),
  72         MK_FLOW_ITEM(VXLAN_GPE, sizeof(struct rte_flow_item_vxlan_gpe)),
  73         MK_FLOW_ITEM(ARP_ETH_IPV4, sizeof(struct rte_flow_item_arp_eth_ipv4)),
  74         MK_FLOW_ITEM(IPV6_EXT, sizeof(struct rte_flow_item_ipv6_ext)),
  75         MK_FLOW_ITEM(IPV6_FRAG_EXT, sizeof(struct rte_flow_item_ipv6_frag_ext)),
  76         MK_FLOW_ITEM(ICMP6, sizeof(struct rte_flow_item_icmp6)),
  77         MK_FLOW_ITEM(ICMP6_ND_NS, sizeof(struct rte_flow_item_icmp6_nd_ns)),
  78         MK_FLOW_ITEM(ICMP6_ND_NA, sizeof(struct rte_flow_item_icmp6_nd_na)),
  79         MK_FLOW_ITEM(ICMP6_ND_OPT, sizeof(struct rte_flow_item_icmp6_nd_opt)),
  80         MK_FLOW_ITEM(ICMP6_ND_OPT_SLA_ETH,
  81                      sizeof(struct rte_flow_item_icmp6_nd_opt_sla_eth)),
  82         MK_FLOW_ITEM(ICMP6_ND_OPT_TLA_ETH,
  83                      sizeof(struct rte_flow_item_icmp6_nd_opt_tla_eth)),
  84         MK_FLOW_ITEM(MARK, sizeof(struct rte_flow_item_mark)),
  85         MK_FLOW_ITEM(META, sizeof(struct rte_flow_item_meta)),
  86         MK_FLOW_ITEM(TAG, sizeof(struct rte_flow_item_tag)),
  87         MK_FLOW_ITEM(GRE_KEY, sizeof(rte_be32_t)),
  88         MK_FLOW_ITEM(GTP_PSC, sizeof(struct rte_flow_item_gtp_psc)),
  89         MK_FLOW_ITEM(PPPOES, sizeof(struct rte_flow_item_pppoe)),
  90         MK_FLOW_ITEM(PPPOED, sizeof(struct rte_flow_item_pppoe)),
  91         MK_FLOW_ITEM(PPPOE_PROTO_ID,
  92                         sizeof(struct rte_flow_item_pppoe_proto_id)),
  93         MK_FLOW_ITEM(NSH, sizeof(struct rte_flow_item_nsh)),
  94         MK_FLOW_ITEM(IGMP, sizeof(struct rte_flow_item_igmp)),
  95         MK_FLOW_ITEM(AH, sizeof(struct rte_flow_item_ah)),
  96         MK_FLOW_ITEM(HIGIG2, sizeof(struct rte_flow_item_higig2_hdr)),
  97         MK_FLOW_ITEM(L2TPV3OIP, sizeof(struct rte_flow_item_l2tpv3oip)),
  98         MK_FLOW_ITEM(PFCP, sizeof(struct rte_flow_item_pfcp)),
  99         MK_FLOW_ITEM(ECPRI, sizeof(struct rte_flow_item_ecpri)),
 100         MK_FLOW_ITEM(GENEVE_OPT, sizeof(struct rte_flow_item_geneve_opt)),
 101         MK_FLOW_ITEM(INTEGRITY, sizeof(struct rte_flow_item_integrity)),
 102         MK_FLOW_ITEM(CONNTRACK, sizeof(uint32_t)),
 103         MK_FLOW_ITEM(PORT_REPRESENTOR, sizeof(struct rte_flow_item_ethdev)),
 104         MK_FLOW_ITEM(REPRESENTED_PORT, sizeof(struct rte_flow_item_ethdev)),
 105 };
 106
 107 /** Generate flow_action[] entry. */
 108 #define MK_FLOW_ACTION(t, s) \
 109         [RTE_FLOW_ACTION_TYPE_ ## t] = { \
 110                 .name = # t, \
 111                 .size = s, \
 112         }
 113
 114 /** Information about known flow actions. */
 115 static const struct rte_flow_desc_data rte_flow_desc_action[] = {
 116         MK_FLOW_ACTION(END, 0),
 117         MK_FLOW_ACTION(VOID, 0),
 118         MK_FLOW_ACTION(PASSTHRU, 0),
 119         MK_FLOW_ACTION(JUMP, sizeof(struct rte_flow_action_jump)),
 120         MK_FLOW_ACTION(MARK, sizeof(struct rte_flow_action_mark)),
 121         MK_FLOW_ACTION(FLAG, 0),
 122         MK_FLOW_ACTION(QUEUE, sizeof(struct rte_flow_action_queue)),
 123         MK_FLOW_ACTION(DROP, 0),
 124         MK_FLOW_ACTION(COUNT, sizeof(struct rte_flow_action_count)),
 125         MK_FLOW_ACTION(RSS, sizeof(struct rte_flow_action_rss)),
 126         MK_FLOW_ACTION(PF, 0),
 127         MK_FLOW_ACTION(VF, sizeof(struct rte_flow_action_vf)),
 128         MK_FLOW_ACTION(PHY_PORT, sizeof(struct rte_flow_action_phy_port)),
 129         MK_FLOW_ACTION(PORT_ID, sizeof(struct rte_flow_action_port_id)),
 130         MK_FLOW_ACTION(METER, sizeof(struct rte_flow_action_meter)),
 131         MK_FLOW_ACTION(SECURITY, sizeof(struct rte_flow_action_security)),
 132         MK_FLOW_ACTION(OF_SET_MPLS_TTL,
 133                        sizeof(struct rte_flow_action_of_set_mpls_ttl)),
 134         MK_FLOW_ACTION(OF_DEC_MPLS_TTL, 0),
 135         MK_FLOW_ACTION(OF_SET_NW_TTL,
 136                        sizeof(struct rte_flow_action_of_set_nw_ttl)),
 137         MK_FLOW_ACTION(OF_DEC_NW_TTL, 0),
 138         MK_FLOW_ACTION(OF_COPY_TTL_OUT, 0),
 139         MK_FLOW_ACTION(OF_COPY_TTL_IN, 0),
 140         MK_FLOW_ACTION(OF_POP_VLAN, 0),
 141         MK_FLOW_ACTION(OF_PUSH_VLAN,
 142                        sizeof(struct rte_flow_action_of_push_vlan)),
 143         MK_FLOW_ACTION(OF_SET_VLAN_VID,
 144                        sizeof(struct rte_flow_action_of_set_vlan_vid)),
 145         MK_FLOW_ACTION(OF_SET_VLAN_PCP,
 146                        sizeof(struct rte_flow_action_of_set_vlan_pcp)),
 147         MK_FLOW_ACTION(OF_POP_MPLS,
 148                        sizeof(struct rte_flow_action_of_pop_mpls)),
 149         MK_FLOW_ACTION(OF_PUSH_MPLS,
 150                        sizeof(struct rte_flow_action_of_push_mpls)),
 151         MK_FLOW_ACTION(VXLAN_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)),
 152         MK_FLOW_ACTION(VXLAN_DECAP, 0),
 153         MK_FLOW_ACTION(NVGRE_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)),
 154         MK_FLOW_ACTION(NVGRE_DECAP, 0),
 155         MK_FLOW_ACTION(RAW_ENCAP, sizeof(struct rte_flow_action_raw_encap)),
 156         MK_FLOW_ACTION(RAW_DECAP, sizeof(struct rte_flow_action_raw_decap)),
 157         MK_FLOW_ACTION(SET_IPV4_SRC,
 158                        sizeof(struct rte_flow_action_set_ipv4)),
 159         MK_FLOW_ACTION(SET_IPV4_DST,
 160                        sizeof(struct rte_flow_action_set_ipv4)),
 161         MK_FLOW_ACTION(SET_IPV6_SRC,
 162                        sizeof(struct rte_flow_action_set_ipv6)),
 163         MK_FLOW_ACTION(SET_IPV6_DST,
 164                        sizeof(struct rte_flow_action_set_ipv6)),
 165         MK_FLOW_ACTION(SET_TP_SRC,
 166                        sizeof(struct rte_flow_action_set_tp)),
 167         MK_FLOW_ACTION(SET_TP_DST,
 168                        sizeof(struct rte_flow_action_set_tp)),
 169         MK_FLOW_ACTION(MAC_SWAP, 0),
 170         MK_FLOW_ACTION(DEC_TTL, 0),
 171         MK_FLOW_ACTION(SET_TTL, sizeof(struct rte_flow_action_set_ttl)),
 172         MK_FLOW_ACTION(SET_MAC_SRC, sizeof(struct rte_flow_action_set_mac)),
 173         MK_FLOW_ACTION(SET_MAC_DST, sizeof(struct rte_flow_action_set_mac)),
 174         MK_FLOW_ACTION(INC_TCP_SEQ, sizeof(rte_be32_t)),
 175         MK_FLOW_ACTION(DEC_TCP_SEQ, sizeof(rte_be32_t)),
 176         MK_FLOW_ACTION(INC_TCP_ACK, sizeof(rte_be32_t)),
 177         MK_FLOW_ACTION(DEC_TCP_ACK, sizeof(rte_be32_t)),
 178         MK_FLOW_ACTION(SET_TAG, sizeof(struct rte_flow_action_set_tag)),
 179         MK_FLOW_ACTION(SET_META, sizeof(struct rte_flow_action_set_meta)),
 180         MK_FLOW_ACTION(SET_IPV4_DSCP, sizeof(struct rte_flow_action_set_dscp)),
 181         MK_FLOW_ACTION(SET_IPV6_DSCP, sizeof(struct rte_flow_action_set_dscp)),
 182         MK_FLOW_ACTION(AGE, sizeof(struct rte_flow_action_age)),
 183         MK_FLOW_ACTION(SAMPLE, sizeof(struct rte_flow_action_sample)),
 184         MK_FLOW_ACTION(MODIFY_FIELD,
 185                        sizeof(struct rte_flow_action_modify_field)),
 186         /**
 187          * Indirect action represented as handle of type
 188          * (struct rte_flow_action_handle *) stored in conf field (see
 189          * struct rte_flow_action); no need for additional structure to * store
 190          * indirect action handle.
 191          */
 192         MK_FLOW_ACTION(INDIRECT, 0),
 193         MK_FLOW_ACTION(CONNTRACK, sizeof(struct rte_flow_action_conntrack)),
 194         MK_FLOW_ACTION(PORT_REPRESENTOR, sizeof(struct rte_flow_action_ethdev)),
 195 };
 196
 197 int
 198 rte_flow_dynf_metadata_register(void)
 199 {
 200         int offset;
 201         int flag;
 202
 203         static const struct rte_mbuf_dynfield desc_offs = {
 204                 .name = RTE_MBUF_DYNFIELD_METADATA_NAME,
 205                 .size = sizeof(uint32_t),
 206                 .align = __alignof__(uint32_t),
 207         };
 208         static const struct rte_mbuf_dynflag desc_flag = {
 209                 .name = RTE_MBUF_DYNFLAG_METADATA_NAME,
 210         };
 211
 212         offset = rte_mbuf_dynfield_register(&desc_offs);
 213         if (offset < 0)
 214                 goto error;
 215         flag = rte_mbuf_dynflag_register(&desc_flag);
 216         if (flag < 0)
 217                 goto error;
 218         rte_flow_dynf_metadata_offs = offset;
 219         rte_flow_dynf_metadata_mask = (1ULL << flag);
 220         return 0;
 221
 222 error:
 223         rte_flow_dynf_metadata_offs = -1;
 224         rte_flow_dynf_metadata_mask = 0ULL;
 225         return -rte_errno;
 226 }
 227
 228 static inline void
 229 fts_enter(struct rte_eth_dev *dev)
 230 {
 231         if (!(dev->data->dev_flags & RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE))
 232                 pthread_mutex_lock(&dev->data->flow_ops_mutex);
 233 }
 234
 235 static inline void
 236 fts_exit(struct rte_eth_dev *dev)
 237 {
 238         if (!(dev->data->dev_flags & RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE))
 239                 pthread_mutex_unlock(&dev->data->flow_ops_mutex);
 240 }
 241
 242 static int
 243 flow_err(uint16_t port_id, int ret, struct rte_flow_error *error)
 244 {
 245         if (ret == 0)
 246                 return 0;
 247         if (rte_eth_dev_is_removed(port_id))
 248                 return rte_flow_error_set(error, EIO,
 249                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 250                                           NULL, rte_strerror(EIO));
 251         return ret;
 252 }
 253
 254 /* Get generic flow operations structure from a port. */
 255 const struct rte_flow_ops *
 256 rte_flow_ops_get(uint16_t port_id, struct rte_flow_error *error)
 257 {
 258         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 259         const struct rte_flow_ops *ops;
 260         int code;
 261
 262         if (unlikely(!rte_eth_dev_is_valid_port(port_id)))
 263                 code = ENODEV;
 264         else if (unlikely(dev->dev_ops->flow_ops_get == NULL))
 265                 /* flow API not supported with this driver dev_ops */
 266                 code = ENOSYS;
 267         else
 268                 code = dev->dev_ops->flow_ops_get(dev, &ops);
 269         if (code == 0 && ops == NULL)
 270                 /* flow API not supported with this device */
 271                 code = ENOSYS;
 272
 273         if (code != 0) {
 274                 rte_flow_error_set(error, code, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 275                                    NULL, rte_strerror(code));
 276                 return NULL;
 277         }
 278         return ops;
 279 }
 280
 281 /* Check whether a flow rule can be created on a given port. */
 282 int
 283 rte_flow_validate(uint16_t port_id,
 284                   const struct rte_flow_attr *attr,
 285                   const struct rte_flow_item pattern[],
 286                   const struct rte_flow_action actions[],
 287                   struct rte_flow_error *error)
 288 {
 289         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 290         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 291         int ret;
 292
 293         if (unlikely(!ops))
 294                 return -rte_errno;
 295         if (likely(!!ops->validate)) {
 296                 fts_enter(dev);
 297                 ret = ops->validate(dev, attr, pattern, actions, error);
 298                 fts_exit(dev);
 299                 return flow_err(port_id, ret, error);
 300         }
 301         return rte_flow_error_set(error, ENOSYS,
 302                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 303                                   NULL, rte_strerror(ENOSYS));
 304 }
 305
 306 /* Create a flow rule on a given port. */
 307 struct rte_flow *
 308 rte_flow_create(uint16_t port_id,
 309                 const struct rte_flow_attr *attr,
 310                 const struct rte_flow_item pattern[],
 311                 const struct rte_flow_action actions[],
 312                 struct rte_flow_error *error)
 313 {
 314         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 315         struct rte_flow *flow;
 316         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 317
 318         if (unlikely(!ops))
 319                 return NULL;
 320         if (likely(!!ops->create)) {
 321                 fts_enter(dev);
 322                 flow = ops->create(dev, attr, pattern, actions, error);
 323                 fts_exit(dev);
 324                 if (flow == NULL)
 325                         flow_err(port_id, -rte_errno, error);
 326                 return flow;
 327         }
 328         rte_flow_error_set(error, ENOSYS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 329                            NULL, rte_strerror(ENOSYS));
 330         return NULL;
 331 }
 332
 333 /* Destroy a flow rule on a given port. */
 334 int
 335 rte_flow_destroy(uint16_t port_id,
 336                  struct rte_flow *flow,
 337                  struct rte_flow_error *error)
 338 {
 339         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 340         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 341         int ret;
 342
 343         if (unlikely(!ops))
 344                 return -rte_errno;
 345         if (likely(!!ops->destroy)) {
 346                 fts_enter(dev);
 347                 ret = ops->destroy(dev, flow, error);
 348                 fts_exit(dev);
 349                 return flow_err(port_id, ret, error);
 350         }
 351         return rte_flow_error_set(error, ENOSYS,
 352                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 353                                   NULL, rte_strerror(ENOSYS));
 354 }
 355
 356 /* Destroy all flow rules associated with a port. */
 357 int
 358 rte_flow_flush(uint16_t port_id,
 359                struct rte_flow_error *error)
 360 {
 361         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 362         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 363         int ret;
 364
 365         if (unlikely(!ops))
 366                 return -rte_errno;
 367         if (likely(!!ops->flush)) {
 368                 fts_enter(dev);
 369                 ret = ops->flush(dev, error);
 370                 fts_exit(dev);
 371                 return flow_err(port_id, ret, error);
 372         }
 373         return rte_flow_error_set(error, ENOSYS,
 374                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 375                                   NULL, rte_strerror(ENOSYS));
 376 }
 377
 378 /* Query an existing flow rule. */
 379 int
 380 rte_flow_query(uint16_t port_id,
 381                struct rte_flow *flow,
 382                const struct rte_flow_action *action,
 383                void *data,
 384                struct rte_flow_error *error)
 385 {
 386         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 387         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 388         int ret;
 389
 390         if (!ops)
 391                 return -rte_errno;
 392         if (likely(!!ops->query)) {
 393                 fts_enter(dev);
 394                 ret = ops->query(dev, flow, action, data, error);
 395                 fts_exit(dev);
 396                 return flow_err(port_id, ret, error);
 397         }
 398         return rte_flow_error_set(error, ENOSYS,
 399                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 400                                   NULL, rte_strerror(ENOSYS));
 401 }
 402
 403 /* Restrict ingress traffic to the defined flow rules. */
 404 int
 405 rte_flow_isolate(uint16_t port_id,
 406                  int set,
 407                  struct rte_flow_error *error)
 408 {
 409         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 410         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 411         int ret;
 412
 413         if (!ops)
 414                 return -rte_errno;
 415         if (likely(!!ops->isolate)) {
 416                 fts_enter(dev);
 417                 ret = ops->isolate(dev, set, error);
 418                 fts_exit(dev);
 419                 return flow_err(port_id, ret, error);
 420         }
 421         return rte_flow_error_set(error, ENOSYS,
 422                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 423                                   NULL, rte_strerror(ENOSYS));
 424 }
 425
 426 /* Initialize flow error structure. */
 427 int
 428 rte_flow_error_set(struct rte_flow_error *error,
 429                    int code,
 430                    enum rte_flow_error_type type,
 431                    const void *cause,
 432                    const char *message)
 433 {
 434         if (error) {
 435                 *error = (struct rte_flow_error){
 436                         .type = type,
 437                         .cause = cause,
 438                         .message = message,
 439                 };
 440         }
 441         rte_errno = code;
 442         return -code;
 443 }
 444
 445 /** Pattern item specification types. */
 446 enum rte_flow_conv_item_spec_type {
 447         RTE_FLOW_CONV_ITEM_SPEC,
 448         RTE_FLOW_CONV_ITEM_LAST,
 449         RTE_FLOW_CONV_ITEM_MASK,
 450 };
 451
 452 /**
 453  * Copy pattern item specification.
 454  *
 455  * @param[out] buf
 456  *   Output buffer. Can be NULL if @p size is zero.
 457  * @param size
 458  *   Size of @p buf in bytes.
 459  * @param[in] item
 460  *   Pattern item to copy specification from.
 461  * @param type
 462  *   Specification selector for either @p spec, @p last or @p mask.
 463  *
 464  * @return
 465  *   Number of bytes needed to store pattern item specification regardless
 466  *   of @p size. @p buf contents are truncated to @p size if not large
 467  *   enough.
 468  */
 469 static size_t
 470 rte_flow_conv_item_spec(void *buf, const size_t size,
 471                         const struct rte_flow_item *item,
 472                         enum rte_flow_conv_item_spec_type type)
 473 {
 474         size_t off;
 475         const void *data =
 476                 type == RTE_FLOW_CONV_ITEM_SPEC ? item->spec :
 477                 type == RTE_FLOW_CONV_ITEM_LAST ? item->last :
 478                 type == RTE_FLOW_CONV_ITEM_MASK ? item->mask :
 479                 NULL;
 480
 481         switch (item->type) {
 482                 union {
 483                         const struct rte_flow_item_raw *raw;
 484                 } spec;
 485                 union {
 486                         const struct rte_flow_item_raw *raw;
 487                 } last;
 488                 union {
 489                         const struct rte_flow_item_raw *raw;
 490                 } mask;
 491                 union {
 492                         const struct rte_flow_item_raw *raw;
 493                 } src;
 494                 union {
 495                         struct rte_flow_item_raw *raw;
 496                 } dst;
 497                 size_t tmp;
 498
 499         case RTE_FLOW_ITEM_TYPE_RAW:
 500                 spec.raw = item->spec;
 501                 last.raw = item->last ? item->last : item->spec;
 502                 mask.raw = item->mask ? item->mask : &rte_flow_item_raw_mask;
 503                 src.raw = data;
 504                 dst.raw = buf;
 505                 rte_memcpy(dst.raw,
 506                            (&(struct rte_flow_item_raw){
 507                                 .relative = src.raw->relative,
 508                                 .search = src.raw->search,
 509                                 .reserved = src.raw->reserved,
 510                                 .offset = src.raw->offset,
 511                                 .limit = src.raw->limit,
 512                                 .length = src.raw->length,
 513                            }),
 514                            size > sizeof(*dst.raw) ? sizeof(*dst.raw) : size);
 515                 off = sizeof(*dst.raw);
 516                 if (type == RTE_FLOW_CONV_ITEM_SPEC ||
 517                     (type == RTE_FLOW_CONV_ITEM_MASK &&
 518                      ((spec.raw->length & mask.raw->length) >=
 519                       (last.raw->length & mask.raw->length))))
 520                         tmp = spec.raw->length & mask.raw->length;
 521                 else
 522                         tmp = last.raw->length & mask.raw->length;
 523                 if (tmp) {
 524                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.raw->pattern));
 525                         if (size >= off + tmp)
 526                                 dst.raw->pattern = rte_memcpy
 527                                         ((void *)((uintptr_t)dst.raw + off),
 528                                          src.raw->pattern, tmp);
 529                         off += tmp;
 530                 }
 531                 break;
 532         default:
 533                 /**
 534                  * allow PMD private flow item
 535                  */
 536                 off = (int)item->type >= 0 ?
 537                       rte_flow_desc_item[item->type].size : sizeof(void *);
 538                 rte_memcpy(buf, data, (size > off ? off : size));
 539                 break;
 540         }
 541         return off;
 542 }
 543
 544 /**
 545  * Copy action configuration.
 546  *
 547  * @param[out] buf
 548  *   Output buffer. Can be NULL if @p size is zero.
 549  * @param size
 550  *   Size of @p buf in bytes.
 551  * @param[in] action
 552  *   Action to copy configuration from.
 553  *
 554  * @return
 555  *   Number of bytes needed to store pattern item specification regardless
 556  *   of @p size. @p buf contents are truncated to @p size if not large
 557  *   enough.
 558  */
 559 static size_t
 560 rte_flow_conv_action_conf(void *buf, const size_t size,
 561                           const struct rte_flow_action *action)
 562 {
 563         size_t off;
 564
 565         switch (action->type) {
 566                 union {
 567                         const struct rte_flow_action_rss *rss;
 568                         const struct rte_flow_action_vxlan_encap *vxlan_encap;
 569                         const struct rte_flow_action_nvgre_encap *nvgre_encap;
 570                 } src;
 571                 union {
 572                         struct rte_flow_action_rss *rss;
 573                         struct rte_flow_action_vxlan_encap *vxlan_encap;
 574                         struct rte_flow_action_nvgre_encap *nvgre_encap;
 575                 } dst;
 576                 size_t tmp;
 577                 int ret;
 578
 579         case RTE_FLOW_ACTION_TYPE_RSS:
 580                 src.rss = action->conf;
 581                 dst.rss = buf;
 582                 rte_memcpy(dst.rss,
 583                            (&(struct rte_flow_action_rss){
 584                                 .func = src.rss->func,
 585                                 .level = src.rss->level,
 586                                 .types = src.rss->types,
 587                                 .key_len = src.rss->key_len,
 588                                 .queue_num = src.rss->queue_num,
 589                            }),
 590                            size > sizeof(*dst.rss) ? sizeof(*dst.rss) : size);
 591                 off = sizeof(*dst.rss);
 592                 if (src.rss->key_len && src.rss->key) {
 593                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->key));
 594                         tmp = sizeof(*src.rss->key) * src.rss->key_len;
 595                         if (size >= off + tmp)
 596                                 dst.rss->key = rte_memcpy
 597                                         ((void *)((uintptr_t)dst.rss + off),
 598                                          src.rss->key, tmp);
 599                         off += tmp;
 600                 }
 601                 if (src.rss->queue_num) {
 602                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->queue));
 603                         tmp = sizeof(*src.rss->queue) * src.rss->queue_num;
 604                         if (size >= off + tmp)
 605                                 dst.rss->queue = rte_memcpy
 606                                         ((void *)((uintptr_t)dst.rss + off),
 607                                          src.rss->queue, tmp);
 608                         off += tmp;
 609                 }
 610                 break;
 611         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
 612         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
 613                 src.vxlan_encap = action->conf;
 614                 dst.vxlan_encap = buf;
 615                 RTE_BUILD_BUG_ON(sizeof(*src.vxlan_encap) !=
 616                                  sizeof(*src.nvgre_encap) ||
 617                                  offsetof(struct rte_flow_action_vxlan_encap,
 618                                           definition) !=
 619                                  offsetof(struct rte_flow_action_nvgre_encap,
 620                                           definition));
 621                 off = sizeof(*dst.vxlan_encap);
 622                 if (src.vxlan_encap->definition) {
 623                         off = RTE_ALIGN_CEIL
 624                                 (off, sizeof(*dst.vxlan_encap->definition));
 625                         ret = rte_flow_conv
 626                                 (RTE_FLOW_CONV_OP_PATTERN,
 627                                  (void *)((uintptr_t)dst.vxlan_encap + off),
 628                                  size > off ? size - off : 0,
 629                                  src.vxlan_encap->definition, NULL);
 630                         if (ret < 0)
 631                                 return 0;
 632                         if (size >= off + ret)
 633                                 dst.vxlan_encap->definition =
 634                                         (void *)((uintptr_t)dst.vxlan_encap +
 635                                                  off);
 636                         off += ret;
 637                 }
 638                 break;
 639         default:
 640                 /**
 641                  * allow PMD private flow action
 642                  */
 643                 off = (int)action->type >= 0 ?
 644                       rte_flow_desc_action[action->type].size : sizeof(void *);
 645                 rte_memcpy(buf, action->conf, (size > off ? off : size));
 646                 break;
 647         }
 648         return off;
 649 }
 650
 651 /**
 652  * Copy a list of pattern items.
 653  *
 654  * @param[out] dst
 655  *   Destination buffer. Can be NULL if @p size is zero.
 656  * @param size
 657  *   Size of @p dst in bytes.
 658  * @param[in] src
 659  *   Source pattern items.
 660  * @param num
 661  *   Maximum number of pattern items to process from @p src or 0 to process
 662  *   the entire list. In both cases, processing stops after
 663  *   RTE_FLOW_ITEM_TYPE_END is encountered.
 664  * @param[out] error
 665  *   Perform verbose error reporting if not NULL.
 666  *
 667  * @return
 668  *   A positive value representing the number of bytes needed to store
 669  *   pattern items regardless of @p size on success (@p buf contents are
 670  *   truncated to @p size if not large enough), a negative errno value
 671  *   otherwise and rte_errno is set.
 672  */
 673 static int
 674 rte_flow_conv_pattern(struct rte_flow_item *dst,
 675                       const size_t size,
 676                       const struct rte_flow_item *src,
 677                       unsigned int num,
 678                       struct rte_flow_error *error)
 679 {
 680         uintptr_t data = (uintptr_t)dst;
 681         size_t off;
 682         size_t ret;
 683         unsigned int i;
 684
 685         for (i = 0, off = 0; !num || i != num; ++i, ++src, ++dst) {
 686                 /**
 687                  * allow PMD private flow item
 688                  */
 689                 if (((int)src->type >= 0) &&
 690                         ((size_t)src->type >= RTE_DIM(rte_flow_desc_item) ||
 691                     !rte_flow_desc_item[src->type].name))
 692                         return rte_flow_error_set
 693                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, src,
 694                                  "cannot convert unknown item type");
 695                 if (size >= off + sizeof(*dst))
 696                         *dst = (struct rte_flow_item){
 697                                 .type = src->type,
 698                         };
 699                 off += sizeof(*dst);
 700                 if (!src->type)
 701                         num = i + 1;
 702         }
 703         num = i;
 704         src -= num;
 705         dst -= num;
 706         do {
 707                 if (src->spec) {
 708                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 709                         ret = rte_flow_conv_item_spec
 710                                 ((void *)(data + off),
 711                                  size > off ? size - off : 0, src,
 712                                  RTE_FLOW_CONV_ITEM_SPEC);
 713                         if (size && size >= off + ret)
 714                                 dst->spec = (void *)(data + off);
 715                         off += ret;
 716
 717                 }
 718                 if (src->last) {
 719                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 720                         ret = rte_flow_conv_item_spec
 721                                 ((void *)(data + off),
 722                                  size > off ? size - off : 0, src,
 723                                  RTE_FLOW_CONV_ITEM_LAST);
 724                         if (size && size >= off + ret)
 725                                 dst->last = (void *)(data + off);
 726                         off += ret;
 727                 }
 728                 if (src->mask) {
 729                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 730                         ret = rte_flow_conv_item_spec
 731                                 ((void *)(data + off),
 732                                  size > off ? size - off : 0, src,
 733                                  RTE_FLOW_CONV_ITEM_MASK);
 734                         if (size && size >= off + ret)
 735                                 dst->mask = (void *)(data + off);
 736                         off += ret;
 737                 }
 738                 ++src;
 739                 ++dst;
 740         } while (--num);
 741         return off;
 742 }
 743
 744 /**
 745  * Copy a list of actions.
 746  *
 747  * @param[out] dst
 748  *   Destination buffer. Can be NULL if @p size is zero.
 749  * @param size
 750  *   Size of @p dst in bytes.
 751  * @param[in] src
 752  *   Source actions.
 753  * @param num
 754  *   Maximum number of actions to process from @p src or 0 to process the
 755  *   entire list. In both cases, processing stops after
 756  *   RTE_FLOW_ACTION_TYPE_END is encountered.
 757  * @param[out] error
 758  *   Perform verbose error reporting if not NULL.
 759  *
 760  * @return
 761  *   A positive value representing the number of bytes needed to store
 762  *   actions regardless of @p size on success (@p buf contents are truncated
 763  *   to @p size if not large enough), a negative errno value otherwise and
 764  *   rte_errno is set.
 765  */
 766 static int
 767 rte_flow_conv_actions(struct rte_flow_action *dst,
 768                       const size_t size,
 769                       const struct rte_flow_action *src,
 770                       unsigned int num,
 771                       struct rte_flow_error *error)
 772 {
 773         uintptr_t data = (uintptr_t)dst;
 774         size_t off;
 775         size_t ret;
 776         unsigned int i;
 777
 778         for (i = 0, off = 0; !num || i != num; ++i, ++src, ++dst) {
 779                 /**
 780                  * allow PMD private flow action
 781                  */
 782                 if (((int)src->type >= 0) &&
 783                     ((size_t)src->type >= RTE_DIM(rte_flow_desc_action) ||
 784                     !rte_flow_desc_action[src->type].name))
 785                         return rte_flow_error_set
 786                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 787                                  src, "cannot convert unknown action type");
 788                 if (size >= off + sizeof(*dst))
 789                         *dst = (struct rte_flow_action){
 790                                 .type = src->type,
 791                         };
 792                 off += sizeof(*dst);
 793                 if (!src->type)
 794                         num = i + 1;
 795         }
 796         num = i;
 797         src -= num;
 798         dst -= num;
 799         do {
 800                 if (src->conf) {
 801                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 802                         ret = rte_flow_conv_action_conf
 803                                 ((void *)(data + off),
 804                                  size > off ? size - off : 0, src);
 805                         if (size && size >= off + ret)
 806                                 dst->conf = (void *)(data + off);
 807                         off += ret;
 808                 }
 809                 ++src;
 810                 ++dst;
 811         } while (--num);
 812         return off;
 813 }
 814
 815 /**
 816  * Copy flow rule components.
 817  *
 818  * This comprises the flow rule descriptor itself, attributes, pattern and
 819  * actions list. NULL components in @p src are skipped.
 820  *
 821  * @param[out] dst
 822  *   Destination buffer. Can be NULL if @p size is zero.
 823  * @param size
 824  *   Size of @p dst in bytes.
 825  * @param[in] src
 826  *   Source flow rule descriptor.
 827  * @param[out] error
 828  *   Perform verbose error reporting if not NULL.
 829  *
 830  * @return
 831  *   A positive value representing the number of bytes needed to store all
 832  *   components including the descriptor regardless of @p size on success
 833  *   (@p buf contents are truncated to @p size if not large enough), a
 834  *   negative errno value otherwise and rte_errno is set.
 835  */
 836 static int
 837 rte_flow_conv_rule(struct rte_flow_conv_rule *dst,
 838                    const size_t size,
 839                    const struct rte_flow_conv_rule *src,
 840                    struct rte_flow_error *error)
 841 {
 842         size_t off;
 843         int ret;
 844
 845         rte_memcpy(dst,
 846                    (&(struct rte_flow_conv_rule){
 847                         .attr = NULL,
 848                         .pattern = NULL,
 849                         .actions = NULL,
 850                    }),
 851                    size > sizeof(*dst) ? sizeof(*dst) : size);
 852         off = sizeof(*dst);
 853         if (src->attr_ro) {
 854                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 855                 if (size && size >= off + sizeof(*dst->attr))
 856                         dst->attr = rte_memcpy
 857                                 ((void *)((uintptr_t)dst + off),
 858                                  src->attr_ro, sizeof(*dst->attr));
 859                 off += sizeof(*dst->attr);
 860         }
 861         if (src->pattern_ro) {
 862                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 863                 ret = rte_flow_conv_pattern((void *)((uintptr_t)dst + off),
 864                                             size > off ? size - off : 0,
 865                                             src->pattern_ro, 0, error);
 866                 if (ret < 0)
 867                         return ret;
 868                 if (size && size >= off + (size_t)ret)
 869                         dst->pattern = (void *)((uintptr_t)dst + off);
 870                 off += ret;
 871         }
 872         if (src->actions_ro) {
 873                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 874                 ret = rte_flow_conv_actions((void *)((uintptr_t)dst + off),
 875                                             size > off ? size - off : 0,
 876                                             src->actions_ro, 0, error);
 877                 if (ret < 0)
 878                         return ret;
 879                 if (size >= off + (size_t)ret)
 880                         dst->actions = (void *)((uintptr_t)dst + off);
 881                 off += ret;
 882         }
 883         return off;
 884 }
 885
 886 /**
 887  * Retrieve the name of a pattern item/action type.
 888  *
 889  * @param is_action
 890  *   Nonzero when @p src represents an action type instead of a pattern item
 891  *   type.
 892  * @param is_ptr
 893  *   Nonzero to write string address instead of contents into @p dst.
 894  * @param[out] dst
 895  *   Destination buffer. Can be NULL if @p size is zero.
 896  * @param size
 897  *   Size of @p dst in bytes.
 898  * @param[in] src
 899  *   Depending on @p is_action, source pattern item or action type cast as a
 900  *   pointer.
 901  * @param[out] error
 902  *   Perform verbose error reporting if not NULL.
 903  *
 904  * @return
 905  *   A positive value representing the number of bytes needed to store the
 906  *   name or its address regardless of @p size on success (@p buf contents
 907  *   are truncated to @p size if not large enough), a negative errno value
 908  *   otherwise and rte_errno is set.
 909  */
 910 static int
 911 rte_flow_conv_name(int is_action,
 912                    int is_ptr,
 913                    char *dst,
 914                    const size_t size,
 915                    const void *src,
 916                    struct rte_flow_error *error)
 917 {
 918         struct desc_info {
 919                 const struct rte_flow_desc_data *data;
 920                 size_t num;
 921         };
 922         static const struct desc_info info_rep[2] = {
 923                 { rte_flow_desc_item, RTE_DIM(rte_flow_desc_item), },
 924                 { rte_flow_desc_action, RTE_DIM(rte_flow_desc_action), },
 925         };
 926         const struct desc_info *const info = &info_rep[!!is_action];
 927         unsigned int type = (uintptr_t)src;
 928
 929         if (type >= info->num)
 930                 return rte_flow_error_set
 931                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 932                          "unknown object type to retrieve the name of");
 933         if (!is_ptr)
 934                 return strlcpy(dst, info->data[type].name, size);
 935         if (size >= sizeof(const char **))
 936                 *((const char **)dst) = info->data[type].name;
 937         return sizeof(const char **);
 938 }
 939
 940 /** Helper function to convert flow API objects. */
 941 int
 942 rte_flow_conv(enum rte_flow_conv_op op,
 943               void *dst,
 944               size_t size,
 945               const void *src,
 946               struct rte_flow_error *error)
 947 {
 948         switch (op) {
 949                 const struct rte_flow_attr *attr;
 950
 951         case RTE_FLOW_CONV_OP_NONE:
 952                 return 0;
 953         case RTE_FLOW_CONV_OP_ATTR:
 954                 attr = src;
 955                 if (size > sizeof(*attr))
 956                         size = sizeof(*attr);
 957                 rte_memcpy(dst, attr, size);
 958                 return sizeof(*attr);
 959         case RTE_FLOW_CONV_OP_ITEM:
 960                 return rte_flow_conv_pattern(dst, size, src, 1, error);
 961         case RTE_FLOW_CONV_OP_ACTION:
 962                 return rte_flow_conv_actions(dst, size, src, 1, error);
 963         case RTE_FLOW_CONV_OP_PATTERN:
 964                 return rte_flow_conv_pattern(dst, size, src, 0, error);
 965         case RTE_FLOW_CONV_OP_ACTIONS:
 966                 return rte_flow_conv_actions(dst, size, src, 0, error);
 967         case RTE_FLOW_CONV_OP_RULE:
 968                 return rte_flow_conv_rule(dst, size, src, error);
 969         case RTE_FLOW_CONV_OP_ITEM_NAME:
 970                 return rte_flow_conv_name(0, 0, dst, size, src, error);
 971         case RTE_FLOW_CONV_OP_ACTION_NAME:
 972                 return rte_flow_conv_name(1, 0, dst, size, src, error);
 973         case RTE_FLOW_CONV_OP_ITEM_NAME_PTR:
 974                 return rte_flow_conv_name(0, 1, dst, size, src, error);
 975         case RTE_FLOW_CONV_OP_ACTION_NAME_PTR:
 976                 return rte_flow_conv_name(1, 1, dst, size, src, error);
 977         }
 978         return rte_flow_error_set
 979                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 980                  "unknown object conversion operation");
 981 }
 982
 983 /** Store a full rte_flow description. */
 984 size_t
 985 rte_flow_copy(struct rte_flow_desc *desc, size_t len,
 986               const struct rte_flow_attr *attr,
 987               const struct rte_flow_item *items,
 988               const struct rte_flow_action *actions)
 989 {
 990         /*
 991          * Overlap struct rte_flow_conv with struct rte_flow_desc in order
 992          * to convert the former to the latter without wasting space.
 993          */
 994         struct rte_flow_conv_rule *dst =
 995                 len ?
 996                 (void *)((uintptr_t)desc +
 997                          (offsetof(struct rte_flow_desc, actions) -
 998                           offsetof(struct rte_flow_conv_rule, actions))) :
 999                 NULL;
1000         size_t dst_size =
1001                 len > sizeof(*desc) - sizeof(*dst) ?
1002                 len - (sizeof(*desc) - sizeof(*dst)) :
1003                 0;
1004         struct rte_flow_conv_rule src = {
1005                 .attr_ro = NULL,
1006                 .pattern_ro = items,
1007                 .actions_ro = actions,
1008         };
1009         int ret;
1010
1011         RTE_BUILD_BUG_ON(sizeof(struct rte_flow_desc) <
1012                          sizeof(struct rte_flow_conv_rule));
1013         if (dst_size &&
1014             (&dst->pattern != &desc->items ||
1015              &dst->actions != &desc->actions ||
1016              (uintptr_t)(dst + 1) != (uintptr_t)(desc + 1))) {
1017                 rte_errno = EINVAL;
1018                 return 0;
1019         }
1020         ret = rte_flow_conv(RTE_FLOW_CONV_OP_RULE, dst, dst_size, &src, NULL);
1021         if (ret < 0)
1022                 return 0;
1023         ret += sizeof(*desc) - sizeof(*dst);
1024         rte_memcpy(desc,
1025                    (&(struct rte_flow_desc){
1026                         .size = ret,
1027                         .attr = *attr,
1028                         .items = dst_size ? dst->pattern : NULL,
1029                         .actions = dst_size ? dst->actions : NULL,
1030                    }),
1031                    len > sizeof(*desc) ? sizeof(*desc) : len);
1032         return ret;
1033 }
1034
1035 int
1036 rte_flow_dev_dump(uint16_t port_id, struct rte_flow *flow,
1037                         FILE *file, struct rte_flow_error *error)
1038 {
1039         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1040         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1041         int ret;
1042
1043         if (unlikely(!ops))
1044                 return -rte_errno;
1045         if (likely(!!ops->dev_dump)) {
1046                 fts_enter(dev);
1047                 ret = ops->dev_dump(dev, flow, file, error);
1048                 fts_exit(dev);
1049                 return flow_err(port_id, ret, error);
1050         }
1051         return rte_flow_error_set(error, ENOSYS,
1052                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1053                                   NULL, rte_strerror(ENOSYS));
1054 }
1055
1056 int
1057 rte_flow_get_aged_flows(uint16_t port_id, void **contexts,
1058                     uint32_t nb_contexts, struct rte_flow_error *error)
1059 {
1060         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1061         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1062         int ret;
1063
1064         if (unlikely(!ops))
1065                 return -rte_errno;
1066         if (likely(!!ops->get_aged_flows)) {
1067                 fts_enter(dev);
1068                 ret = ops->get_aged_flows(dev, contexts, nb_contexts, error);
1069                 fts_exit(dev);
1070                 return flow_err(port_id, ret, error);
1071         }
1072         return rte_flow_error_set(error, ENOTSUP,
1073                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1074                                   NULL, rte_strerror(ENOTSUP));
1075 }
1076
1077 struct rte_flow_action_handle *
1078 rte_flow_action_handle_create(uint16_t port_id,
1079                               const struct rte_flow_indir_action_conf *conf,
1080                               const struct rte_flow_action *action,
1081                               struct rte_flow_error *error)
1082 {
1083         struct rte_flow_action_handle *handle;
1084         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1085
1086         if (unlikely(!ops))
1087                 return NULL;
1088         if (unlikely(!ops->action_handle_create)) {
1089                 rte_flow_error_set(error, ENOSYS,
1090                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1091                                    rte_strerror(ENOSYS));
1092                 return NULL;
1093         }
1094         handle = ops->action_handle_create(&rte_eth_devices[port_id],
1095                                            conf, action, error);
1096         if (handle == NULL)
1097                 flow_err(port_id, -rte_errno, error);
1098         return handle;
1099 }
1100
1101 int
1102 rte_flow_action_handle_destroy(uint16_t port_id,
1103                                struct rte_flow_action_handle *handle,
1104                                struct rte_flow_error *error)
1105 {
1106         int ret;
1107         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1108
1109         if (unlikely(!ops))
1110                 return -rte_errno;
1111         if (unlikely(!ops->action_handle_destroy))
1112                 return rte_flow_error_set(error, ENOSYS,
1113                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1114                                           NULL, rte_strerror(ENOSYS));
1115         ret = ops->action_handle_destroy(&rte_eth_devices[port_id],
1116                                          handle, error);
1117         return flow_err(port_id, ret, error);
1118 }
1119
1120 int
1121 rte_flow_action_handle_update(uint16_t port_id,
1122                               struct rte_flow_action_handle *handle,
1123                               const void *update,
1124                               struct rte_flow_error *error)
1125 {
1126         int ret;
1127         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1128
1129         if (unlikely(!ops))
1130                 return -rte_errno;
1131         if (unlikely(!ops->action_handle_update))
1132                 return rte_flow_error_set(error, ENOSYS,
1133                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1134                                           NULL, rte_strerror(ENOSYS));
1135         ret = ops->action_handle_update(&rte_eth_devices[port_id], handle,
1136                                         update, error);
1137         return flow_err(port_id, ret, error);
1138 }
1139
1140 int
1141 rte_flow_action_handle_query(uint16_t port_id,
1142                              const struct rte_flow_action_handle *handle,
1143                              void *data,
1144                              struct rte_flow_error *error)
1145 {
1146         int ret;
1147         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1148
1149         if (unlikely(!ops))
1150                 return -rte_errno;
1151         if (unlikely(!ops->action_handle_query))
1152                 return rte_flow_error_set(error, ENOSYS,
1153                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1154                                           NULL, rte_strerror(ENOSYS));
1155         ret = ops->action_handle_query(&rte_eth_devices[port_id], handle,
1156                                        data, error);
1157         return flow_err(port_id, ret, error);
1158 }
1159
1160 int
1161 rte_flow_tunnel_decap_set(uint16_t port_id,
1162                           struct rte_flow_tunnel *tunnel,
1163                           struct rte_flow_action **actions,
1164                           uint32_t *num_of_actions,
1165                           struct rte_flow_error *error)
1166 {
1167         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1168         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1169
1170         if (unlikely(!ops))
1171                 return -rte_errno;
1172         if (likely(!!ops->tunnel_decap_set)) {
1173                 return flow_err(port_id,
1174                                 ops->tunnel_decap_set(dev, tunnel, actions,
1175                                                       num_of_actions, error),
1176                                 error);
1177         }
1178         return rte_flow_error_set(error, ENOTSUP,
1179                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1180                                   NULL, rte_strerror(ENOTSUP));
1181 }
1182
1183 int
1184 rte_flow_tunnel_match(uint16_t port_id,
1185                       struct rte_flow_tunnel *tunnel,
1186                       struct rte_flow_item **items,
1187                       uint32_t *num_of_items,
1188                       struct rte_flow_error *error)
1189 {
1190         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1191         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1192
1193         if (unlikely(!ops))
1194                 return -rte_errno;
1195         if (likely(!!ops->tunnel_match)) {
1196                 return flow_err(port_id,
1197                                 ops->tunnel_match(dev, tunnel, items,
1198                                                   num_of_items, error),
1199                                 error);
1200         }
1201         return rte_flow_error_set(error, ENOTSUP,
1202                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1203                                   NULL, rte_strerror(ENOTSUP));
1204 }
1205
1206 int
1207 rte_flow_get_restore_info(uint16_t port_id,
1208                           struct rte_mbuf *m,
1209                           struct rte_flow_restore_info *restore_info,
1210                           struct rte_flow_error *error)
1211 {
1212         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1213         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1214
1215         if (unlikely(!ops))
1216                 return -rte_errno;
1217         if (likely(!!ops->get_restore_info)) {
1218                 return flow_err(port_id,
1219                                 ops->get_restore_info(dev, m, restore_info,
1220                                                       error),
1221                                 error);
1222         }
1223         return rte_flow_error_set(error, ENOTSUP,
1224                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1225                                   NULL, rte_strerror(ENOTSUP));
1226 }
1227
1228 int
1229 rte_flow_tunnel_action_decap_release(uint16_t port_id,
1230                                      struct rte_flow_action *actions,
1231                                      uint32_t num_of_actions,
1232                                      struct rte_flow_error *error)
1233 {
1234         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1235         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1236
1237         if (unlikely(!ops))
1238                 return -rte_errno;
1239         if (likely(!!ops->tunnel_action_decap_release)) {
1240                 return flow_err(port_id,
1241                                 ops->tunnel_action_decap_release(dev, actions,
1242                                                                  num_of_actions,
1243                                                                  error),
1244                                 error);
1245         }
1246         return rte_flow_error_set(error, ENOTSUP,
1247                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1248                                   NULL, rte_strerror(ENOTSUP));
1249 }
1250
1251 int
1252 rte_flow_tunnel_item_release(uint16_t port_id,
1253                              struct rte_flow_item *items,
1254                              uint32_t num_of_items,
1255                              struct rte_flow_error *error)
1256 {
1257         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1258         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1259
1260         if (unlikely(!ops))
1261                 return -rte_errno;
1262         if (likely(!!ops->tunnel_item_release)) {
1263                 return flow_err(port_id,
1264                                 ops->tunnel_item_release(dev, items,
1265                                                          num_of_items, error),
1266                                 error);
1267         }
1268         return rte_flow_error_set(error, ENOTSUP,
1269                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1270                                   NULL, rte_strerror(ENOTSUP));
1271 }