lib/ethdev/rte_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stddef.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_errno.h>
  13 #include <rte_branch_prediction.h>
  14 #include <rte_string_fns.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_mbuf_dyn.h>
  17 #include "rte_ethdev.h"
  18 #include "rte_flow_driver.h"
  19 #include "rte_flow.h"
  20
  21 /* Mbuf dynamic field name for metadata. */
  22 int32_t rte_flow_dynf_metadata_offs = -1;
  23
  24 /* Mbuf dynamic field flag bit number for metadata. */
  25 uint64_t rte_flow_dynf_metadata_mask;
  26
  27 /**
  28  * Flow elements description tables.
  29  */
  30 struct rte_flow_desc_data {
  31         const char *name;
  32         size_t size;
  33 };
  34
  35 /** Generate flow_item[] entry. */
  36 #define MK_FLOW_ITEM(t, s) \
  37         [RTE_FLOW_ITEM_TYPE_ ## t] = { \
  38                 .name = # t, \
  39                 .size = s, \
  40         }
  41
  42 /** Information about known flow pattern items. */
  43 static const struct rte_flow_desc_data rte_flow_desc_item[] = {
  44         MK_FLOW_ITEM(END, 0),
  45         MK_FLOW_ITEM(VOID, 0),
  46         MK_FLOW_ITEM(INVERT, 0),
  47         MK_FLOW_ITEM(ANY, sizeof(struct rte_flow_item_any)),
  48         MK_FLOW_ITEM(PF, 0),
  49         MK_FLOW_ITEM(VF, sizeof(struct rte_flow_item_vf)),
  50         MK_FLOW_ITEM(PHY_PORT, sizeof(struct rte_flow_item_phy_port)),
  51         MK_FLOW_ITEM(PORT_ID, sizeof(struct rte_flow_item_port_id)),
  52         MK_FLOW_ITEM(RAW, sizeof(struct rte_flow_item_raw)),
  53         MK_FLOW_ITEM(ETH, sizeof(struct rte_flow_item_eth)),
  54         MK_FLOW_ITEM(VLAN, sizeof(struct rte_flow_item_vlan)),
  55         MK_FLOW_ITEM(IPV4, sizeof(struct rte_flow_item_ipv4)),
  56         MK_FLOW_ITEM(IPV6, sizeof(struct rte_flow_item_ipv6)),
  57         MK_FLOW_ITEM(ICMP, sizeof(struct rte_flow_item_icmp)),
  58         MK_FLOW_ITEM(UDP, sizeof(struct rte_flow_item_udp)),
  59         MK_FLOW_ITEM(TCP, sizeof(struct rte_flow_item_tcp)),
  60         MK_FLOW_ITEM(SCTP, sizeof(struct rte_flow_item_sctp)),
  61         MK_FLOW_ITEM(VXLAN, sizeof(struct rte_flow_item_vxlan)),
  62         MK_FLOW_ITEM(E_TAG, sizeof(struct rte_flow_item_e_tag)),
  63         MK_FLOW_ITEM(NVGRE, sizeof(struct rte_flow_item_nvgre)),
  64         MK_FLOW_ITEM(MPLS, sizeof(struct rte_flow_item_mpls)),
  65         MK_FLOW_ITEM(GRE, sizeof(struct rte_flow_item_gre)),
  66         MK_FLOW_ITEM(FUZZY, sizeof(struct rte_flow_item_fuzzy)),
  67         MK_FLOW_ITEM(GTP, sizeof(struct rte_flow_item_gtp)),
  68         MK_FLOW_ITEM(GTPC, sizeof(struct rte_flow_item_gtp)),
  69         MK_FLOW_ITEM(GTPU, sizeof(struct rte_flow_item_gtp)),
  70         MK_FLOW_ITEM(ESP, sizeof(struct rte_flow_item_esp)),
  71         MK_FLOW_ITEM(GENEVE, sizeof(struct rte_flow_item_geneve)),
  72         MK_FLOW_ITEM(VXLAN_GPE, sizeof(struct rte_flow_item_vxlan_gpe)),
  73         MK_FLOW_ITEM(ARP_ETH_IPV4, sizeof(struct rte_flow_item_arp_eth_ipv4)),
  74         MK_FLOW_ITEM(IPV6_EXT, sizeof(struct rte_flow_item_ipv6_ext)),
  75         MK_FLOW_ITEM(IPV6_FRAG_EXT, sizeof(struct rte_flow_item_ipv6_frag_ext)),
  76         MK_FLOW_ITEM(ICMP6, sizeof(struct rte_flow_item_icmp6)),
  77         MK_FLOW_ITEM(ICMP6_ND_NS, sizeof(struct rte_flow_item_icmp6_nd_ns)),
  78         MK_FLOW_ITEM(ICMP6_ND_NA, sizeof(struct rte_flow_item_icmp6_nd_na)),
  79         MK_FLOW_ITEM(ICMP6_ND_OPT, sizeof(struct rte_flow_item_icmp6_nd_opt)),
  80         MK_FLOW_ITEM(ICMP6_ND_OPT_SLA_ETH,
  81                      sizeof(struct rte_flow_item_icmp6_nd_opt_sla_eth)),
  82         MK_FLOW_ITEM(ICMP6_ND_OPT_TLA_ETH,
  83                      sizeof(struct rte_flow_item_icmp6_nd_opt_tla_eth)),
  84         MK_FLOW_ITEM(MARK, sizeof(struct rte_flow_item_mark)),
  85         MK_FLOW_ITEM(META, sizeof(struct rte_flow_item_meta)),
  86         MK_FLOW_ITEM(TAG, sizeof(struct rte_flow_item_tag)),
  87         MK_FLOW_ITEM(GRE_KEY, sizeof(rte_be32_t)),
  88         MK_FLOW_ITEM(GTP_PSC, sizeof(struct rte_flow_item_gtp_psc)),
  89         MK_FLOW_ITEM(PPPOES, sizeof(struct rte_flow_item_pppoe)),
  90         MK_FLOW_ITEM(PPPOED, sizeof(struct rte_flow_item_pppoe)),
  91         MK_FLOW_ITEM(PPPOE_PROTO_ID,
  92                         sizeof(struct rte_flow_item_pppoe_proto_id)),
  93         MK_FLOW_ITEM(NSH, sizeof(struct rte_flow_item_nsh)),
  94         MK_FLOW_ITEM(IGMP, sizeof(struct rte_flow_item_igmp)),
  95         MK_FLOW_ITEM(AH, sizeof(struct rte_flow_item_ah)),
  96         MK_FLOW_ITEM(HIGIG2, sizeof(struct rte_flow_item_higig2_hdr)),
  97         MK_FLOW_ITEM(L2TPV3OIP, sizeof(struct rte_flow_item_l2tpv3oip)),
  98         MK_FLOW_ITEM(PFCP, sizeof(struct rte_flow_item_pfcp)),
  99         MK_FLOW_ITEM(ECPRI, sizeof(struct rte_flow_item_ecpri)),
 100         MK_FLOW_ITEM(GENEVE_OPT, sizeof(struct rte_flow_item_geneve_opt)),
 101         MK_FLOW_ITEM(INTEGRITY, sizeof(struct rte_flow_item_integrity)),
 102         MK_FLOW_ITEM(CONNTRACK, sizeof(uint32_t)),
 103         MK_FLOW_ITEM(PORT_REPRESENTOR, sizeof(struct rte_flow_item_ethdev)),
 104         MK_FLOW_ITEM(REPRESENTED_PORT, sizeof(struct rte_flow_item_ethdev)),
 105 };
 106
 107 /** Generate flow_action[] entry. */
 108 #define MK_FLOW_ACTION(t, s) \
 109         [RTE_FLOW_ACTION_TYPE_ ## t] = { \
 110                 .name = # t, \
 111                 .size = s, \
 112         }
 113
 114 /** Information about known flow actions. */
 115 static const struct rte_flow_desc_data rte_flow_desc_action[] = {
 116         MK_FLOW_ACTION(END, 0),
 117         MK_FLOW_ACTION(VOID, 0),
 118         MK_FLOW_ACTION(PASSTHRU, 0),
 119         MK_FLOW_ACTION(JUMP, sizeof(struct rte_flow_action_jump)),
 120         MK_FLOW_ACTION(MARK, sizeof(struct rte_flow_action_mark)),
 121         MK_FLOW_ACTION(FLAG, 0),
 122         MK_FLOW_ACTION(QUEUE, sizeof(struct rte_flow_action_queue)),
 123         MK_FLOW_ACTION(DROP, 0),
 124         MK_FLOW_ACTION(COUNT, sizeof(struct rte_flow_action_count)),
 125         MK_FLOW_ACTION(RSS, sizeof(struct rte_flow_action_rss)),
 126         MK_FLOW_ACTION(PF, 0),
 127         MK_FLOW_ACTION(VF, sizeof(struct rte_flow_action_vf)),
 128         MK_FLOW_ACTION(PHY_PORT, sizeof(struct rte_flow_action_phy_port)),
 129         MK_FLOW_ACTION(PORT_ID, sizeof(struct rte_flow_action_port_id)),
 130         MK_FLOW_ACTION(METER, sizeof(struct rte_flow_action_meter)),
 131         MK_FLOW_ACTION(SECURITY, sizeof(struct rte_flow_action_security)),
 132         MK_FLOW_ACTION(OF_SET_MPLS_TTL,
 133                        sizeof(struct rte_flow_action_of_set_mpls_ttl)),
 134         MK_FLOW_ACTION(OF_DEC_MPLS_TTL, 0),
 135         MK_FLOW_ACTION(OF_SET_NW_TTL,
 136                        sizeof(struct rte_flow_action_of_set_nw_ttl)),
 137         MK_FLOW_ACTION(OF_DEC_NW_TTL, 0),
 138         MK_FLOW_ACTION(OF_COPY_TTL_OUT, 0),
 139         MK_FLOW_ACTION(OF_COPY_TTL_IN, 0),
 140         MK_FLOW_ACTION(OF_POP_VLAN, 0),
 141         MK_FLOW_ACTION(OF_PUSH_VLAN,
 142                        sizeof(struct rte_flow_action_of_push_vlan)),
 143         MK_FLOW_ACTION(OF_SET_VLAN_VID,
 144                        sizeof(struct rte_flow_action_of_set_vlan_vid)),
 145         MK_FLOW_ACTION(OF_SET_VLAN_PCP,
 146                        sizeof(struct rte_flow_action_of_set_vlan_pcp)),
 147         MK_FLOW_ACTION(OF_POP_MPLS,
 148                        sizeof(struct rte_flow_action_of_pop_mpls)),
 149         MK_FLOW_ACTION(OF_PUSH_MPLS,
 150                        sizeof(struct rte_flow_action_of_push_mpls)),
 151         MK_FLOW_ACTION(VXLAN_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)),
 152         MK_FLOW_ACTION(VXLAN_DECAP, 0),
 153         MK_FLOW_ACTION(NVGRE_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)),
 154         MK_FLOW_ACTION(NVGRE_DECAP, 0),
 155         MK_FLOW_ACTION(RAW_ENCAP, sizeof(struct rte_flow_action_raw_encap)),
 156         MK_FLOW_ACTION(RAW_DECAP, sizeof(struct rte_flow_action_raw_decap)),
 157         MK_FLOW_ACTION(SET_IPV4_SRC,
 158                        sizeof(struct rte_flow_action_set_ipv4)),
 159         MK_FLOW_ACTION(SET_IPV4_DST,
 160                        sizeof(struct rte_flow_action_set_ipv4)),
 161         MK_FLOW_ACTION(SET_IPV6_SRC,
 162                        sizeof(struct rte_flow_action_set_ipv6)),
 163         MK_FLOW_ACTION(SET_IPV6_DST,
 164                        sizeof(struct rte_flow_action_set_ipv6)),
 165         MK_FLOW_ACTION(SET_TP_SRC,
 166                        sizeof(struct rte_flow_action_set_tp)),
 167         MK_FLOW_ACTION(SET_TP_DST,
 168                        sizeof(struct rte_flow_action_set_tp)),
 169         MK_FLOW_ACTION(MAC_SWAP, 0),
 170         MK_FLOW_ACTION(DEC_TTL, 0),
 171         MK_FLOW_ACTION(SET_TTL, sizeof(struct rte_flow_action_set_ttl)),
 172         MK_FLOW_ACTION(SET_MAC_SRC, sizeof(struct rte_flow_action_set_mac)),
 173         MK_FLOW_ACTION(SET_MAC_DST, sizeof(struct rte_flow_action_set_mac)),
 174         MK_FLOW_ACTION(INC_TCP_SEQ, sizeof(rte_be32_t)),
 175         MK_FLOW_ACTION(DEC_TCP_SEQ, sizeof(rte_be32_t)),
 176         MK_FLOW_ACTION(INC_TCP_ACK, sizeof(rte_be32_t)),
 177         MK_FLOW_ACTION(DEC_TCP_ACK, sizeof(rte_be32_t)),
 178         MK_FLOW_ACTION(SET_TAG, sizeof(struct rte_flow_action_set_tag)),
 179         MK_FLOW_ACTION(SET_META, sizeof(struct rte_flow_action_set_meta)),
 180         MK_FLOW_ACTION(SET_IPV4_DSCP, sizeof(struct rte_flow_action_set_dscp)),
 181         MK_FLOW_ACTION(SET_IPV6_DSCP, sizeof(struct rte_flow_action_set_dscp)),
 182         MK_FLOW_ACTION(AGE, sizeof(struct rte_flow_action_age)),
 183         MK_FLOW_ACTION(SAMPLE, sizeof(struct rte_flow_action_sample)),
 184         MK_FLOW_ACTION(MODIFY_FIELD,
 185                        sizeof(struct rte_flow_action_modify_field)),
 186         /**
 187          * Indirect action represented as handle of type
 188          * (struct rte_flow_action_handle *) stored in conf field (see
 189          * struct rte_flow_action); no need for additional structure to * store
 190          * indirect action handle.
 191          */
 192         MK_FLOW_ACTION(INDIRECT, 0),
 193         MK_FLOW_ACTION(CONNTRACK, sizeof(struct rte_flow_action_conntrack)),
 194         MK_FLOW_ACTION(PORT_REPRESENTOR, sizeof(struct rte_flow_action_ethdev)),
 195         MK_FLOW_ACTION(REPRESENTED_PORT, sizeof(struct rte_flow_action_ethdev)),
 196 };
 197
 198 int
 199 rte_flow_dynf_metadata_register(void)
 200 {
 201         int offset;
 202         int flag;
 203
 204         static const struct rte_mbuf_dynfield desc_offs = {
 205                 .name = RTE_MBUF_DYNFIELD_METADATA_NAME,
 206                 .size = sizeof(uint32_t),
 207                 .align = __alignof__(uint32_t),
 208         };
 209         static const struct rte_mbuf_dynflag desc_flag = {
 210                 .name = RTE_MBUF_DYNFLAG_METADATA_NAME,
 211         };
 212
 213         offset = rte_mbuf_dynfield_register(&desc_offs);
 214         if (offset < 0)
 215                 goto error;
 216         flag = rte_mbuf_dynflag_register(&desc_flag);
 217         if (flag < 0)
 218                 goto error;
 219         rte_flow_dynf_metadata_offs = offset;
 220         rte_flow_dynf_metadata_mask = (1ULL << flag);
 221         return 0;
 222
 223 error:
 224         rte_flow_dynf_metadata_offs = -1;
 225         rte_flow_dynf_metadata_mask = 0ULL;
 226         return -rte_errno;
 227 }
 228
 229 static inline void
 230 fts_enter(struct rte_eth_dev *dev)
 231 {
 232         if (!(dev->data->dev_flags & RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE))
 233                 pthread_mutex_lock(&dev->data->flow_ops_mutex);
 234 }
 235
 236 static inline void
 237 fts_exit(struct rte_eth_dev *dev)
 238 {
 239         if (!(dev->data->dev_flags & RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE))
 240                 pthread_mutex_unlock(&dev->data->flow_ops_mutex);
 241 }
 242
 243 static int
 244 flow_err(uint16_t port_id, int ret, struct rte_flow_error *error)
 245 {
 246         if (ret == 0)
 247                 return 0;
 248         if (rte_eth_dev_is_removed(port_id))
 249                 return rte_flow_error_set(error, EIO,
 250                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 251                                           NULL, rte_strerror(EIO));
 252         return ret;
 253 }
 254
 255 /* Get generic flow operations structure from a port. */
 256 const struct rte_flow_ops *
 257 rte_flow_ops_get(uint16_t port_id, struct rte_flow_error *error)
 258 {
 259         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 260         const struct rte_flow_ops *ops;
 261         int code;
 262
 263         if (unlikely(!rte_eth_dev_is_valid_port(port_id)))
 264                 code = ENODEV;
 265         else if (unlikely(dev->dev_ops->flow_ops_get == NULL))
 266                 /* flow API not supported with this driver dev_ops */
 267                 code = ENOSYS;
 268         else
 269                 code = dev->dev_ops->flow_ops_get(dev, &ops);
 270         if (code == 0 && ops == NULL)
 271                 /* flow API not supported with this device */
 272                 code = ENOSYS;
 273
 274         if (code != 0) {
 275                 rte_flow_error_set(error, code, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 276                                    NULL, rte_strerror(code));
 277                 return NULL;
 278         }
 279         return ops;
 280 }
 281
 282 /* Check whether a flow rule can be created on a given port. */
 283 int
 284 rte_flow_validate(uint16_t port_id,
 285                   const struct rte_flow_attr *attr,
 286                   const struct rte_flow_item pattern[],
 287                   const struct rte_flow_action actions[],
 288                   struct rte_flow_error *error)
 289 {
 290         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 291         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 292         int ret;
 293
 294         if (unlikely(!ops))
 295                 return -rte_errno;
 296         if (likely(!!ops->validate)) {
 297                 fts_enter(dev);
 298                 ret = ops->validate(dev, attr, pattern, actions, error);
 299                 fts_exit(dev);
 300                 return flow_err(port_id, ret, error);
 301         }
 302         return rte_flow_error_set(error, ENOSYS,
 303                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 304                                   NULL, rte_strerror(ENOSYS));
 305 }
 306
 307 /* Create a flow rule on a given port. */
 308 struct rte_flow *
 309 rte_flow_create(uint16_t port_id,
 310                 const struct rte_flow_attr *attr,
 311                 const struct rte_flow_item pattern[],
 312                 const struct rte_flow_action actions[],
 313                 struct rte_flow_error *error)
 314 {
 315         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 316         struct rte_flow *flow;
 317         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 318
 319         if (unlikely(!ops))
 320                 return NULL;
 321         if (likely(!!ops->create)) {
 322                 fts_enter(dev);
 323                 flow = ops->create(dev, attr, pattern, actions, error);
 324                 fts_exit(dev);
 325                 if (flow == NULL)
 326                         flow_err(port_id, -rte_errno, error);
 327                 return flow;
 328         }
 329         rte_flow_error_set(error, ENOSYS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 330                            NULL, rte_strerror(ENOSYS));
 331         return NULL;
 332 }
 333
 334 /* Destroy a flow rule on a given port. */
 335 int
 336 rte_flow_destroy(uint16_t port_id,
 337                  struct rte_flow *flow,
 338                  struct rte_flow_error *error)
 339 {
 340         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 341         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 342         int ret;
 343
 344         if (unlikely(!ops))
 345                 return -rte_errno;
 346         if (likely(!!ops->destroy)) {
 347                 fts_enter(dev);
 348                 ret = ops->destroy(dev, flow, error);
 349                 fts_exit(dev);
 350                 return flow_err(port_id, ret, error);
 351         }
 352         return rte_flow_error_set(error, ENOSYS,
 353                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 354                                   NULL, rte_strerror(ENOSYS));
 355 }
 356
 357 /* Destroy all flow rules associated with a port. */
 358 int
 359 rte_flow_flush(uint16_t port_id,
 360                struct rte_flow_error *error)
 361 {
 362         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 363         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 364         int ret;
 365
 366         if (unlikely(!ops))
 367                 return -rte_errno;
 368         if (likely(!!ops->flush)) {
 369                 fts_enter(dev);
 370                 ret = ops->flush(dev, error);
 371                 fts_exit(dev);
 372                 return flow_err(port_id, ret, error);
 373         }
 374         return rte_flow_error_set(error, ENOSYS,
 375                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 376                                   NULL, rte_strerror(ENOSYS));
 377 }
 378
 379 /* Query an existing flow rule. */
 380 int
 381 rte_flow_query(uint16_t port_id,
 382                struct rte_flow *flow,
 383                const struct rte_flow_action *action,
 384                void *data,
 385                struct rte_flow_error *error)
 386 {
 387         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 388         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 389         int ret;
 390
 391         if (!ops)
 392                 return -rte_errno;
 393         if (likely(!!ops->query)) {
 394                 fts_enter(dev);
 395                 ret = ops->query(dev, flow, action, data, error);
 396                 fts_exit(dev);
 397                 return flow_err(port_id, ret, error);
 398         }
 399         return rte_flow_error_set(error, ENOSYS,
 400                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 401                                   NULL, rte_strerror(ENOSYS));
 402 }
 403
 404 /* Restrict ingress traffic to the defined flow rules. */
 405 int
 406 rte_flow_isolate(uint16_t port_id,
 407                  int set,
 408                  struct rte_flow_error *error)
 409 {
 410         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 411         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 412         int ret;
 413
 414         if (!ops)
 415                 return -rte_errno;
 416         if (likely(!!ops->isolate)) {
 417                 fts_enter(dev);
 418                 ret = ops->isolate(dev, set, error);
 419                 fts_exit(dev);
 420                 return flow_err(port_id, ret, error);
 421         }
 422         return rte_flow_error_set(error, ENOSYS,
 423                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 424                                   NULL, rte_strerror(ENOSYS));
 425 }
 426
 427 /* Initialize flow error structure. */
 428 int
 429 rte_flow_error_set(struct rte_flow_error *error,
 430                    int code,
 431                    enum rte_flow_error_type type,
 432                    const void *cause,
 433                    const char *message)
 434 {
 435         if (error) {
 436                 *error = (struct rte_flow_error){
 437                         .type = type,
 438                         .cause = cause,
 439                         .message = message,
 440                 };
 441         }
 442         rte_errno = code;
 443         return -code;
 444 }
 445
 446 /** Pattern item specification types. */
 447 enum rte_flow_conv_item_spec_type {
 448         RTE_FLOW_CONV_ITEM_SPEC,
 449         RTE_FLOW_CONV_ITEM_LAST,
 450         RTE_FLOW_CONV_ITEM_MASK,
 451 };
 452
 453 /**
 454  * Copy pattern item specification.
 455  *
 456  * @param[out] buf
 457  *   Output buffer. Can be NULL if @p size is zero.
 458  * @param size
 459  *   Size of @p buf in bytes.
 460  * @param[in] item
 461  *   Pattern item to copy specification from.
 462  * @param type
 463  *   Specification selector for either @p spec, @p last or @p mask.
 464  *
 465  * @return
 466  *   Number of bytes needed to store pattern item specification regardless
 467  *   of @p size. @p buf contents are truncated to @p size if not large
 468  *   enough.
 469  */
 470 static size_t
 471 rte_flow_conv_item_spec(void *buf, const size_t size,
 472                         const struct rte_flow_item *item,
 473                         enum rte_flow_conv_item_spec_type type)
 474 {
 475         size_t off;
 476         const void *data =
 477                 type == RTE_FLOW_CONV_ITEM_SPEC ? item->spec :
 478                 type == RTE_FLOW_CONV_ITEM_LAST ? item->last :
 479                 type == RTE_FLOW_CONV_ITEM_MASK ? item->mask :
 480                 NULL;
 481
 482         switch (item->type) {
 483                 union {
 484                         const struct rte_flow_item_raw *raw;
 485                 } spec;
 486                 union {
 487                         const struct rte_flow_item_raw *raw;
 488                 } last;
 489                 union {
 490                         const struct rte_flow_item_raw *raw;
 491                 } mask;
 492                 union {
 493                         const struct rte_flow_item_raw *raw;
 494                 } src;
 495                 union {
 496                         struct rte_flow_item_raw *raw;
 497                 } dst;
 498                 size_t tmp;
 499
 500         case RTE_FLOW_ITEM_TYPE_RAW:
 501                 spec.raw = item->spec;
 502                 last.raw = item->last ? item->last : item->spec;
 503                 mask.raw = item->mask ? item->mask : &rte_flow_item_raw_mask;
 504                 src.raw = data;
 505                 dst.raw = buf;
 506                 rte_memcpy(dst.raw,
 507                            (&(struct rte_flow_item_raw){
 508                                 .relative = src.raw->relative,
 509                                 .search = src.raw->search,
 510                                 .reserved = src.raw->reserved,
 511                                 .offset = src.raw->offset,
 512                                 .limit = src.raw->limit,
 513                                 .length = src.raw->length,
 514                            }),
 515                            size > sizeof(*dst.raw) ? sizeof(*dst.raw) : size);
 516                 off = sizeof(*dst.raw);
 517                 if (type == RTE_FLOW_CONV_ITEM_SPEC ||
 518                     (type == RTE_FLOW_CONV_ITEM_MASK &&
 519                      ((spec.raw->length & mask.raw->length) >=
 520                       (last.raw->length & mask.raw->length))))
 521                         tmp = spec.raw->length & mask.raw->length;
 522                 else
 523                         tmp = last.raw->length & mask.raw->length;
 524                 if (tmp) {
 525                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.raw->pattern));
 526                         if (size >= off + tmp)
 527                                 dst.raw->pattern = rte_memcpy
 528                                         ((void *)((uintptr_t)dst.raw + off),
 529                                          src.raw->pattern, tmp);
 530                         off += tmp;
 531                 }
 532                 break;
 533         default:
 534                 /**
 535                  * allow PMD private flow item
 536                  */
 537                 off = (int)item->type >= 0 ?
 538                       rte_flow_desc_item[item->type].size : sizeof(void *);
 539                 rte_memcpy(buf, data, (size > off ? off : size));
 540                 break;
 541         }
 542         return off;
 543 }
 544
 545 /**
 546  * Copy action configuration.
 547  *
 548  * @param[out] buf
 549  *   Output buffer. Can be NULL if @p size is zero.
 550  * @param size
 551  *   Size of @p buf in bytes.
 552  * @param[in] action
 553  *   Action to copy configuration from.
 554  *
 555  * @return
 556  *   Number of bytes needed to store pattern item specification regardless
 557  *   of @p size. @p buf contents are truncated to @p size if not large
 558  *   enough.
 559  */
 560 static size_t
 561 rte_flow_conv_action_conf(void *buf, const size_t size,
 562                           const struct rte_flow_action *action)
 563 {
 564         size_t off;
 565
 566         switch (action->type) {
 567                 union {
 568                         const struct rte_flow_action_rss *rss;
 569                         const struct rte_flow_action_vxlan_encap *vxlan_encap;
 570                         const struct rte_flow_action_nvgre_encap *nvgre_encap;
 571                 } src;
 572                 union {
 573                         struct rte_flow_action_rss *rss;
 574                         struct rte_flow_action_vxlan_encap *vxlan_encap;
 575                         struct rte_flow_action_nvgre_encap *nvgre_encap;
 576                 } dst;
 577                 size_t tmp;
 578                 int ret;
 579
 580         case RTE_FLOW_ACTION_TYPE_RSS:
 581                 src.rss = action->conf;
 582                 dst.rss = buf;
 583                 rte_memcpy(dst.rss,
 584                            (&(struct rte_flow_action_rss){
 585                                 .func = src.rss->func,
 586                                 .level = src.rss->level,
 587                                 .types = src.rss->types,
 588                                 .key_len = src.rss->key_len,
 589                                 .queue_num = src.rss->queue_num,
 590                            }),
 591                            size > sizeof(*dst.rss) ? sizeof(*dst.rss) : size);
 592                 off = sizeof(*dst.rss);
 593                 if (src.rss->key_len && src.rss->key) {
 594                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->key));
 595                         tmp = sizeof(*src.rss->key) * src.rss->key_len;
 596                         if (size >= off + tmp)
 597                                 dst.rss->key = rte_memcpy
 598                                         ((void *)((uintptr_t)dst.rss + off),
 599                                          src.rss->key, tmp);
 600                         off += tmp;
 601                 }
 602                 if (src.rss->queue_num) {
 603                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->queue));
 604                         tmp = sizeof(*src.rss->queue) * src.rss->queue_num;
 605                         if (size >= off + tmp)
 606                                 dst.rss->queue = rte_memcpy
 607                                         ((void *)((uintptr_t)dst.rss + off),
 608                                          src.rss->queue, tmp);
 609                         off += tmp;
 610                 }
 611                 break;
 612         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
 613         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
 614                 src.vxlan_encap = action->conf;
 615                 dst.vxlan_encap = buf;
 616                 RTE_BUILD_BUG_ON(sizeof(*src.vxlan_encap) !=
 617                                  sizeof(*src.nvgre_encap) ||
 618                                  offsetof(struct rte_flow_action_vxlan_encap,
 619                                           definition) !=
 620                                  offsetof(struct rte_flow_action_nvgre_encap,
 621                                           definition));
 622                 off = sizeof(*dst.vxlan_encap);
 623                 if (src.vxlan_encap->definition) {
 624                         off = RTE_ALIGN_CEIL
 625                                 (off, sizeof(*dst.vxlan_encap->definition));
 626                         ret = rte_flow_conv
 627                                 (RTE_FLOW_CONV_OP_PATTERN,
 628                                  (void *)((uintptr_t)dst.vxlan_encap + off),
 629                                  size > off ? size - off : 0,
 630                                  src.vxlan_encap->definition, NULL);
 631                         if (ret < 0)
 632                                 return 0;
 633                         if (size >= off + ret)
 634                                 dst.vxlan_encap->definition =
 635                                         (void *)((uintptr_t)dst.vxlan_encap +
 636                                                  off);
 637                         off += ret;
 638                 }
 639                 break;
 640         default:
 641                 /**
 642                  * allow PMD private flow action
 643                  */
 644                 off = (int)action->type >= 0 ?
 645                       rte_flow_desc_action[action->type].size : sizeof(void *);
 646                 rte_memcpy(buf, action->conf, (size > off ? off : size));
 647                 break;
 648         }
 649         return off;
 650 }
 651
 652 /**
 653  * Copy a list of pattern items.
 654  *
 655  * @param[out] dst
 656  *   Destination buffer. Can be NULL if @p size is zero.
 657  * @param size
 658  *   Size of @p dst in bytes.
 659  * @param[in] src
 660  *   Source pattern items.
 661  * @param num
 662  *   Maximum number of pattern items to process from @p src or 0 to process
 663  *   the entire list. In both cases, processing stops after
 664  *   RTE_FLOW_ITEM_TYPE_END is encountered.
 665  * @param[out] error
 666  *   Perform verbose error reporting if not NULL.
 667  *
 668  * @return
 669  *   A positive value representing the number of bytes needed to store
 670  *   pattern items regardless of @p size on success (@p buf contents are
 671  *   truncated to @p size if not large enough), a negative errno value
 672  *   otherwise and rte_errno is set.
 673  */
 674 static int
 675 rte_flow_conv_pattern(struct rte_flow_item *dst,
 676                       const size_t size,
 677                       const struct rte_flow_item *src,
 678                       unsigned int num,
 679                       struct rte_flow_error *error)
 680 {
 681         uintptr_t data = (uintptr_t)dst;
 682         size_t off;
 683         size_t ret;
 684         unsigned int i;
 685
 686         for (i = 0, off = 0; !num || i != num; ++i, ++src, ++dst) {
 687                 /**
 688                  * allow PMD private flow item
 689                  */
 690                 if (((int)src->type >= 0) &&
 691                         ((size_t)src->type >= RTE_DIM(rte_flow_desc_item) ||
 692                     !rte_flow_desc_item[src->type].name))
 693                         return rte_flow_error_set
 694                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, src,
 695                                  "cannot convert unknown item type");
 696                 if (size >= off + sizeof(*dst))
 697                         *dst = (struct rte_flow_item){
 698                                 .type = src->type,
 699                         };
 700                 off += sizeof(*dst);
 701                 if (!src->type)
 702                         num = i + 1;
 703         }
 704         num = i;
 705         src -= num;
 706         dst -= num;
 707         do {
 708                 if (src->spec) {
 709                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 710                         ret = rte_flow_conv_item_spec
 711                                 ((void *)(data + off),
 712                                  size > off ? size - off : 0, src,
 713                                  RTE_FLOW_CONV_ITEM_SPEC);
 714                         if (size && size >= off + ret)
 715                                 dst->spec = (void *)(data + off);
 716                         off += ret;
 717
 718                 }
 719                 if (src->last) {
 720                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 721                         ret = rte_flow_conv_item_spec
 722                                 ((void *)(data + off),
 723                                  size > off ? size - off : 0, src,
 724                                  RTE_FLOW_CONV_ITEM_LAST);
 725                         if (size && size >= off + ret)
 726                                 dst->last = (void *)(data + off);
 727                         off += ret;
 728                 }
 729                 if (src->mask) {
 730                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 731                         ret = rte_flow_conv_item_spec
 732                                 ((void *)(data + off),
 733                                  size > off ? size - off : 0, src,
 734                                  RTE_FLOW_CONV_ITEM_MASK);
 735                         if (size && size >= off + ret)
 736                                 dst->mask = (void *)(data + off);
 737                         off += ret;
 738                 }
 739                 ++src;
 740                 ++dst;
 741         } while (--num);
 742         return off;
 743 }
 744
 745 /**
 746  * Copy a list of actions.
 747  *
 748  * @param[out] dst
 749  *   Destination buffer. Can be NULL if @p size is zero.
 750  * @param size
 751  *   Size of @p dst in bytes.
 752  * @param[in] src
 753  *   Source actions.
 754  * @param num
 755  *   Maximum number of actions to process from @p src or 0 to process the
 756  *   entire list. In both cases, processing stops after
 757  *   RTE_FLOW_ACTION_TYPE_END is encountered.
 758  * @param[out] error
 759  *   Perform verbose error reporting if not NULL.
 760  *
 761  * @return
 762  *   A positive value representing the number of bytes needed to store
 763  *   actions regardless of @p size on success (@p buf contents are truncated
 764  *   to @p size if not large enough), a negative errno value otherwise and
 765  *   rte_errno is set.
 766  */
 767 static int
 768 rte_flow_conv_actions(struct rte_flow_action *dst,
 769                       const size_t size,
 770                       const struct rte_flow_action *src,
 771                       unsigned int num,
 772                       struct rte_flow_error *error)
 773 {
 774         uintptr_t data = (uintptr_t)dst;
 775         size_t off;
 776         size_t ret;
 777         unsigned int i;
 778
 779         for (i = 0, off = 0; !num || i != num; ++i, ++src, ++dst) {
 780                 /**
 781                  * allow PMD private flow action
 782                  */
 783                 if (((int)src->type >= 0) &&
 784                     ((size_t)src->type >= RTE_DIM(rte_flow_desc_action) ||
 785                     !rte_flow_desc_action[src->type].name))
 786                         return rte_flow_error_set
 787                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 788                                  src, "cannot convert unknown action type");
 789                 if (size >= off + sizeof(*dst))
 790                         *dst = (struct rte_flow_action){
 791                                 .type = src->type,
 792                         };
 793                 off += sizeof(*dst);
 794                 if (!src->type)
 795                         num = i + 1;
 796         }
 797         num = i;
 798         src -= num;
 799         dst -= num;
 800         do {
 801                 if (src->conf) {
 802                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 803                         ret = rte_flow_conv_action_conf
 804                                 ((void *)(data + off),
 805                                  size > off ? size - off : 0, src);
 806                         if (size && size >= off + ret)
 807                                 dst->conf = (void *)(data + off);
 808                         off += ret;
 809                 }
 810                 ++src;
 811                 ++dst;
 812         } while (--num);
 813         return off;
 814 }
 815
 816 /**
 817  * Copy flow rule components.
 818  *
 819  * This comprises the flow rule descriptor itself, attributes, pattern and
 820  * actions list. NULL components in @p src are skipped.
 821  *
 822  * @param[out] dst
 823  *   Destination buffer. Can be NULL if @p size is zero.
 824  * @param size
 825  *   Size of @p dst in bytes.
 826  * @param[in] src
 827  *   Source flow rule descriptor.
 828  * @param[out] error
 829  *   Perform verbose error reporting if not NULL.
 830  *
 831  * @return
 832  *   A positive value representing the number of bytes needed to store all
 833  *   components including the descriptor regardless of @p size on success
 834  *   (@p buf contents are truncated to @p size if not large enough), a
 835  *   negative errno value otherwise and rte_errno is set.
 836  */
 837 static int
 838 rte_flow_conv_rule(struct rte_flow_conv_rule *dst,
 839                    const size_t size,
 840                    const struct rte_flow_conv_rule *src,
 841                    struct rte_flow_error *error)
 842 {
 843         size_t off;
 844         int ret;
 845
 846         rte_memcpy(dst,
 847                    (&(struct rte_flow_conv_rule){
 848                         .attr = NULL,
 849                         .pattern = NULL,
 850                         .actions = NULL,
 851                    }),
 852                    size > sizeof(*dst) ? sizeof(*dst) : size);
 853         off = sizeof(*dst);
 854         if (src->attr_ro) {
 855                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 856                 if (size && size >= off + sizeof(*dst->attr))
 857                         dst->attr = rte_memcpy
 858                                 ((void *)((uintptr_t)dst + off),
 859                                  src->attr_ro, sizeof(*dst->attr));
 860                 off += sizeof(*dst->attr);
 861         }
 862         if (src->pattern_ro) {
 863                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 864                 ret = rte_flow_conv_pattern((void *)((uintptr_t)dst + off),
 865                                             size > off ? size - off : 0,
 866                                             src->pattern_ro, 0, error);
 867                 if (ret < 0)
 868                         return ret;
 869                 if (size && size >= off + (size_t)ret)
 870                         dst->pattern = (void *)((uintptr_t)dst + off);
 871                 off += ret;
 872         }
 873         if (src->actions_ro) {
 874                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 875                 ret = rte_flow_conv_actions((void *)((uintptr_t)dst + off),
 876                                             size > off ? size - off : 0,
 877                                             src->actions_ro, 0, error);
 878                 if (ret < 0)
 879                         return ret;
 880                 if (size >= off + (size_t)ret)
 881                         dst->actions = (void *)((uintptr_t)dst + off);
 882                 off += ret;
 883         }
 884         return off;
 885 }
 886
 887 /**
 888  * Retrieve the name of a pattern item/action type.
 889  *
 890  * @param is_action
 891  *   Nonzero when @p src represents an action type instead of a pattern item
 892  *   type.
 893  * @param is_ptr
 894  *   Nonzero to write string address instead of contents into @p dst.
 895  * @param[out] dst
 896  *   Destination buffer. Can be NULL if @p size is zero.
 897  * @param size
 898  *   Size of @p dst in bytes.
 899  * @param[in] src
 900  *   Depending on @p is_action, source pattern item or action type cast as a
 901  *   pointer.
 902  * @param[out] error
 903  *   Perform verbose error reporting if not NULL.
 904  *
 905  * @return
 906  *   A positive value representing the number of bytes needed to store the
 907  *   name or its address regardless of @p size on success (@p buf contents
 908  *   are truncated to @p size if not large enough), a negative errno value
 909  *   otherwise and rte_errno is set.
 910  */
 911 static int
 912 rte_flow_conv_name(int is_action,
 913                    int is_ptr,
 914                    char *dst,
 915                    const size_t size,
 916                    const void *src,
 917                    struct rte_flow_error *error)
 918 {
 919         struct desc_info {
 920                 const struct rte_flow_desc_data *data;
 921                 size_t num;
 922         };
 923         static const struct desc_info info_rep[2] = {
 924                 { rte_flow_desc_item, RTE_DIM(rte_flow_desc_item), },
 925                 { rte_flow_desc_action, RTE_DIM(rte_flow_desc_action), },
 926         };
 927         const struct desc_info *const info = &info_rep[!!is_action];
 928         unsigned int type = (uintptr_t)src;
 929
 930         if (type >= info->num)
 931                 return rte_flow_error_set
 932                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 933                          "unknown object type to retrieve the name of");
 934         if (!is_ptr)
 935                 return strlcpy(dst, info->data[type].name, size);
 936         if (size >= sizeof(const char **))
 937                 *((const char **)dst) = info->data[type].name;
 938         return sizeof(const char **);
 939 }
 940
 941 /** Helper function to convert flow API objects. */
 942 int
 943 rte_flow_conv(enum rte_flow_conv_op op,
 944               void *dst,
 945               size_t size,
 946               const void *src,
 947               struct rte_flow_error *error)
 948 {
 949         switch (op) {
 950                 const struct rte_flow_attr *attr;
 951
 952         case RTE_FLOW_CONV_OP_NONE:
 953                 return 0;
 954         case RTE_FLOW_CONV_OP_ATTR:
 955                 attr = src;
 956                 if (size > sizeof(*attr))
 957                         size = sizeof(*attr);
 958                 rte_memcpy(dst, attr, size);
 959                 return sizeof(*attr);
 960         case RTE_FLOW_CONV_OP_ITEM:
 961                 return rte_flow_conv_pattern(dst, size, src, 1, error);
 962         case RTE_FLOW_CONV_OP_ACTION:
 963                 return rte_flow_conv_actions(dst, size, src, 1, error);
 964         case RTE_FLOW_CONV_OP_PATTERN:
 965                 return rte_flow_conv_pattern(dst, size, src, 0, error);
 966         case RTE_FLOW_CONV_OP_ACTIONS:
 967                 return rte_flow_conv_actions(dst, size, src, 0, error);
 968         case RTE_FLOW_CONV_OP_RULE:
 969                 return rte_flow_conv_rule(dst, size, src, error);
 970         case RTE_FLOW_CONV_OP_ITEM_NAME:
 971                 return rte_flow_conv_name(0, 0, dst, size, src, error);
 972         case RTE_FLOW_CONV_OP_ACTION_NAME:
 973                 return rte_flow_conv_name(1, 0, dst, size, src, error);
 974         case RTE_FLOW_CONV_OP_ITEM_NAME_PTR:
 975                 return rte_flow_conv_name(0, 1, dst, size, src, error);
 976         case RTE_FLOW_CONV_OP_ACTION_NAME_PTR:
 977                 return rte_flow_conv_name(1, 1, dst, size, src, error);
 978         }
 979         return rte_flow_error_set
 980                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 981                  "unknown object conversion operation");
 982 }
 983
 984 /** Store a full rte_flow description. */
 985 size_t
 986 rte_flow_copy(struct rte_flow_desc *desc, size_t len,
 987               const struct rte_flow_attr *attr,
 988               const struct rte_flow_item *items,
 989               const struct rte_flow_action *actions)
 990 {
 991         /*
 992          * Overlap struct rte_flow_conv with struct rte_flow_desc in order
 993          * to convert the former to the latter without wasting space.
 994          */
 995         struct rte_flow_conv_rule *dst =
 996                 len ?
 997                 (void *)((uintptr_t)desc +
 998                          (offsetof(struct rte_flow_desc, actions) -
 999                           offsetof(struct rte_flow_conv_rule, actions))) :
1000                 NULL;
1001         size_t dst_size =
1002                 len > sizeof(*desc) - sizeof(*dst) ?
1003                 len - (sizeof(*desc) - sizeof(*dst)) :
1004                 0;
1005         struct rte_flow_conv_rule src = {
1006                 .attr_ro = NULL,
1007                 .pattern_ro = items,
1008                 .actions_ro = actions,
1009         };
1010         int ret;
1011
1012         RTE_BUILD_BUG_ON(sizeof(struct rte_flow_desc) <
1013                          sizeof(struct rte_flow_conv_rule));
1014         if (dst_size &&
1015             (&dst->pattern != &desc->items ||
1016              &dst->actions != &desc->actions ||
1017              (uintptr_t)(dst + 1) != (uintptr_t)(desc + 1))) {
1018                 rte_errno = EINVAL;
1019                 return 0;
1020         }
1021         ret = rte_flow_conv(RTE_FLOW_CONV_OP_RULE, dst, dst_size, &src, NULL);
1022         if (ret < 0)
1023                 return 0;
1024         ret += sizeof(*desc) - sizeof(*dst);
1025         rte_memcpy(desc,
1026                    (&(struct rte_flow_desc){
1027                         .size = ret,
1028                         .attr = *attr,
1029                         .items = dst_size ? dst->pattern : NULL,
1030                         .actions = dst_size ? dst->actions : NULL,
1031                    }),
1032                    len > sizeof(*desc) ? sizeof(*desc) : len);
1033         return ret;
1034 }
1035
1036 int
1037 rte_flow_dev_dump(uint16_t port_id, struct rte_flow *flow,
1038                         FILE *file, struct rte_flow_error *error)
1039 {
1040         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1041         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1042         int ret;
1043
1044         if (unlikely(!ops))
1045                 return -rte_errno;
1046         if (likely(!!ops->dev_dump)) {
1047                 fts_enter(dev);
1048                 ret = ops->dev_dump(dev, flow, file, error);
1049                 fts_exit(dev);
1050                 return flow_err(port_id, ret, error);
1051         }
1052         return rte_flow_error_set(error, ENOSYS,
1053                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1054                                   NULL, rte_strerror(ENOSYS));
1055 }
1056
1057 int
1058 rte_flow_get_aged_flows(uint16_t port_id, void **contexts,
1059                     uint32_t nb_contexts, struct rte_flow_error *error)
1060 {
1061         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1062         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1063         int ret;
1064
1065         if (unlikely(!ops))
1066                 return -rte_errno;
1067         if (likely(!!ops->get_aged_flows)) {
1068                 fts_enter(dev);
1069                 ret = ops->get_aged_flows(dev, contexts, nb_contexts, error);
1070                 fts_exit(dev);
1071                 return flow_err(port_id, ret, error);
1072         }
1073         return rte_flow_error_set(error, ENOTSUP,
1074                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1075                                   NULL, rte_strerror(ENOTSUP));
1076 }
1077
1078 struct rte_flow_action_handle *
1079 rte_flow_action_handle_create(uint16_t port_id,
1080                               const struct rte_flow_indir_action_conf *conf,
1081                               const struct rte_flow_action *action,
1082                               struct rte_flow_error *error)
1083 {
1084         struct rte_flow_action_handle *handle;
1085         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1086
1087         if (unlikely(!ops))
1088                 return NULL;
1089         if (unlikely(!ops->action_handle_create)) {
1090                 rte_flow_error_set(error, ENOSYS,
1091                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1092                                    rte_strerror(ENOSYS));
1093                 return NULL;
1094         }
1095         handle = ops->action_handle_create(&rte_eth_devices[port_id],
1096                                            conf, action, error);
1097         if (handle == NULL)
1098                 flow_err(port_id, -rte_errno, error);
1099         return handle;
1100 }
1101
1102 int
1103 rte_flow_action_handle_destroy(uint16_t port_id,
1104                                struct rte_flow_action_handle *handle,
1105                                struct rte_flow_error *error)
1106 {
1107         int ret;
1108         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1109
1110         if (unlikely(!ops))
1111                 return -rte_errno;
1112         if (unlikely(!ops->action_handle_destroy))
1113                 return rte_flow_error_set(error, ENOSYS,
1114                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1115                                           NULL, rte_strerror(ENOSYS));
1116         ret = ops->action_handle_destroy(&rte_eth_devices[port_id],
1117                                          handle, error);
1118         return flow_err(port_id, ret, error);
1119 }
1120
1121 int
1122 rte_flow_action_handle_update(uint16_t port_id,
1123                               struct rte_flow_action_handle *handle,
1124                               const void *update,
1125                               struct rte_flow_error *error)
1126 {
1127         int ret;
1128         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1129
1130         if (unlikely(!ops))
1131                 return -rte_errno;
1132         if (unlikely(!ops->action_handle_update))
1133                 return rte_flow_error_set(error, ENOSYS,
1134                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1135                                           NULL, rte_strerror(ENOSYS));
1136         ret = ops->action_handle_update(&rte_eth_devices[port_id], handle,
1137                                         update, error);
1138         return flow_err(port_id, ret, error);
1139 }
1140
1141 int
1142 rte_flow_action_handle_query(uint16_t port_id,
1143                              const struct rte_flow_action_handle *handle,
1144                              void *data,
1145                              struct rte_flow_error *error)
1146 {
1147         int ret;
1148         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1149
1150         if (unlikely(!ops))
1151                 return -rte_errno;
1152         if (unlikely(!ops->action_handle_query))
1153                 return rte_flow_error_set(error, ENOSYS,
1154                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1155                                           NULL, rte_strerror(ENOSYS));
1156         ret = ops->action_handle_query(&rte_eth_devices[port_id], handle,
1157                                        data, error);
1158         return flow_err(port_id, ret, error);
1159 }
1160
1161 int
1162 rte_flow_tunnel_decap_set(uint16_t port_id,
1163                           struct rte_flow_tunnel *tunnel,
1164                           struct rte_flow_action **actions,
1165                           uint32_t *num_of_actions,
1166                           struct rte_flow_error *error)
1167 {
1168         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1169         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1170
1171         if (unlikely(!ops))
1172                 return -rte_errno;
1173         if (likely(!!ops->tunnel_decap_set)) {
1174                 return flow_err(port_id,
1175                                 ops->tunnel_decap_set(dev, tunnel, actions,
1176                                                       num_of_actions, error),
1177                                 error);
1178         }
1179         return rte_flow_error_set(error, ENOTSUP,
1180                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1181                                   NULL, rte_strerror(ENOTSUP));
1182 }
1183
1184 int
1185 rte_flow_tunnel_match(uint16_t port_id,
1186                       struct rte_flow_tunnel *tunnel,
1187                       struct rte_flow_item **items,
1188                       uint32_t *num_of_items,
1189                       struct rte_flow_error *error)
1190 {
1191         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1192         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1193
1194         if (unlikely(!ops))
1195                 return -rte_errno;
1196         if (likely(!!ops->tunnel_match)) {
1197                 return flow_err(port_id,
1198                                 ops->tunnel_match(dev, tunnel, items,
1199                                                   num_of_items, error),
1200                                 error);
1201         }
1202         return rte_flow_error_set(error, ENOTSUP,
1203                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1204                                   NULL, rte_strerror(ENOTSUP));
1205 }
1206
1207 int
1208 rte_flow_get_restore_info(uint16_t port_id,
1209                           struct rte_mbuf *m,
1210                           struct rte_flow_restore_info *restore_info,
1211                           struct rte_flow_error *error)
1212 {
1213         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1214         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1215
1216         if (unlikely(!ops))
1217                 return -rte_errno;
1218         if (likely(!!ops->get_restore_info)) {
1219                 return flow_err(port_id,
1220                                 ops->get_restore_info(dev, m, restore_info,
1221                                                       error),
1222                                 error);
1223         }
1224         return rte_flow_error_set(error, ENOTSUP,
1225                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1226                                   NULL, rte_strerror(ENOTSUP));
1227 }
1228
1229 int
1230 rte_flow_tunnel_action_decap_release(uint16_t port_id,
1231                                      struct rte_flow_action *actions,
1232                                      uint32_t num_of_actions,
1233                                      struct rte_flow_error *error)
1234 {
1235         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1236         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1237
1238         if (unlikely(!ops))
1239                 return -rte_errno;
1240         if (likely(!!ops->tunnel_action_decap_release)) {
1241                 return flow_err(port_id,
1242                                 ops->tunnel_action_decap_release(dev, actions,
1243                                                                  num_of_actions,
1244                                                                  error),
1245                                 error);
1246         }
1247         return rte_flow_error_set(error, ENOTSUP,
1248                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1249                                   NULL, rte_strerror(ENOTSUP));
1250 }
1251
1252 int
1253 rte_flow_tunnel_item_release(uint16_t port_id,
1254                              struct rte_flow_item *items,
1255                              uint32_t num_of_items,
1256                              struct rte_flow_error *error)
1257 {
1258         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1259         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1260
1261         if (unlikely(!ops))
1262                 return -rte_errno;
1263         if (likely(!!ops->tunnel_item_release)) {
1264                 return flow_err(port_id,
1265                                 ops->tunnel_item_release(dev, items,
1266                                                          num_of_items, error),
1267                                 error);
1268         }
1269         return rte_flow_error_set(error, ENOTSUP,
1270                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1271                                   NULL, rte_strerror(ENOTSUP));
1272 }
1273
1274 int
1275 rte_flow_pick_transfer_proxy(uint16_t port_id, uint16_t *proxy_port_id,
1276                              struct rte_flow_error *error)
1277 {
1278         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
1279         struct rte_eth_dev *dev;
1280
1281         if (unlikely(ops == NULL))
1282                 return -rte_errno;
1283
1284         if (ops->pick_transfer_proxy == NULL) {
1285                 *proxy_port_id = port_id;
1286                 return 0;
1287         }
1288
1289         dev = &rte_eth_devices[port_id];
1290
1291         return flow_err(port_id,
1292                         ops->pick_transfer_proxy(dev, proxy_port_id, error),
1293                         error);
1294 }