lib/librte_ethdev/rte_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stddef.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_errno.h>
  13 #include <rte_branch_prediction.h>
  14 #include <rte_string_fns.h>
  15 #include "rte_ethdev.h"
  16 #include "rte_flow_driver.h"
  17 #include "rte_flow.h"
  18
  19 /**
  20  * Flow elements description tables.
  21  */
  22 struct rte_flow_desc_data {
  23         const char *name;
  24         size_t size;
  25 };
  26
  27 /** Generate flow_item[] entry. */
  28 #define MK_FLOW_ITEM(t, s) \
  29         [RTE_FLOW_ITEM_TYPE_ ## t] = { \
  30                 .name = # t, \
  31                 .size = s, \
  32         }
  33
  34 /** Information about known flow pattern items. */
  35 static const struct rte_flow_desc_data rte_flow_desc_item[] = {
  36         MK_FLOW_ITEM(END, 0),
  37         MK_FLOW_ITEM(VOID, 0),
  38         MK_FLOW_ITEM(INVERT, 0),
  39         MK_FLOW_ITEM(ANY, sizeof(struct rte_flow_item_any)),
  40         MK_FLOW_ITEM(PF, 0),
  41         MK_FLOW_ITEM(VF, sizeof(struct rte_flow_item_vf)),
  42         MK_FLOW_ITEM(PHY_PORT, sizeof(struct rte_flow_item_phy_port)),
  43         MK_FLOW_ITEM(PORT_ID, sizeof(struct rte_flow_item_port_id)),
  44         MK_FLOW_ITEM(RAW, sizeof(struct rte_flow_item_raw)),
  45         MK_FLOW_ITEM(ETH, sizeof(struct rte_flow_item_eth)),
  46         MK_FLOW_ITEM(VLAN, sizeof(struct rte_flow_item_vlan)),
  47         MK_FLOW_ITEM(IPV4, sizeof(struct rte_flow_item_ipv4)),
  48         MK_FLOW_ITEM(IPV6, sizeof(struct rte_flow_item_ipv6)),
  49         MK_FLOW_ITEM(ICMP, sizeof(struct rte_flow_item_icmp)),
  50         MK_FLOW_ITEM(UDP, sizeof(struct rte_flow_item_udp)),
  51         MK_FLOW_ITEM(TCP, sizeof(struct rte_flow_item_tcp)),
  52         MK_FLOW_ITEM(SCTP, sizeof(struct rte_flow_item_sctp)),
  53         MK_FLOW_ITEM(VXLAN, sizeof(struct rte_flow_item_vxlan)),
  54         MK_FLOW_ITEM(E_TAG, sizeof(struct rte_flow_item_e_tag)),
  55         MK_FLOW_ITEM(NVGRE, sizeof(struct rte_flow_item_nvgre)),
  56         MK_FLOW_ITEM(MPLS, sizeof(struct rte_flow_item_mpls)),
  57         MK_FLOW_ITEM(GRE, sizeof(struct rte_flow_item_gre)),
  58         MK_FLOW_ITEM(FUZZY, sizeof(struct rte_flow_item_fuzzy)),
  59         MK_FLOW_ITEM(GTP, sizeof(struct rte_flow_item_gtp)),
  60         MK_FLOW_ITEM(GTPC, sizeof(struct rte_flow_item_gtp)),
  61         MK_FLOW_ITEM(GTPU, sizeof(struct rte_flow_item_gtp)),
  62         MK_FLOW_ITEM(ESP, sizeof(struct rte_flow_item_esp)),
  63         MK_FLOW_ITEM(GENEVE, sizeof(struct rte_flow_item_geneve)),
  64         MK_FLOW_ITEM(VXLAN_GPE, sizeof(struct rte_flow_item_vxlan_gpe)),
  65         MK_FLOW_ITEM(ARP_ETH_IPV4, sizeof(struct rte_flow_item_arp_eth_ipv4)),
  66         MK_FLOW_ITEM(IPV6_EXT, sizeof(struct rte_flow_item_ipv6_ext)),
  67         MK_FLOW_ITEM(ICMP6, sizeof(struct rte_flow_item_icmp6)),
  68         MK_FLOW_ITEM(ICMP6_ND_NS, sizeof(struct rte_flow_item_icmp6_nd_ns)),
  69         MK_FLOW_ITEM(ICMP6_ND_NA, sizeof(struct rte_flow_item_icmp6_nd_na)),
  70         MK_FLOW_ITEM(ICMP6_ND_OPT, sizeof(struct rte_flow_item_icmp6_nd_opt)),
  71         MK_FLOW_ITEM(ICMP6_ND_OPT_SLA_ETH,
  72                      sizeof(struct rte_flow_item_icmp6_nd_opt_sla_eth)),
  73         MK_FLOW_ITEM(ICMP6_ND_OPT_TLA_ETH,
  74                      sizeof(struct rte_flow_item_icmp6_nd_opt_tla_eth)),
  75         MK_FLOW_ITEM(MARK, sizeof(struct rte_flow_item_mark)),
  76         MK_FLOW_ITEM(META, sizeof(struct rte_flow_item_meta)),
  77         MK_FLOW_ITEM(GRE_KEY, sizeof(rte_be32_t)),
  78         MK_FLOW_ITEM(GTP_PSC, sizeof(struct rte_flow_item_gtp_psc)),
  79         MK_FLOW_ITEM(PPPOES, sizeof(struct rte_flow_item_pppoe)),
  80         MK_FLOW_ITEM(PPPOED, sizeof(struct rte_flow_item_pppoe)),
  81         MK_FLOW_ITEM(PPPOE_PROTO_ID,
  82                         sizeof(struct rte_flow_item_pppoe_proto_id)),
  83         MK_FLOW_ITEM(NSH, sizeof(struct rte_flow_item_nsh)),
  84 };
  85
  86 /** Generate flow_action[] entry. */
  87 #define MK_FLOW_ACTION(t, s) \
  88         [RTE_FLOW_ACTION_TYPE_ ## t] = { \
  89                 .name = # t, \
  90                 .size = s, \
  91         }
  92
  93 /** Information about known flow actions. */
  94 static const struct rte_flow_desc_data rte_flow_desc_action[] = {
  95         MK_FLOW_ACTION(END, 0),
  96         MK_FLOW_ACTION(VOID, 0),
  97         MK_FLOW_ACTION(PASSTHRU, 0),
  98         MK_FLOW_ACTION(JUMP, sizeof(struct rte_flow_action_jump)),
  99         MK_FLOW_ACTION(MARK, sizeof(struct rte_flow_action_mark)),
 100         MK_FLOW_ACTION(FLAG, 0),
 101         MK_FLOW_ACTION(QUEUE, sizeof(struct rte_flow_action_queue)),
 102         MK_FLOW_ACTION(DROP, 0),
 103         MK_FLOW_ACTION(COUNT, sizeof(struct rte_flow_action_count)),
 104         MK_FLOW_ACTION(RSS, sizeof(struct rte_flow_action_rss)),
 105         MK_FLOW_ACTION(PF, 0),
 106         MK_FLOW_ACTION(VF, sizeof(struct rte_flow_action_vf)),
 107         MK_FLOW_ACTION(PHY_PORT, sizeof(struct rte_flow_action_phy_port)),
 108         MK_FLOW_ACTION(PORT_ID, sizeof(struct rte_flow_action_port_id)),
 109         MK_FLOW_ACTION(METER, sizeof(struct rte_flow_action_meter)),
 110         MK_FLOW_ACTION(SECURITY, sizeof(struct rte_flow_action_security)),
 111         MK_FLOW_ACTION(OF_SET_MPLS_TTL,
 112                        sizeof(struct rte_flow_action_of_set_mpls_ttl)),
 113         MK_FLOW_ACTION(OF_DEC_MPLS_TTL, 0),
 114         MK_FLOW_ACTION(OF_SET_NW_TTL,
 115                        sizeof(struct rte_flow_action_of_set_nw_ttl)),
 116         MK_FLOW_ACTION(OF_DEC_NW_TTL, 0),
 117         MK_FLOW_ACTION(OF_COPY_TTL_OUT, 0),
 118         MK_FLOW_ACTION(OF_COPY_TTL_IN, 0),
 119         MK_FLOW_ACTION(OF_POP_VLAN, 0),
 120         MK_FLOW_ACTION(OF_PUSH_VLAN,
 121                        sizeof(struct rte_flow_action_of_push_vlan)),
 122         MK_FLOW_ACTION(OF_SET_VLAN_VID,
 123                        sizeof(struct rte_flow_action_of_set_vlan_vid)),
 124         MK_FLOW_ACTION(OF_SET_VLAN_PCP,
 125                        sizeof(struct rte_flow_action_of_set_vlan_pcp)),
 126         MK_FLOW_ACTION(OF_POP_MPLS,
 127                        sizeof(struct rte_flow_action_of_pop_mpls)),
 128         MK_FLOW_ACTION(OF_PUSH_MPLS,
 129                        sizeof(struct rte_flow_action_of_push_mpls)),
 130         MK_FLOW_ACTION(VXLAN_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)),
 131         MK_FLOW_ACTION(VXLAN_DECAP, 0),
 132         MK_FLOW_ACTION(NVGRE_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)),
 133         MK_FLOW_ACTION(NVGRE_DECAP, 0),
 134         MK_FLOW_ACTION(RAW_ENCAP, sizeof(struct rte_flow_action_raw_encap)),
 135         MK_FLOW_ACTION(RAW_DECAP, sizeof(struct rte_flow_action_raw_decap)),
 136         MK_FLOW_ACTION(SET_IPV4_SRC,
 137                        sizeof(struct rte_flow_action_set_ipv4)),
 138         MK_FLOW_ACTION(SET_IPV4_DST,
 139                        sizeof(struct rte_flow_action_set_ipv4)),
 140         MK_FLOW_ACTION(SET_IPV6_SRC,
 141                        sizeof(struct rte_flow_action_set_ipv6)),
 142         MK_FLOW_ACTION(SET_IPV6_DST,
 143                        sizeof(struct rte_flow_action_set_ipv6)),
 144         MK_FLOW_ACTION(SET_TP_SRC,
 145                        sizeof(struct rte_flow_action_set_tp)),
 146         MK_FLOW_ACTION(SET_TP_DST,
 147                        sizeof(struct rte_flow_action_set_tp)),
 148         MK_FLOW_ACTION(MAC_SWAP, 0),
 149         MK_FLOW_ACTION(DEC_TTL, 0),
 150         MK_FLOW_ACTION(SET_TTL, sizeof(struct rte_flow_action_set_ttl)),
 151         MK_FLOW_ACTION(SET_MAC_SRC, sizeof(struct rte_flow_action_set_mac)),
 152         MK_FLOW_ACTION(SET_MAC_DST, sizeof(struct rte_flow_action_set_mac)),
 153         MK_FLOW_ACTION(INC_TCP_SEQ, sizeof(rte_be32_t)),
 154         MK_FLOW_ACTION(DEC_TCP_SEQ, sizeof(rte_be32_t)),
 155         MK_FLOW_ACTION(INC_TCP_ACK, sizeof(rte_be32_t)),
 156         MK_FLOW_ACTION(DEC_TCP_ACK, sizeof(rte_be32_t)),
 157 };
 158
 159 static int
 160 flow_err(uint16_t port_id, int ret, struct rte_flow_error *error)
 161 {
 162         if (ret == 0)
 163                 return 0;
 164         if (rte_eth_dev_is_removed(port_id))
 165                 return rte_flow_error_set(error, EIO,
 166                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 167                                           NULL, rte_strerror(EIO));
 168         return ret;
 169 }
 170
 171 /* Get generic flow operations structure from a port. */
 172 const struct rte_flow_ops *
 173 rte_flow_ops_get(uint16_t port_id, struct rte_flow_error *error)
 174 {
 175         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 176         const struct rte_flow_ops *ops;
 177         int code;
 178
 179         if (unlikely(!rte_eth_dev_is_valid_port(port_id)))
 180                 code = ENODEV;
 181         else if (unlikely(!dev->dev_ops->filter_ctrl ||
 182                           dev->dev_ops->filter_ctrl(dev,
 183                                                     RTE_ETH_FILTER_GENERIC,
 184                                                     RTE_ETH_FILTER_GET,
 185                                                     &ops) ||
 186                           !ops))
 187                 code = ENOSYS;
 188         else
 189                 return ops;
 190         rte_flow_error_set(error, code, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 191                            NULL, rte_strerror(code));
 192         return NULL;
 193 }
 194
 195 /* Check whether a flow rule can be created on a given port. */
 196 int
 197 rte_flow_validate(uint16_t port_id,
 198                   const struct rte_flow_attr *attr,
 199                   const struct rte_flow_item pattern[],
 200                   const struct rte_flow_action actions[],
 201                   struct rte_flow_error *error)
 202 {
 203         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 204         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 205
 206         if (unlikely(!ops))
 207                 return -rte_errno;
 208         if (likely(!!ops->validate))
 209                 return flow_err(port_id, ops->validate(dev, attr, pattern,
 210                                                        actions, error), error);
 211         return rte_flow_error_set(error, ENOSYS,
 212                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 213                                   NULL, rte_strerror(ENOSYS));
 214 }
 215
 216 /* Create a flow rule on a given port. */
 217 struct rte_flow *
 218 rte_flow_create(uint16_t port_id,
 219                 const struct rte_flow_attr *attr,
 220                 const struct rte_flow_item pattern[],
 221                 const struct rte_flow_action actions[],
 222                 struct rte_flow_error *error)
 223 {
 224         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 225         struct rte_flow *flow;
 226         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 227
 228         if (unlikely(!ops))
 229                 return NULL;
 230         if (likely(!!ops->create)) {
 231                 flow = ops->create(dev, attr, pattern, actions, error);
 232                 if (flow == NULL)
 233                         flow_err(port_id, -rte_errno, error);
 234                 return flow;
 235         }
 236         rte_flow_error_set(error, ENOSYS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 237                            NULL, rte_strerror(ENOSYS));
 238         return NULL;
 239 }
 240
 241 /* Destroy a flow rule on a given port. */
 242 int
 243 rte_flow_destroy(uint16_t port_id,
 244                  struct rte_flow *flow,
 245                  struct rte_flow_error *error)
 246 {
 247         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 248         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 249
 250         if (unlikely(!ops))
 251                 return -rte_errno;
 252         if (likely(!!ops->destroy))
 253                 return flow_err(port_id, ops->destroy(dev, flow, error),
 254                                 error);
 255         return rte_flow_error_set(error, ENOSYS,
 256                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 257                                   NULL, rte_strerror(ENOSYS));
 258 }
 259
 260 /* Destroy all flow rules associated with a port. */
 261 int
 262 rte_flow_flush(uint16_t port_id,
 263                struct rte_flow_error *error)
 264 {
 265         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 266         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 267
 268         if (unlikely(!ops))
 269                 return -rte_errno;
 270         if (likely(!!ops->flush))
 271                 return flow_err(port_id, ops->flush(dev, error), error);
 272         return rte_flow_error_set(error, ENOSYS,
 273                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 274                                   NULL, rte_strerror(ENOSYS));
 275 }
 276
 277 /* Query an existing flow rule. */
 278 int
 279 rte_flow_query(uint16_t port_id,
 280                struct rte_flow *flow,
 281                const struct rte_flow_action *action,
 282                void *data,
 283                struct rte_flow_error *error)
 284 {
 285         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 286         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 287
 288         if (!ops)
 289                 return -rte_errno;
 290         if (likely(!!ops->query))
 291                 return flow_err(port_id, ops->query(dev, flow, action, data,
 292                                                     error), error);
 293         return rte_flow_error_set(error, ENOSYS,
 294                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 295                                   NULL, rte_strerror(ENOSYS));
 296 }
 297
 298 /* Restrict ingress traffic to the defined flow rules. */
 299 int
 300 rte_flow_isolate(uint16_t port_id,
 301                  int set,
 302                  struct rte_flow_error *error)
 303 {
 304         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 305         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 306
 307         if (!ops)
 308                 return -rte_errno;
 309         if (likely(!!ops->isolate))
 310                 return flow_err(port_id, ops->isolate(dev, set, error), error);
 311         return rte_flow_error_set(error, ENOSYS,
 312                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 313                                   NULL, rte_strerror(ENOSYS));
 314 }
 315
 316 /* Initialize flow error structure. */
 317 int
 318 rte_flow_error_set(struct rte_flow_error *error,
 319                    int code,
 320                    enum rte_flow_error_type type,
 321                    const void *cause,
 322                    const char *message)
 323 {
 324         if (error) {
 325                 *error = (struct rte_flow_error){
 326                         .type = type,
 327                         .cause = cause,
 328                         .message = message,
 329                 };
 330         }
 331         rte_errno = code;
 332         return -code;
 333 }
 334
 335 /** Pattern item specification types. */
 336 enum rte_flow_conv_item_spec_type {
 337         RTE_FLOW_CONV_ITEM_SPEC,
 338         RTE_FLOW_CONV_ITEM_LAST,
 339         RTE_FLOW_CONV_ITEM_MASK,
 340 };
 341
 342 /**
 343  * Copy pattern item specification.
 344  *
 345  * @param[out] buf
 346  *   Output buffer. Can be NULL if @p size is zero.
 347  * @param size
 348  *   Size of @p buf in bytes.
 349  * @param[in] item
 350  *   Pattern item to copy specification from.
 351  * @param type
 352  *   Specification selector for either @p spec, @p last or @p mask.
 353  *
 354  * @return
 355  *   Number of bytes needed to store pattern item specification regardless
 356  *   of @p size. @p buf contents are truncated to @p size if not large
 357  *   enough.
 358  */
 359 static size_t
 360 rte_flow_conv_item_spec(void *buf, const size_t size,
 361                         const struct rte_flow_item *item,
 362                         enum rte_flow_conv_item_spec_type type)
 363 {
 364         size_t off;
 365         const void *data =
 366                 type == RTE_FLOW_CONV_ITEM_SPEC ? item->spec :
 367                 type == RTE_FLOW_CONV_ITEM_LAST ? item->last :
 368                 type == RTE_FLOW_CONV_ITEM_MASK ? item->mask :
 369                 NULL;
 370
 371         switch (item->type) {
 372                 union {
 373                         const struct rte_flow_item_raw *raw;
 374                 } spec;
 375                 union {
 376                         const struct rte_flow_item_raw *raw;
 377                 } last;
 378                 union {
 379                         const struct rte_flow_item_raw *raw;
 380                 } mask;
 381                 union {
 382                         const struct rte_flow_item_raw *raw;
 383                 } src;
 384                 union {
 385                         struct rte_flow_item_raw *raw;
 386                 } dst;
 387                 size_t tmp;
 388
 389         case RTE_FLOW_ITEM_TYPE_RAW:
 390                 spec.raw = item->spec;
 391                 last.raw = item->last ? item->last : item->spec;
 392                 mask.raw = item->mask ? item->mask : &rte_flow_item_raw_mask;
 393                 src.raw = data;
 394                 dst.raw = buf;
 395                 rte_memcpy(dst.raw,
 396                            (&(struct rte_flow_item_raw){
 397                                 .relative = src.raw->relative,
 398                                 .search = src.raw->search,
 399                                 .reserved = src.raw->reserved,
 400                                 .offset = src.raw->offset,
 401                                 .limit = src.raw->limit,
 402                                 .length = src.raw->length,
 403                            }),
 404                            size > sizeof(*dst.raw) ? sizeof(*dst.raw) : size);
 405                 off = sizeof(*dst.raw);
 406                 if (type == RTE_FLOW_CONV_ITEM_SPEC ||
 407                     (type == RTE_FLOW_CONV_ITEM_MASK &&
 408                      ((spec.raw->length & mask.raw->length) >=
 409                       (last.raw->length & mask.raw->length))))
 410                         tmp = spec.raw->length & mask.raw->length;
 411                 else
 412                         tmp = last.raw->length & mask.raw->length;
 413                 if (tmp) {
 414                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.raw->pattern));
 415                         if (size >= off + tmp)
 416                                 dst.raw->pattern = rte_memcpy
 417                                         ((void *)((uintptr_t)dst.raw + off),
 418                                          src.raw->pattern, tmp);
 419                         off += tmp;
 420                 }
 421                 break;
 422         default:
 423                 off = rte_flow_desc_item[item->type].size;
 424                 rte_memcpy(buf, data, (size > off ? off : size));
 425                 break;
 426         }
 427         return off;
 428 }
 429
 430 /**
 431  * Copy action configuration.
 432  *
 433  * @param[out] buf
 434  *   Output buffer. Can be NULL if @p size is zero.
 435  * @param size
 436  *   Size of @p buf in bytes.
 437  * @param[in] action
 438  *   Action to copy configuration from.
 439  *
 440  * @return
 441  *   Number of bytes needed to store pattern item specification regardless
 442  *   of @p size. @p buf contents are truncated to @p size if not large
 443  *   enough.
 444  */
 445 static size_t
 446 rte_flow_conv_action_conf(void *buf, const size_t size,
 447                           const struct rte_flow_action *action)
 448 {
 449         size_t off;
 450
 451         switch (action->type) {
 452                 union {
 453                         const struct rte_flow_action_rss *rss;
 454                         const struct rte_flow_action_vxlan_encap *vxlan_encap;
 455                         const struct rte_flow_action_nvgre_encap *nvgre_encap;
 456                 } src;
 457                 union {
 458                         struct rte_flow_action_rss *rss;
 459                         struct rte_flow_action_vxlan_encap *vxlan_encap;
 460                         struct rte_flow_action_nvgre_encap *nvgre_encap;
 461                 } dst;
 462                 size_t tmp;
 463                 int ret;
 464
 465         case RTE_FLOW_ACTION_TYPE_RSS:
 466                 src.rss = action->conf;
 467                 dst.rss = buf;
 468                 rte_memcpy(dst.rss,
 469                            (&(struct rte_flow_action_rss){
 470                                 .func = src.rss->func,
 471                                 .level = src.rss->level,
 472                                 .types = src.rss->types,
 473                                 .key_len = src.rss->key_len,
 474                                 .queue_num = src.rss->queue_num,
 475                            }),
 476                            size > sizeof(*dst.rss) ? sizeof(*dst.rss) : size);
 477                 off = sizeof(*dst.rss);
 478                 if (src.rss->key_len) {
 479                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->key));
 480                         tmp = sizeof(*src.rss->key) * src.rss->key_len;
 481                         if (size >= off + tmp)
 482                                 dst.rss->key = rte_memcpy
 483                                         ((void *)((uintptr_t)dst.rss + off),
 484                                          src.rss->key, tmp);
 485                         off += tmp;
 486                 }
 487                 if (src.rss->queue_num) {
 488                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->queue));
 489                         tmp = sizeof(*src.rss->queue) * src.rss->queue_num;
 490                         if (size >= off + tmp)
 491                                 dst.rss->queue = rte_memcpy
 492                                         ((void *)((uintptr_t)dst.rss + off),
 493                                          src.rss->queue, tmp);
 494                         off += tmp;
 495                 }
 496                 break;
 497         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
 498         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
 499                 src.vxlan_encap = action->conf;
 500                 dst.vxlan_encap = buf;
 501                 RTE_BUILD_BUG_ON(sizeof(*src.vxlan_encap) !=
 502                                  sizeof(*src.nvgre_encap) ||
 503                                  offsetof(struct rte_flow_action_vxlan_encap,
 504                                           definition) !=
 505                                  offsetof(struct rte_flow_action_nvgre_encap,
 506                                           definition));
 507                 off = sizeof(*dst.vxlan_encap);
 508                 if (src.vxlan_encap->definition) {
 509                         off = RTE_ALIGN_CEIL
 510                                 (off, sizeof(*dst.vxlan_encap->definition));
 511                         ret = rte_flow_conv
 512                                 (RTE_FLOW_CONV_OP_PATTERN,
 513                                  (void *)((uintptr_t)dst.vxlan_encap + off),
 514                                  size > off ? size - off : 0,
 515                                  src.vxlan_encap->definition, NULL);
 516                         if (ret < 0)
 517                                 return 0;
 518                         if (size >= off + ret)
 519                                 dst.vxlan_encap->definition =
 520                                         (void *)((uintptr_t)dst.vxlan_encap +
 521                                                  off);
 522                         off += ret;
 523                 }
 524                 break;
 525         default:
 526                 off = rte_flow_desc_action[action->type].size;
 527                 rte_memcpy(buf, action->conf, (size > off ? off : size));
 528                 break;
 529         }
 530         return off;
 531 }
 532
 533 /**
 534  * Copy a list of pattern items.
 535  *
 536  * @param[out] dst
 537  *   Destination buffer. Can be NULL if @p size is zero.
 538  * @param size
 539  *   Size of @p dst in bytes.
 540  * @param[in] src
 541  *   Source pattern items.
 542  * @param num
 543  *   Maximum number of pattern items to process from @p src or 0 to process
 544  *   the entire list. In both cases, processing stops after
 545  *   RTE_FLOW_ITEM_TYPE_END is encountered.
 546  * @param[out] error
 547  *   Perform verbose error reporting if not NULL.
 548  *
 549  * @return
 550  *   A positive value representing the number of bytes needed to store
 551  *   pattern items regardless of @p size on success (@p buf contents are
 552  *   truncated to @p size if not large enough), a negative errno value
 553  *   otherwise and rte_errno is set.
 554  */
 555 static int
 556 rte_flow_conv_pattern(struct rte_flow_item *dst,
 557                       const size_t size,
 558                       const struct rte_flow_item *src,
 559                       unsigned int num,
 560                       struct rte_flow_error *error)
 561 {
 562         uintptr_t data = (uintptr_t)dst;
 563         size_t off;
 564         size_t ret;
 565         unsigned int i;
 566
 567         for (i = 0, off = 0; !num || i != num; ++i, ++src, ++dst) {
 568                 if ((size_t)src->type >= RTE_DIM(rte_flow_desc_item) ||
 569                     !rte_flow_desc_item[src->type].name)
 570                         return rte_flow_error_set
 571                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, src,
 572                                  "cannot convert unknown item type");
 573                 if (size >= off + sizeof(*dst))
 574                         *dst = (struct rte_flow_item){
 575                                 .type = src->type,
 576                         };
 577                 off += sizeof(*dst);
 578                 if (!src->type)
 579                         num = i + 1;
 580         }
 581         num = i;
 582         src -= num;
 583         dst -= num;
 584         do {
 585                 if (src->spec) {
 586                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 587                         ret = rte_flow_conv_item_spec
 588                                 ((void *)(data + off),
 589                                  size > off ? size - off : 0, src,
 590                                  RTE_FLOW_CONV_ITEM_SPEC);
 591                         if (size && size >= off + ret)
 592                                 dst->spec = (void *)(data + off);
 593                         off += ret;
 594
 595                 }
 596                 if (src->last) {
 597                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 598                         ret = rte_flow_conv_item_spec
 599                                 ((void *)(data + off),
 600                                  size > off ? size - off : 0, src,
 601                                  RTE_FLOW_CONV_ITEM_LAST);
 602                         if (size && size >= off + ret)
 603                                 dst->last = (void *)(data + off);
 604                         off += ret;
 605                 }
 606                 if (src->mask) {
 607                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 608                         ret = rte_flow_conv_item_spec
 609                                 ((void *)(data + off),
 610                                  size > off ? size - off : 0, src,
 611                                  RTE_FLOW_CONV_ITEM_MASK);
 612                         if (size && size >= off + ret)
 613                                 dst->mask = (void *)(data + off);
 614                         off += ret;
 615                 }
 616                 ++src;
 617                 ++dst;
 618         } while (--num);
 619         return off;
 620 }
 621
 622 /**
 623  * Copy a list of actions.
 624  *
 625  * @param[out] dst
 626  *   Destination buffer. Can be NULL if @p size is zero.
 627  * @param size
 628  *   Size of @p dst in bytes.
 629  * @param[in] src
 630  *   Source actions.
 631  * @param num
 632  *   Maximum number of actions to process from @p src or 0 to process the
 633  *   entire list. In both cases, processing stops after
 634  *   RTE_FLOW_ACTION_TYPE_END is encountered.
 635  * @param[out] error
 636  *   Perform verbose error reporting if not NULL.
 637  *
 638  * @return
 639  *   A positive value representing the number of bytes needed to store
 640  *   actions regardless of @p size on success (@p buf contents are truncated
 641  *   to @p size if not large enough), a negative errno value otherwise and
 642  *   rte_errno is set.
 643  */
 644 static int
 645 rte_flow_conv_actions(struct rte_flow_action *dst,
 646                       const size_t size,
 647                       const struct rte_flow_action *src,
 648                       unsigned int num,
 649                       struct rte_flow_error *error)
 650 {
 651         uintptr_t data = (uintptr_t)dst;
 652         size_t off;
 653         size_t ret;
 654         unsigned int i;
 655
 656         for (i = 0, off = 0; !num || i != num; ++i, ++src, ++dst) {
 657                 if ((size_t)src->type >= RTE_DIM(rte_flow_desc_action) ||
 658                     !rte_flow_desc_action[src->type].name)
 659                         return rte_flow_error_set
 660                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 661                                  src, "cannot convert unknown action type");
 662                 if (size >= off + sizeof(*dst))
 663                         *dst = (struct rte_flow_action){
 664                                 .type = src->type,
 665                         };
 666                 off += sizeof(*dst);
 667                 if (!src->type)
 668                         num = i + 1;
 669         }
 670         num = i;
 671         src -= num;
 672         dst -= num;
 673         do {
 674                 if (src->conf) {
 675                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 676                         ret = rte_flow_conv_action_conf
 677                                 ((void *)(data + off),
 678                                  size > off ? size - off : 0, src);
 679                         if (size && size >= off + ret)
 680                                 dst->conf = (void *)(data + off);
 681                         off += ret;
 682                 }
 683                 ++src;
 684                 ++dst;
 685         } while (--num);
 686         return off;
 687 }
 688
 689 /**
 690  * Copy flow rule components.
 691  *
 692  * This comprises the flow rule descriptor itself, attributes, pattern and
 693  * actions list. NULL components in @p src are skipped.
 694  *
 695  * @param[out] dst
 696  *   Destination buffer. Can be NULL if @p size is zero.
 697  * @param size
 698  *   Size of @p dst in bytes.
 699  * @param[in] src
 700  *   Source flow rule descriptor.
 701  * @param[out] error
 702  *   Perform verbose error reporting if not NULL.
 703  *
 704  * @return
 705  *   A positive value representing the number of bytes needed to store all
 706  *   components including the descriptor regardless of @p size on success
 707  *   (@p buf contents are truncated to @p size if not large enough), a
 708  *   negative errno value otherwise and rte_errno is set.
 709  */
 710 static int
 711 rte_flow_conv_rule(struct rte_flow_conv_rule *dst,
 712                    const size_t size,
 713                    const struct rte_flow_conv_rule *src,
 714                    struct rte_flow_error *error)
 715 {
 716         size_t off;
 717         int ret;
 718
 719         rte_memcpy(dst,
 720                    (&(struct rte_flow_conv_rule){
 721                         .attr = NULL,
 722                         .pattern = NULL,
 723                         .actions = NULL,
 724                    }),
 725                    size > sizeof(*dst) ? sizeof(*dst) : size);
 726         off = sizeof(*dst);
 727         if (src->attr_ro) {
 728                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 729                 if (size && size >= off + sizeof(*dst->attr))
 730                         dst->attr = rte_memcpy
 731                                 ((void *)((uintptr_t)dst + off),
 732                                  src->attr_ro, sizeof(*dst->attr));
 733                 off += sizeof(*dst->attr);
 734         }
 735         if (src->pattern_ro) {
 736                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 737                 ret = rte_flow_conv_pattern((void *)((uintptr_t)dst + off),
 738                                             size > off ? size - off : 0,
 739                                             src->pattern_ro, 0, error);
 740                 if (ret < 0)
 741                         return ret;
 742                 if (size && size >= off + (size_t)ret)
 743                         dst->pattern = (void *)((uintptr_t)dst + off);
 744                 off += ret;
 745         }
 746         if (src->actions_ro) {
 747                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 748                 ret = rte_flow_conv_actions((void *)((uintptr_t)dst + off),
 749                                             size > off ? size - off : 0,
 750                                             src->actions_ro, 0, error);
 751                 if (ret < 0)
 752                         return ret;
 753                 if (size >= off + (size_t)ret)
 754                         dst->actions = (void *)((uintptr_t)dst + off);
 755                 off += ret;
 756         }
 757         return off;
 758 }
 759
 760 /**
 761  * Retrieve the name of a pattern item/action type.
 762  *
 763  * @param is_action
 764  *   Nonzero when @p src represents an action type instead of a pattern item
 765  *   type.
 766  * @param is_ptr
 767  *   Nonzero to write string address instead of contents into @p dst.
 768  * @param[out] dst
 769  *   Destination buffer. Can be NULL if @p size is zero.
 770  * @param size
 771  *   Size of @p dst in bytes.
 772  * @param[in] src
 773  *   Depending on @p is_action, source pattern item or action type cast as a
 774  *   pointer.
 775  * @param[out] error
 776  *   Perform verbose error reporting if not NULL.
 777  *
 778  * @return
 779  *   A positive value representing the number of bytes needed to store the
 780  *   name or its address regardless of @p size on success (@p buf contents
 781  *   are truncated to @p size if not large enough), a negative errno value
 782  *   otherwise and rte_errno is set.
 783  */
 784 static int
 785 rte_flow_conv_name(int is_action,
 786                    int is_ptr,
 787                    char *dst,
 788                    const size_t size,
 789                    const void *src,
 790                    struct rte_flow_error *error)
 791 {
 792         struct desc_info {
 793                 const struct rte_flow_desc_data *data;
 794                 size_t num;
 795         };
 796         static const struct desc_info info_rep[2] = {
 797                 { rte_flow_desc_item, RTE_DIM(rte_flow_desc_item), },
 798                 { rte_flow_desc_action, RTE_DIM(rte_flow_desc_action), },
 799         };
 800         const struct desc_info *const info = &info_rep[!!is_action];
 801         unsigned int type = (uintptr_t)src;
 802
 803         if (type >= info->num)
 804                 return rte_flow_error_set
 805                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 806                          "unknown object type to retrieve the name of");
 807         if (!is_ptr)
 808                 return strlcpy(dst, info->data[type].name, size);
 809         if (size >= sizeof(const char **))
 810                 *((const char **)dst) = info->data[type].name;
 811         return sizeof(const char **);
 812 }
 813
 814 /** Helper function to convert flow API objects. */
 815 int
 816 rte_flow_conv(enum rte_flow_conv_op op,
 817               void *dst,
 818               size_t size,
 819               const void *src,
 820               struct rte_flow_error *error)
 821 {
 822         switch (op) {
 823                 const struct rte_flow_attr *attr;
 824
 825         case RTE_FLOW_CONV_OP_NONE:
 826                 return 0;
 827         case RTE_FLOW_CONV_OP_ATTR:
 828                 attr = src;
 829                 if (size > sizeof(*attr))
 830                         size = sizeof(*attr);
 831                 rte_memcpy(dst, attr, size);
 832                 return sizeof(*attr);
 833         case RTE_FLOW_CONV_OP_ITEM:
 834                 return rte_flow_conv_pattern(dst, size, src, 1, error);
 835         case RTE_FLOW_CONV_OP_ACTION:
 836                 return rte_flow_conv_actions(dst, size, src, 1, error);
 837         case RTE_FLOW_CONV_OP_PATTERN:
 838                 return rte_flow_conv_pattern(dst, size, src, 0, error);
 839         case RTE_FLOW_CONV_OP_ACTIONS:
 840                 return rte_flow_conv_actions(dst, size, src, 0, error);
 841         case RTE_FLOW_CONV_OP_RULE:
 842                 return rte_flow_conv_rule(dst, size, src, error);
 843         case RTE_FLOW_CONV_OP_ITEM_NAME:
 844                 return rte_flow_conv_name(0, 0, dst, size, src, error);
 845         case RTE_FLOW_CONV_OP_ACTION_NAME:
 846                 return rte_flow_conv_name(1, 0, dst, size, src, error);
 847         case RTE_FLOW_CONV_OP_ITEM_NAME_PTR:
 848                 return rte_flow_conv_name(0, 1, dst, size, src, error);
 849         case RTE_FLOW_CONV_OP_ACTION_NAME_PTR:
 850                 return rte_flow_conv_name(1, 1, dst, size, src, error);
 851         }
 852         return rte_flow_error_set
 853                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 854                  "unknown object conversion operation");
 855 }
 856
 857 /** Store a full rte_flow description. */
 858 size_t
 859 rte_flow_copy(struct rte_flow_desc *desc, size_t len,
 860               const struct rte_flow_attr *attr,
 861               const struct rte_flow_item *items,
 862               const struct rte_flow_action *actions)
 863 {
 864         /*
 865          * Overlap struct rte_flow_conv with struct rte_flow_desc in order
 866          * to convert the former to the latter without wasting space.
 867          */
 868         struct rte_flow_conv_rule *dst =
 869                 len ?
 870                 (void *)((uintptr_t)desc +
 871                          (offsetof(struct rte_flow_desc, actions) -
 872                           offsetof(struct rte_flow_conv_rule, actions))) :
 873                 NULL;
 874         size_t dst_size =
 875                 len > sizeof(*desc) - sizeof(*dst) ?
 876                 len - (sizeof(*desc) - sizeof(*dst)) :
 877                 0;
 878         struct rte_flow_conv_rule src = {
 879                 .attr_ro = NULL,
 880                 .pattern_ro = items,
 881                 .actions_ro = actions,
 882         };
 883         int ret;
 884
 885         RTE_BUILD_BUG_ON(sizeof(struct rte_flow_desc) <
 886                          sizeof(struct rte_flow_conv_rule));
 887         if (dst_size &&
 888             (&dst->pattern != &desc->items ||
 889              &dst->actions != &desc->actions ||
 890              (uintptr_t)(dst + 1) != (uintptr_t)(desc + 1))) {
 891                 rte_errno = EINVAL;
 892                 return 0;
 893         }
 894         ret = rte_flow_conv(RTE_FLOW_CONV_OP_RULE, dst, dst_size, &src, NULL);
 895         if (ret < 0)
 896                 return 0;
 897         ret += sizeof(*desc) - sizeof(*dst);
 898         rte_memcpy(desc,
 899                    (&(struct rte_flow_desc){
 900                         .size = ret,
 901                         .attr = *attr,
 902                         .items = dst_size ? dst->pattern : NULL,
 903                         .actions = dst_size ? dst->actions : NULL,
 904                    }),
 905                    len > sizeof(*desc) ? sizeof(*desc) : len);
 906         return ret;
 907 }
 908
 909 /**
 910  * Expand RSS flows into several possible flows according to the RSS hash
 911  * fields requested and the driver capabilities.
 912  */
 913 int
 914 rte_flow_expand_rss(struct rte_flow_expand_rss *buf, size_t size,
 915                     const struct rte_flow_item *pattern, uint64_t types,
 916                     const struct rte_flow_expand_node graph[],
 917                     int graph_root_index)
 918 {
 919         const int elt_n = 8;
 920         const struct rte_flow_item *item;
 921         const struct rte_flow_expand_node *node = &graph[graph_root_index];
 922         const int *next_node;
 923         const int *stack[elt_n];
 924         int stack_pos = 0;
 925         struct rte_flow_item flow_items[elt_n];
 926         unsigned int i;
 927         size_t lsize;
 928         size_t user_pattern_size = 0;
 929         void *addr = NULL;
 930
 931         lsize = offsetof(struct rte_flow_expand_rss, entry) +
 932                 elt_n * sizeof(buf->entry[0]);
 933         if (lsize <= size) {
 934                 buf->entry[0].priority = 0;
 935                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 936                 buf->entries = 0;
 937                 addr = buf->entry[0].pattern;
 938         }
 939         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 940                 const struct rte_flow_expand_node *next = NULL;
 941
 942                 for (i = 0; node->next && node->next[i]; ++i) {
 943                         next = &graph[node->next[i]];
 944                         if (next->type == item->type)
 945                                 break;
 946                 }
 947                 if (next)
 948                         node = next;
 949                 user_pattern_size += sizeof(*item);
 950         }
 951         user_pattern_size += sizeof(*item); /* Handle END item. */
 952         lsize += user_pattern_size;
 953         /* Copy the user pattern in the first entry of the buffer. */
 954         if (lsize <= size) {
 955                 rte_memcpy(addr, pattern, user_pattern_size);
 956                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 957                 buf->entries = 1;
 958         }
 959         /* Start expanding. */
 960         memset(flow_items, 0, sizeof(flow_items));
 961         user_pattern_size -= sizeof(*item);
 962         next_node = node->next;
 963         stack[stack_pos] = next_node;
 964         node = next_node ? &graph[*next_node] : NULL;
 965         while (node) {
 966                 flow_items[stack_pos].type = node->type;
 967                 if (node->rss_types & types) {
 968                         /*
 969                          * compute the number of items to copy from the
 970                          * expansion and copy it.
 971                          * When the stack_pos is 0, there are 1 element in it,
 972                          * plus the addition END item.
 973                          */
 974                         int elt = stack_pos + 2;
 975
 976                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 977                         lsize += elt * sizeof(*item) + user_pattern_size;
 978                         if (lsize <= size) {
 979                                 size_t n = elt * sizeof(*item);
 980
 981                                 buf->entry[buf->entries].priority =
 982                                         stack_pos + 1;
 983                                 buf->entry[buf->entries].pattern = addr;
 984                                 buf->entries++;
 985                                 rte_memcpy(addr, buf->entry[0].pattern,
 986                                            user_pattern_size);
 987                                 addr = (void *)(((uintptr_t)addr) +
 988                                                 user_pattern_size);
 989                                 rte_memcpy(addr, flow_items, n);
 990                                 addr = (void *)(((uintptr_t)addr) + n);
 991                         }
 992                 }
 993                 /* Go deeper. */
 994                 if (node->next) {
 995                         next_node = node->next;
 996                         if (stack_pos++ == elt_n) {
 997                                 rte_errno = E2BIG;
 998                                 return -rte_errno;
 999                         }
1000                         stack[stack_pos] = next_node;
1001                 } else if (*(next_node + 1)) {
1002                         /* Follow up with the next possibility. */
1003                         ++next_node;
1004                 } else {
1005                         /* Move to the next path. */
1006                         if (stack_pos)
1007                                 next_node = stack[--stack_pos];
1008                         next_node++;
1009                         stack[stack_pos] = next_node;
1010                 }
1011                 node = *next_node ? &graph[*next_node] : NULL;
1012         };
1013         return lsize;
1014 }