lib/librte_ethdev/rte_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stddef.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_errno.h>
  13 #include <rte_branch_prediction.h>
  14 #include <rte_string_fns.h>
  15 #include "rte_ethdev.h"
  16 #include "rte_flow_driver.h"
  17 #include "rte_flow.h"
  18
  19 /**
  20  * Flow elements description tables.
  21  */
  22 struct rte_flow_desc_data {
  23         const char *name;
  24         size_t size;
  25 };
  26
  27 /** Generate flow_item[] entry. */
  28 #define MK_FLOW_ITEM(t, s) \
  29         [RTE_FLOW_ITEM_TYPE_ ## t] = { \
  30                 .name = # t, \
  31                 .size = s, \
  32         }
  33
  34 /** Information about known flow pattern items. */
  35 static const struct rte_flow_desc_data rte_flow_desc_item[] = {
  36         MK_FLOW_ITEM(END, 0),
  37         MK_FLOW_ITEM(VOID, 0),
  38         MK_FLOW_ITEM(INVERT, 0),
  39         MK_FLOW_ITEM(ANY, sizeof(struct rte_flow_item_any)),
  40         MK_FLOW_ITEM(PF, 0),
  41         MK_FLOW_ITEM(VF, sizeof(struct rte_flow_item_vf)),
  42         MK_FLOW_ITEM(PHY_PORT, sizeof(struct rte_flow_item_phy_port)),
  43         MK_FLOW_ITEM(PORT_ID, sizeof(struct rte_flow_item_port_id)),
  44         MK_FLOW_ITEM(RAW, sizeof(struct rte_flow_item_raw)),
  45         MK_FLOW_ITEM(ETH, sizeof(struct rte_flow_item_eth)),
  46         MK_FLOW_ITEM(VLAN, sizeof(struct rte_flow_item_vlan)),
  47         MK_FLOW_ITEM(IPV4, sizeof(struct rte_flow_item_ipv4)),
  48         MK_FLOW_ITEM(IPV6, sizeof(struct rte_flow_item_ipv6)),
  49         MK_FLOW_ITEM(ICMP, sizeof(struct rte_flow_item_icmp)),
  50         MK_FLOW_ITEM(UDP, sizeof(struct rte_flow_item_udp)),
  51         MK_FLOW_ITEM(TCP, sizeof(struct rte_flow_item_tcp)),
  52         MK_FLOW_ITEM(SCTP, sizeof(struct rte_flow_item_sctp)),
  53         MK_FLOW_ITEM(VXLAN, sizeof(struct rte_flow_item_vxlan)),
  54         MK_FLOW_ITEM(E_TAG, sizeof(struct rte_flow_item_e_tag)),
  55         MK_FLOW_ITEM(NVGRE, sizeof(struct rte_flow_item_nvgre)),
  56         MK_FLOW_ITEM(MPLS, sizeof(struct rte_flow_item_mpls)),
  57         MK_FLOW_ITEM(GRE, sizeof(struct rte_flow_item_gre)),
  58         MK_FLOW_ITEM(FUZZY, sizeof(struct rte_flow_item_fuzzy)),
  59         MK_FLOW_ITEM(GTP, sizeof(struct rte_flow_item_gtp)),
  60         MK_FLOW_ITEM(GTPC, sizeof(struct rte_flow_item_gtp)),
  61         MK_FLOW_ITEM(GTPU, sizeof(struct rte_flow_item_gtp)),
  62         MK_FLOW_ITEM(ESP, sizeof(struct rte_flow_item_esp)),
  63         MK_FLOW_ITEM(GENEVE, sizeof(struct rte_flow_item_geneve)),
  64         MK_FLOW_ITEM(VXLAN_GPE, sizeof(struct rte_flow_item_vxlan_gpe)),
  65         MK_FLOW_ITEM(ARP_ETH_IPV4, sizeof(struct rte_flow_item_arp_eth_ipv4)),
  66         MK_FLOW_ITEM(IPV6_EXT, sizeof(struct rte_flow_item_ipv6_ext)),
  67         MK_FLOW_ITEM(ICMP6, sizeof(struct rte_flow_item_icmp6)),
  68         MK_FLOW_ITEM(ICMP6_ND_NS, sizeof(struct rte_flow_item_icmp6_nd_ns)),
  69         MK_FLOW_ITEM(ICMP6_ND_NA, sizeof(struct rte_flow_item_icmp6_nd_na)),
  70         MK_FLOW_ITEM(ICMP6_ND_OPT, sizeof(struct rte_flow_item_icmp6_nd_opt)),
  71         MK_FLOW_ITEM(ICMP6_ND_OPT_SLA_ETH,
  72                      sizeof(struct rte_flow_item_icmp6_nd_opt_sla_eth)),
  73         MK_FLOW_ITEM(ICMP6_ND_OPT_TLA_ETH,
  74                      sizeof(struct rte_flow_item_icmp6_nd_opt_tla_eth)),
  75         MK_FLOW_ITEM(MARK, sizeof(struct rte_flow_item_mark)),
  76 };
  77
  78 /** Generate flow_action[] entry. */
  79 #define MK_FLOW_ACTION(t, s) \
  80         [RTE_FLOW_ACTION_TYPE_ ## t] = { \
  81                 .name = # t, \
  82                 .size = s, \
  83         }
  84
  85 /** Information about known flow actions. */
  86 static const struct rte_flow_desc_data rte_flow_desc_action[] = {
  87         MK_FLOW_ACTION(END, 0),
  88         MK_FLOW_ACTION(VOID, 0),
  89         MK_FLOW_ACTION(PASSTHRU, 0),
  90         MK_FLOW_ACTION(JUMP, sizeof(struct rte_flow_action_jump)),
  91         MK_FLOW_ACTION(MARK, sizeof(struct rte_flow_action_mark)),
  92         MK_FLOW_ACTION(FLAG, 0),
  93         MK_FLOW_ACTION(QUEUE, sizeof(struct rte_flow_action_queue)),
  94         MK_FLOW_ACTION(DROP, 0),
  95         MK_FLOW_ACTION(COUNT, sizeof(struct rte_flow_action_count)),
  96         MK_FLOW_ACTION(RSS, sizeof(struct rte_flow_action_rss)),
  97         MK_FLOW_ACTION(PF, 0),
  98         MK_FLOW_ACTION(VF, sizeof(struct rte_flow_action_vf)),
  99         MK_FLOW_ACTION(PHY_PORT, sizeof(struct rte_flow_action_phy_port)),
 100         MK_FLOW_ACTION(PORT_ID, sizeof(struct rte_flow_action_port_id)),
 101         MK_FLOW_ACTION(METER, sizeof(struct rte_flow_action_meter)),
 102         MK_FLOW_ACTION(SECURITY, sizeof(struct rte_flow_action_security)),
 103         MK_FLOW_ACTION(OF_SET_MPLS_TTL,
 104                        sizeof(struct rte_flow_action_of_set_mpls_ttl)),
 105         MK_FLOW_ACTION(OF_DEC_MPLS_TTL, 0),
 106         MK_FLOW_ACTION(OF_SET_NW_TTL,
 107                        sizeof(struct rte_flow_action_of_set_nw_ttl)),
 108         MK_FLOW_ACTION(OF_DEC_NW_TTL, 0),
 109         MK_FLOW_ACTION(OF_COPY_TTL_OUT, 0),
 110         MK_FLOW_ACTION(OF_COPY_TTL_IN, 0),
 111         MK_FLOW_ACTION(OF_POP_VLAN, 0),
 112         MK_FLOW_ACTION(OF_PUSH_VLAN,
 113                        sizeof(struct rte_flow_action_of_push_vlan)),
 114         MK_FLOW_ACTION(OF_SET_VLAN_VID,
 115                        sizeof(struct rte_flow_action_of_set_vlan_vid)),
 116         MK_FLOW_ACTION(OF_SET_VLAN_PCP,
 117                        sizeof(struct rte_flow_action_of_set_vlan_pcp)),
 118         MK_FLOW_ACTION(OF_POP_MPLS,
 119                        sizeof(struct rte_flow_action_of_pop_mpls)),
 120         MK_FLOW_ACTION(OF_PUSH_MPLS,
 121                        sizeof(struct rte_flow_action_of_push_mpls)),
 122         MK_FLOW_ACTION(VXLAN_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)),
 123         MK_FLOW_ACTION(VXLAN_DECAP, 0),
 124         MK_FLOW_ACTION(NVGRE_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)),
 125         MK_FLOW_ACTION(NVGRE_DECAP, 0),
 126 };
 127
 128 static int
 129 flow_err(uint16_t port_id, int ret, struct rte_flow_error *error)
 130 {
 131         if (ret == 0)
 132                 return 0;
 133         if (rte_eth_dev_is_removed(port_id))
 134                 return rte_flow_error_set(error, EIO,
 135                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 136                                           NULL, rte_strerror(EIO));
 137         return ret;
 138 }
 139
 140 /* Get generic flow operations structure from a port. */
 141 const struct rte_flow_ops *
 142 rte_flow_ops_get(uint16_t port_id, struct rte_flow_error *error)
 143 {
 144         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 145         const struct rte_flow_ops *ops;
 146         int code;
 147
 148         if (unlikely(!rte_eth_dev_is_valid_port(port_id)))
 149                 code = ENODEV;
 150         else if (unlikely(!dev->dev_ops->filter_ctrl ||
 151                           dev->dev_ops->filter_ctrl(dev,
 152                                                     RTE_ETH_FILTER_GENERIC,
 153                                                     RTE_ETH_FILTER_GET,
 154                                                     &ops) ||
 155                           !ops))
 156                 code = ENOSYS;
 157         else
 158                 return ops;
 159         rte_flow_error_set(error, code, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 160                            NULL, rte_strerror(code));
 161         return NULL;
 162 }
 163
 164 /* Check whether a flow rule can be created on a given port. */
 165 int
 166 rte_flow_validate(uint16_t port_id,
 167                   const struct rte_flow_attr *attr,
 168                   const struct rte_flow_item pattern[],
 169                   const struct rte_flow_action actions[],
 170                   struct rte_flow_error *error)
 171 {
 172         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 173         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 174
 175         if (unlikely(!ops))
 176                 return -rte_errno;
 177         if (likely(!!ops->validate))
 178                 return flow_err(port_id, ops->validate(dev, attr, pattern,
 179                                                        actions, error), error);
 180         return rte_flow_error_set(error, ENOSYS,
 181                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 182                                   NULL, rte_strerror(ENOSYS));
 183 }
 184
 185 /* Create a flow rule on a given port. */
 186 struct rte_flow *
 187 rte_flow_create(uint16_t port_id,
 188                 const struct rte_flow_attr *attr,
 189                 const struct rte_flow_item pattern[],
 190                 const struct rte_flow_action actions[],
 191                 struct rte_flow_error *error)
 192 {
 193         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 194         struct rte_flow *flow;
 195         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 196
 197         if (unlikely(!ops))
 198                 return NULL;
 199         if (likely(!!ops->create)) {
 200                 flow = ops->create(dev, attr, pattern, actions, error);
 201                 if (flow == NULL)
 202                         flow_err(port_id, -rte_errno, error);
 203                 return flow;
 204         }
 205         rte_flow_error_set(error, ENOSYS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 206                            NULL, rte_strerror(ENOSYS));
 207         return NULL;
 208 }
 209
 210 /* Destroy a flow rule on a given port. */
 211 int
 212 rte_flow_destroy(uint16_t port_id,
 213                  struct rte_flow *flow,
 214                  struct rte_flow_error *error)
 215 {
 216         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 217         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 218
 219         if (unlikely(!ops))
 220                 return -rte_errno;
 221         if (likely(!!ops->destroy))
 222                 return flow_err(port_id, ops->destroy(dev, flow, error),
 223                                 error);
 224         return rte_flow_error_set(error, ENOSYS,
 225                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 226                                   NULL, rte_strerror(ENOSYS));
 227 }
 228
 229 /* Destroy all flow rules associated with a port. */
 230 int
 231 rte_flow_flush(uint16_t port_id,
 232                struct rte_flow_error *error)
 233 {
 234         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 235         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 236
 237         if (unlikely(!ops))
 238                 return -rte_errno;
 239         if (likely(!!ops->flush))
 240                 return flow_err(port_id, ops->flush(dev, error), error);
 241         return rte_flow_error_set(error, ENOSYS,
 242                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 243                                   NULL, rte_strerror(ENOSYS));
 244 }
 245
 246 /* Query an existing flow rule. */
 247 int
 248 rte_flow_query(uint16_t port_id,
 249                struct rte_flow *flow,
 250                const struct rte_flow_action *action,
 251                void *data,
 252                struct rte_flow_error *error)
 253 {
 254         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 255         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 256
 257         if (!ops)
 258                 return -rte_errno;
 259         if (likely(!!ops->query))
 260                 return flow_err(port_id, ops->query(dev, flow, action, data,
 261                                                     error), error);
 262         return rte_flow_error_set(error, ENOSYS,
 263                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 264                                   NULL, rte_strerror(ENOSYS));
 265 }
 266
 267 /* Restrict ingress traffic to the defined flow rules. */
 268 int
 269 rte_flow_isolate(uint16_t port_id,
 270                  int set,
 271                  struct rte_flow_error *error)
 272 {
 273         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 274         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 275
 276         if (!ops)
 277                 return -rte_errno;
 278         if (likely(!!ops->isolate))
 279                 return flow_err(port_id, ops->isolate(dev, set, error), error);
 280         return rte_flow_error_set(error, ENOSYS,
 281                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 282                                   NULL, rte_strerror(ENOSYS));
 283 }
 284
 285 /* Initialize flow error structure. */
 286 int
 287 rte_flow_error_set(struct rte_flow_error *error,
 288                    int code,
 289                    enum rte_flow_error_type type,
 290                    const void *cause,
 291                    const char *message)
 292 {
 293         if (error) {
 294                 *error = (struct rte_flow_error){
 295                         .type = type,
 296                         .cause = cause,
 297                         .message = message,
 298                 };
 299         }
 300         rte_errno = code;
 301         return -code;
 302 }
 303
 304 /** Pattern item specification types. */
 305 enum rte_flow_conv_item_spec_type {
 306         RTE_FLOW_CONV_ITEM_SPEC,
 307         RTE_FLOW_CONV_ITEM_LAST,
 308         RTE_FLOW_CONV_ITEM_MASK,
 309 };
 310
 311 /**
 312  * Copy pattern item specification.
 313  *
 314  * @param[out] buf
 315  *   Output buffer. Can be NULL if @p size is zero.
 316  * @param size
 317  *   Size of @p buf in bytes.
 318  * @param[in] item
 319  *   Pattern item to copy specification from.
 320  * @param type
 321  *   Specification selector for either @p spec, @p last or @p mask.
 322  *
 323  * @return
 324  *   Number of bytes needed to store pattern item specification regardless
 325  *   of @p size. @p buf contents are truncated to @p size if not large
 326  *   enough.
 327  */
 328 static size_t
 329 rte_flow_conv_item_spec(void *buf, const size_t size,
 330                         const struct rte_flow_item *item,
 331                         enum rte_flow_conv_item_spec_type type)
 332 {
 333         size_t off;
 334         const void *data =
 335                 type == RTE_FLOW_CONV_ITEM_SPEC ? item->spec :
 336                 type == RTE_FLOW_CONV_ITEM_LAST ? item->last :
 337                 type == RTE_FLOW_CONV_ITEM_MASK ? item->mask :
 338                 NULL;
 339
 340         switch (item->type) {
 341                 union {
 342                         const struct rte_flow_item_raw *raw;
 343                 } spec;
 344                 union {
 345                         const struct rte_flow_item_raw *raw;
 346                 } last;
 347                 union {
 348                         const struct rte_flow_item_raw *raw;
 349                 } mask;
 350                 union {
 351                         const struct rte_flow_item_raw *raw;
 352                 } src;
 353                 union {
 354                         struct rte_flow_item_raw *raw;
 355                 } dst;
 356                 size_t tmp;
 357
 358         case RTE_FLOW_ITEM_TYPE_RAW:
 359                 spec.raw = item->spec;
 360                 last.raw = item->last ? item->last : item->spec;
 361                 mask.raw = item->mask ? item->mask : &rte_flow_item_raw_mask;
 362                 src.raw = data;
 363                 dst.raw = buf;
 364                 rte_memcpy(dst.raw,
 365                            (&(struct rte_flow_item_raw){
 366                                 .relative = src.raw->relative,
 367                                 .search = src.raw->search,
 368                                 .reserved = src.raw->reserved,
 369                                 .offset = src.raw->offset,
 370                                 .limit = src.raw->limit,
 371                                 .length = src.raw->length,
 372                            }),
 373                            size > sizeof(*dst.raw) ? sizeof(*dst.raw) : size);
 374                 off = sizeof(*dst.raw);
 375                 if (type == RTE_FLOW_CONV_ITEM_SPEC ||
 376                     (type == RTE_FLOW_CONV_ITEM_MASK &&
 377                      ((spec.raw->length & mask.raw->length) >=
 378                       (last.raw->length & mask.raw->length))))
 379                         tmp = spec.raw->length & mask.raw->length;
 380                 else
 381                         tmp = last.raw->length & mask.raw->length;
 382                 if (tmp) {
 383                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.raw->pattern));
 384                         if (size >= off + tmp)
 385                                 dst.raw->pattern = rte_memcpy
 386                                         ((void *)((uintptr_t)dst.raw + off),
 387                                          src.raw->pattern, tmp);
 388                         off += tmp;
 389                 }
 390                 break;
 391         default:
 392                 off = rte_flow_desc_item[item->type].size;
 393                 rte_memcpy(buf, data, (size > off ? off : size));
 394                 break;
 395         }
 396         return off;
 397 }
 398
 399 /**
 400  * Copy action configuration.
 401  *
 402  * @param[out] buf
 403  *   Output buffer. Can be NULL if @p size is zero.
 404  * @param size
 405  *   Size of @p buf in bytes.
 406  * @param[in] action
 407  *   Action to copy configuration from.
 408  *
 409  * @return
 410  *   Number of bytes needed to store pattern item specification regardless
 411  *   of @p size. @p buf contents are truncated to @p size if not large
 412  *   enough.
 413  */
 414 static size_t
 415 rte_flow_conv_action_conf(void *buf, const size_t size,
 416                           const struct rte_flow_action *action)
 417 {
 418         size_t off;
 419
 420         switch (action->type) {
 421                 union {
 422                         const struct rte_flow_action_rss *rss;
 423                         const struct rte_flow_action_vxlan_encap *vxlan_encap;
 424                         const struct rte_flow_action_nvgre_encap *nvgre_encap;
 425                 } src;
 426                 union {
 427                         struct rte_flow_action_rss *rss;
 428                         struct rte_flow_action_vxlan_encap *vxlan_encap;
 429                         struct rte_flow_action_nvgre_encap *nvgre_encap;
 430                 } dst;
 431                 size_t tmp;
 432                 int ret;
 433
 434         case RTE_FLOW_ACTION_TYPE_RSS:
 435                 src.rss = action->conf;
 436                 dst.rss = buf;
 437                 rte_memcpy(dst.rss,
 438                            (&(struct rte_flow_action_rss){
 439                                 .func = src.rss->func,
 440                                 .level = src.rss->level,
 441                                 .types = src.rss->types,
 442                                 .key_len = src.rss->key_len,
 443                                 .queue_num = src.rss->queue_num,
 444                            }),
 445                            size > sizeof(*dst.rss) ? sizeof(*dst.rss) : size);
 446                 off = sizeof(*dst.rss);
 447                 if (src.rss->key_len) {
 448                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->key));
 449                         tmp = sizeof(*src.rss->key) * src.rss->key_len;
 450                         if (size >= off + tmp)
 451                                 dst.rss->key = rte_memcpy
 452                                         ((void *)((uintptr_t)dst.rss + off),
 453                                          src.rss->key, tmp);
 454                         off += tmp;
 455                 }
 456                 if (src.rss->queue_num) {
 457                         off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->queue));
 458                         tmp = sizeof(*src.rss->queue) * src.rss->queue_num;
 459                         if (size >= off + tmp)
 460                                 dst.rss->queue = rte_memcpy
 461                                         ((void *)((uintptr_t)dst.rss + off),
 462                                          src.rss->queue, tmp);
 463                         off += tmp;
 464                 }
 465                 break;
 466         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
 467         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
 468                 src.vxlan_encap = action->conf;
 469                 dst.vxlan_encap = buf;
 470                 RTE_BUILD_BUG_ON(sizeof(*src.vxlan_encap) !=
 471                                  sizeof(*src.nvgre_encap) ||
 472                                  offsetof(struct rte_flow_action_vxlan_encap,
 473                                           definition) !=
 474                                  offsetof(struct rte_flow_action_nvgre_encap,
 475                                           definition));
 476                 off = sizeof(*dst.vxlan_encap);
 477                 if (src.vxlan_encap->definition) {
 478                         off = RTE_ALIGN_CEIL
 479                                 (off, sizeof(*dst.vxlan_encap->definition));
 480                         ret = rte_flow_conv
 481                                 (RTE_FLOW_CONV_OP_PATTERN,
 482                                  (void *)((uintptr_t)dst.vxlan_encap + off),
 483                                  size > off ? size - off : 0,
 484                                  src.vxlan_encap->definition, NULL);
 485                         if (ret < 0)
 486                                 return 0;
 487                         if (size >= off + ret)
 488                                 dst.vxlan_encap->definition =
 489                                         (void *)((uintptr_t)dst.vxlan_encap +
 490                                                  off);
 491                         off += ret;
 492                 }
 493                 break;
 494         default:
 495                 off = rte_flow_desc_action[action->type].size;
 496                 rte_memcpy(buf, action->conf, (size > off ? off : size));
 497                 break;
 498         }
 499         return off;
 500 }
 501
 502 /**
 503  * Copy a list of pattern items.
 504  *
 505  * @param[out] dst
 506  *   Destination buffer. Can be NULL if @p size is zero.
 507  * @param size
 508  *   Size of @p dst in bytes.
 509  * @param[in] src
 510  *   Source pattern items.
 511  * @param num
 512  *   Maximum number of pattern items to process from @p src or 0 to process
 513  *   the entire list. In both cases, processing stops after
 514  *   RTE_FLOW_ITEM_TYPE_END is encountered.
 515  * @param[out] error
 516  *   Perform verbose error reporting if not NULL.
 517  *
 518  * @return
 519  *   A positive value representing the number of bytes needed to store
 520  *   pattern items regardless of @p size on success (@p buf contents are
 521  *   truncated to @p size if not large enough), a negative errno value
 522  *   otherwise and rte_errno is set.
 523  */
 524 static int
 525 rte_flow_conv_pattern(struct rte_flow_item *dst,
 526                       const size_t size,
 527                       const struct rte_flow_item *src,
 528                       unsigned int num,
 529                       struct rte_flow_error *error)
 530 {
 531         uintptr_t data = (uintptr_t)dst;
 532         size_t off;
 533         size_t ret;
 534         unsigned int i;
 535
 536         for (i = 0, off = 0; !num || i != num; ++i, ++src, ++dst) {
 537                 if ((size_t)src->type >= RTE_DIM(rte_flow_desc_item) ||
 538                     !rte_flow_desc_item[src->type].name)
 539                         return rte_flow_error_set
 540                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, src,
 541                                  "cannot convert unknown item type");
 542                 if (size >= off + sizeof(*dst))
 543                         *dst = (struct rte_flow_item){
 544                                 .type = src->type,
 545                         };
 546                 off += sizeof(*dst);
 547                 if (!src->type)
 548                         num = i + 1;
 549         }
 550         num = i;
 551         src -= num;
 552         dst -= num;
 553         do {
 554                 if (src->spec) {
 555                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 556                         ret = rte_flow_conv_item_spec
 557                                 ((void *)(data + off),
 558                                  size > off ? size - off : 0, src,
 559                                  RTE_FLOW_CONV_ITEM_SPEC);
 560                         if (size && size >= off + ret)
 561                                 dst->spec = (void *)(data + off);
 562                         off += ret;
 563
 564                 }
 565                 if (src->last) {
 566                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 567                         ret = rte_flow_conv_item_spec
 568                                 ((void *)(data + off),
 569                                  size > off ? size - off : 0, src,
 570                                  RTE_FLOW_CONV_ITEM_LAST);
 571                         if (size && size >= off + ret)
 572                                 dst->last = (void *)(data + off);
 573                         off += ret;
 574                 }
 575                 if (src->mask) {
 576                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 577                         ret = rte_flow_conv_item_spec
 578                                 ((void *)(data + off),
 579                                  size > off ? size - off : 0, src,
 580                                  RTE_FLOW_CONV_ITEM_MASK);
 581                         if (size && size >= off + ret)
 582                                 dst->mask = (void *)(data + off);
 583                         off += ret;
 584                 }
 585                 ++src;
 586                 ++dst;
 587         } while (--num);
 588         return off;
 589 }
 590
 591 /**
 592  * Copy a list of actions.
 593  *
 594  * @param[out] dst
 595  *   Destination buffer. Can be NULL if @p size is zero.
 596  * @param size
 597  *   Size of @p dst in bytes.
 598  * @param[in] src
 599  *   Source actions.
 600  * @param num
 601  *   Maximum number of actions to process from @p src or 0 to process the
 602  *   entire list. In both cases, processing stops after
 603  *   RTE_FLOW_ACTION_TYPE_END is encountered.
 604  * @param[out] error
 605  *   Perform verbose error reporting if not NULL.
 606  *
 607  * @return
 608  *   A positive value representing the number of bytes needed to store
 609  *   actions regardless of @p size on success (@p buf contents are truncated
 610  *   to @p size if not large enough), a negative errno value otherwise and
 611  *   rte_errno is set.
 612  */
 613 static int
 614 rte_flow_conv_actions(struct rte_flow_action *dst,
 615                       const size_t size,
 616                       const struct rte_flow_action *src,
 617                       unsigned int num,
 618                       struct rte_flow_error *error)
 619 {
 620         uintptr_t data = (uintptr_t)dst;
 621         size_t off;
 622         size_t ret;
 623         unsigned int i;
 624
 625         for (i = 0, off = 0; !num || i != num; ++i, ++src, ++dst) {
 626                 if ((size_t)src->type >= RTE_DIM(rte_flow_desc_action) ||
 627                     !rte_flow_desc_action[src->type].name)
 628                         return rte_flow_error_set
 629                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 630                                  src, "cannot convert unknown action type");
 631                 if (size >= off + sizeof(*dst))
 632                         *dst = (struct rte_flow_action){
 633                                 .type = src->type,
 634                         };
 635                 off += sizeof(*dst);
 636                 if (!src->type)
 637                         num = i + 1;
 638         }
 639         num = i;
 640         src -= num;
 641         dst -= num;
 642         do {
 643                 if (src->conf) {
 644                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 645                         ret = rte_flow_conv_action_conf
 646                                 ((void *)(data + off),
 647                                  size > off ? size - off : 0, src);
 648                         if (size && size >= off + ret)
 649                                 dst->conf = (void *)(data + off);
 650                         off += ret;
 651                 }
 652                 ++src;
 653                 ++dst;
 654         } while (--num);
 655         return off;
 656 }
 657
 658 /**
 659  * Copy flow rule components.
 660  *
 661  * This comprises the flow rule descriptor itself, attributes, pattern and
 662  * actions list. NULL components in @p src are skipped.
 663  *
 664  * @param[out] dst
 665  *   Destination buffer. Can be NULL if @p size is zero.
 666  * @param size
 667  *   Size of @p dst in bytes.
 668  * @param[in] src
 669  *   Source flow rule descriptor.
 670  * @param[out] error
 671  *   Perform verbose error reporting if not NULL.
 672  *
 673  * @return
 674  *   A positive value representing the number of bytes needed to store all
 675  *   components including the descriptor regardless of @p size on success
 676  *   (@p buf contents are truncated to @p size if not large enough), a
 677  *   negative errno value otherwise and rte_errno is set.
 678  */
 679 static int
 680 rte_flow_conv_rule(struct rte_flow_conv_rule *dst,
 681                    const size_t size,
 682                    const struct rte_flow_conv_rule *src,
 683                    struct rte_flow_error *error)
 684 {
 685         size_t off;
 686         int ret;
 687
 688         rte_memcpy(dst,
 689                    (&(struct rte_flow_conv_rule){
 690                         .attr = NULL,
 691                         .pattern = NULL,
 692                         .actions = NULL,
 693                    }),
 694                    size > sizeof(*dst) ? sizeof(*dst) : size);
 695         off = sizeof(*dst);
 696         if (src->attr_ro) {
 697                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 698                 if (size && size >= off + sizeof(*dst->attr))
 699                         dst->attr = rte_memcpy
 700                                 ((void *)((uintptr_t)dst + off),
 701                                  src->attr_ro, sizeof(*dst->attr));
 702                 off += sizeof(*dst->attr);
 703         }
 704         if (src->pattern_ro) {
 705                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 706                 ret = rte_flow_conv_pattern((void *)((uintptr_t)dst + off),
 707                                             size > off ? size - off : 0,
 708                                             src->pattern_ro, 0, error);
 709                 if (ret < 0)
 710                         return ret;
 711                 if (size && size >= off + (size_t)ret)
 712                         dst->pattern = (void *)((uintptr_t)dst + off);
 713                 off += ret;
 714         }
 715         if (src->actions_ro) {
 716                 off = RTE_ALIGN_CEIL(off, sizeof(double));
 717                 ret = rte_flow_conv_actions((void *)((uintptr_t)dst + off),
 718                                             size > off ? size - off : 0,
 719                                             src->actions_ro, 0, error);
 720                 if (ret < 0)
 721                         return ret;
 722                 if (size >= off + (size_t)ret)
 723                         dst->actions = (void *)((uintptr_t)dst + off);
 724                 off += ret;
 725         }
 726         return off;
 727 }
 728
 729 /**
 730  * Retrieve the name of a pattern item/action type.
 731  *
 732  * @param is_action
 733  *   Nonzero when @p src represents an action type instead of a pattern item
 734  *   type.
 735  * @param is_ptr
 736  *   Nonzero to write string address instead of contents into @p dst.
 737  * @param[out] dst
 738  *   Destination buffer. Can be NULL if @p size is zero.
 739  * @param size
 740  *   Size of @p dst in bytes.
 741  * @param[in] src
 742  *   Depending on @p is_action, source pattern item or action type cast as a
 743  *   pointer.
 744  * @param[out] error
 745  *   Perform verbose error reporting if not NULL.
 746  *
 747  * @return
 748  *   A positive value representing the number of bytes needed to store the
 749  *   name or its address regardless of @p size on success (@p buf contents
 750  *   are truncated to @p size if not large enough), a negative errno value
 751  *   otherwise and rte_errno is set.
 752  */
 753 static int
 754 rte_flow_conv_name(int is_action,
 755                    int is_ptr,
 756                    char *dst,
 757                    const size_t size,
 758                    const void *src,
 759                    struct rte_flow_error *error)
 760 {
 761         struct desc_info {
 762                 const struct rte_flow_desc_data *data;
 763                 size_t num;
 764         };
 765         static const struct desc_info info_rep[2] = {
 766                 { rte_flow_desc_item, RTE_DIM(rte_flow_desc_item), },
 767                 { rte_flow_desc_action, RTE_DIM(rte_flow_desc_action), },
 768         };
 769         const struct desc_info *const info = &info_rep[!!is_action];
 770         unsigned int type = (uintptr_t)src;
 771
 772         if (type >= info->num)
 773                 return rte_flow_error_set
 774                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 775                          "unknown object type to retrieve the name of");
 776         if (!is_ptr)
 777                 return strlcpy(dst, info->data[type].name, size);
 778         if (size >= sizeof(const char **))
 779                 *((const char **)dst) = info->data[type].name;
 780         return sizeof(const char **);
 781 }
 782
 783 /** Helper function to convert flow API objects. */
 784 int
 785 rte_flow_conv(enum rte_flow_conv_op op,
 786               void *dst,
 787               size_t size,
 788               const void *src,
 789               struct rte_flow_error *error)
 790 {
 791         switch (op) {
 792                 const struct rte_flow_attr *attr;
 793
 794         case RTE_FLOW_CONV_OP_NONE:
 795                 return 0;
 796         case RTE_FLOW_CONV_OP_ATTR:
 797                 attr = src;
 798                 if (size > sizeof(*attr))
 799                         size = sizeof(*attr);
 800                 rte_memcpy(dst, attr, size);
 801                 return sizeof(*attr);
 802         case RTE_FLOW_CONV_OP_ITEM:
 803                 return rte_flow_conv_pattern(dst, size, src, 1, error);
 804         case RTE_FLOW_CONV_OP_ACTION:
 805                 return rte_flow_conv_actions(dst, size, src, 1, error);
 806         case RTE_FLOW_CONV_OP_PATTERN:
 807                 return rte_flow_conv_pattern(dst, size, src, 0, error);
 808         case RTE_FLOW_CONV_OP_ACTIONS:
 809                 return rte_flow_conv_actions(dst, size, src, 0, error);
 810         case RTE_FLOW_CONV_OP_RULE:
 811                 return rte_flow_conv_rule(dst, size, src, error);
 812         case RTE_FLOW_CONV_OP_ITEM_NAME:
 813                 return rte_flow_conv_name(0, 0, dst, size, src, error);
 814         case RTE_FLOW_CONV_OP_ACTION_NAME:
 815                 return rte_flow_conv_name(1, 0, dst, size, src, error);
 816         case RTE_FLOW_CONV_OP_ITEM_NAME_PTR:
 817                 return rte_flow_conv_name(0, 1, dst, size, src, error);
 818         case RTE_FLOW_CONV_OP_ACTION_NAME_PTR:
 819                 return rte_flow_conv_name(1, 1, dst, size, src, error);
 820         }
 821         return rte_flow_error_set
 822                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 823                  "unknown object conversion operation");
 824 }
 825
 826 /** Store a full rte_flow description. */
 827 size_t
 828 rte_flow_copy(struct rte_flow_desc *desc, size_t len,
 829               const struct rte_flow_attr *attr,
 830               const struct rte_flow_item *items,
 831               const struct rte_flow_action *actions)
 832 {
 833         /*
 834          * Overlap struct rte_flow_conv with struct rte_flow_desc in order
 835          * to convert the former to the latter without wasting space.
 836          */
 837         struct rte_flow_conv_rule *dst =
 838                 len ?
 839                 (void *)((uintptr_t)desc +
 840                          (offsetof(struct rte_flow_desc, actions) -
 841                           offsetof(struct rte_flow_conv_rule, actions))) :
 842                 NULL;
 843         size_t dst_size =
 844                 len > sizeof(*desc) - sizeof(*dst) ?
 845                 len - (sizeof(*desc) - sizeof(*dst)) :
 846                 0;
 847         struct rte_flow_conv_rule src = {
 848                 .attr_ro = NULL,
 849                 .pattern_ro = items,
 850                 .actions_ro = actions,
 851         };
 852         int ret;
 853
 854         RTE_BUILD_BUG_ON(sizeof(struct rte_flow_desc) <
 855                          sizeof(struct rte_flow_conv_rule));
 856         if (dst_size &&
 857             (&dst->pattern != &desc->items ||
 858              &dst->actions != &desc->actions ||
 859              (uintptr_t)(dst + 1) != (uintptr_t)(desc + 1))) {
 860                 rte_errno = EINVAL;
 861                 return 0;
 862         }
 863         ret = rte_flow_conv(RTE_FLOW_CONV_OP_RULE, dst, dst_size, &src, NULL);
 864         if (ret < 0)
 865                 return 0;
 866         ret += sizeof(*desc) - sizeof(*dst);
 867         rte_memcpy(desc,
 868                    (&(struct rte_flow_desc){
 869                         .size = ret,
 870                         .attr = *attr,
 871                         .items = dst_size ? dst->pattern : NULL,
 872                         .actions = dst_size ? dst->actions : NULL,
 873                    }),
 874                    len > sizeof(*desc) ? sizeof(*desc) : len);
 875         return ret;
 876 }
 877
 878 /**
 879  * Expand RSS flows into several possible flows according to the RSS hash
 880  * fields requested and the driver capabilities.
 881  */
 882 int __rte_experimental
 883 rte_flow_expand_rss(struct rte_flow_expand_rss *buf, size_t size,
 884                     const struct rte_flow_item *pattern, uint64_t types,
 885                     const struct rte_flow_expand_node graph[],
 886                     int graph_root_index)
 887 {
 888         const int elt_n = 8;
 889         const struct rte_flow_item *item;
 890         const struct rte_flow_expand_node *node = &graph[graph_root_index];
 891         const int *next_node;
 892         const int *stack[elt_n];
 893         int stack_pos = 0;
 894         struct rte_flow_item flow_items[elt_n];
 895         unsigned int i;
 896         size_t lsize;
 897         size_t user_pattern_size = 0;
 898         void *addr = NULL;
 899
 900         lsize = offsetof(struct rte_flow_expand_rss, entry) +
 901                 elt_n * sizeof(buf->entry[0]);
 902         if (lsize <= size) {
 903                 buf->entry[0].priority = 0;
 904                 buf->entry[0].pattern = (void *)&buf->entry[elt_n];
 905                 buf->entries = 0;
 906                 addr = buf->entry[0].pattern;
 907         }
 908         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 909                 const struct rte_flow_expand_node *next = NULL;
 910
 911                 for (i = 0; node->next && node->next[i]; ++i) {
 912                         next = &graph[node->next[i]];
 913                         if (next->type == item->type)
 914                                 break;
 915                 }
 916                 if (next)
 917                         node = next;
 918                 user_pattern_size += sizeof(*item);
 919         }
 920         user_pattern_size += sizeof(*item); /* Handle END item. */
 921         lsize += user_pattern_size;
 922         /* Copy the user pattern in the first entry of the buffer. */
 923         if (lsize <= size) {
 924                 rte_memcpy(addr, pattern, user_pattern_size);
 925                 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 926                 buf->entries = 1;
 927         }
 928         /* Start expanding. */
 929         memset(flow_items, 0, sizeof(flow_items));
 930         user_pattern_size -= sizeof(*item);
 931         next_node = node->next;
 932         stack[stack_pos] = next_node;
 933         node = next_node ? &graph[*next_node] : NULL;
 934         while (node) {
 935                 flow_items[stack_pos].type = node->type;
 936                 if (node->rss_types & types) {
 937                         /*
 938                          * compute the number of items to copy from the
 939                          * expansion and copy it.
 940                          * When the stack_pos is 0, there are 1 element in it,
 941                          * plus the addition END item.
 942                          */
 943                         int elt = stack_pos + 2;
 944
 945                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 946                         lsize += elt * sizeof(*item) + user_pattern_size;
 947                         if (lsize <= size) {
 948                                 size_t n = elt * sizeof(*item);
 949
 950                                 buf->entry[buf->entries].priority =
 951                                         stack_pos + 1;
 952                                 buf->entry[buf->entries].pattern = addr;
 953                                 buf->entries++;
 954                                 rte_memcpy(addr, buf->entry[0].pattern,
 955                                            user_pattern_size);
 956                                 addr = (void *)(((uintptr_t)addr) +
 957                                                 user_pattern_size);
 958                                 rte_memcpy(addr, flow_items, n);
 959                                 addr = (void *)(((uintptr_t)addr) + n);
 960                         }
 961                 }
 962                 /* Go deeper. */
 963                 if (node->next) {
 964                         next_node = node->next;
 965                         if (stack_pos++ == elt_n) {
 966                                 rte_errno = E2BIG;
 967                                 return -rte_errno;
 968                         }
 969                         stack[stack_pos] = next_node;
 970                 } else if (*(next_node + 1)) {
 971                         /* Follow up with the next possibility. */
 972                         ++next_node;
 973                 } else {
 974                         /* Move to the next path. */
 975                         if (stack_pos)
 976                                 next_node = stack[--stack_pos];
 977                         next_node++;
 978                         stack[stack_pos] = next_node;
 979                 }
 980                 node = *next_node ? &graph[*next_node] : NULL;
 981         };
 982         return lsize;
 983 }