lib/librte_ether/rte_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <errno.h>
   7 #include <stddef.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_errno.h>
  13 #include <rte_branch_prediction.h>
  14 #include "rte_ethdev.h"
  15 #include "rte_flow_driver.h"
  16 #include "rte_flow.h"
  17
  18 /**
  19  * Flow elements description tables.
  20  */
  21 struct rte_flow_desc_data {
  22         const char *name;
  23         size_t size;
  24 };
  25
  26 /** Generate flow_item[] entry. */
  27 #define MK_FLOW_ITEM(t, s) \
  28         [RTE_FLOW_ITEM_TYPE_ ## t] = { \
  29                 .name = # t, \
  30                 .size = s, \
  31         }
  32
  33 /** Information about known flow pattern items. */
  34 static const struct rte_flow_desc_data rte_flow_desc_item[] = {
  35         MK_FLOW_ITEM(END, 0),
  36         MK_FLOW_ITEM(VOID, 0),
  37         MK_FLOW_ITEM(INVERT, 0),
  38         MK_FLOW_ITEM(ANY, sizeof(struct rte_flow_item_any)),
  39         MK_FLOW_ITEM(PF, 0),
  40         MK_FLOW_ITEM(VF, sizeof(struct rte_flow_item_vf)),
  41         MK_FLOW_ITEM(PHY_PORT, sizeof(struct rte_flow_item_phy_port)),
  42         MK_FLOW_ITEM(PORT_ID, sizeof(struct rte_flow_item_port_id)),
  43         MK_FLOW_ITEM(RAW, sizeof(struct rte_flow_item_raw)),
  44         MK_FLOW_ITEM(ETH, sizeof(struct rte_flow_item_eth)),
  45         MK_FLOW_ITEM(VLAN, sizeof(struct rte_flow_item_vlan)),
  46         MK_FLOW_ITEM(IPV4, sizeof(struct rte_flow_item_ipv4)),
  47         MK_FLOW_ITEM(IPV6, sizeof(struct rte_flow_item_ipv6)),
  48         MK_FLOW_ITEM(ICMP, sizeof(struct rte_flow_item_icmp)),
  49         MK_FLOW_ITEM(UDP, sizeof(struct rte_flow_item_udp)),
  50         MK_FLOW_ITEM(TCP, sizeof(struct rte_flow_item_tcp)),
  51         MK_FLOW_ITEM(SCTP, sizeof(struct rte_flow_item_sctp)),
  52         MK_FLOW_ITEM(VXLAN, sizeof(struct rte_flow_item_vxlan)),
  53         MK_FLOW_ITEM(MPLS, sizeof(struct rte_flow_item_mpls)),
  54         MK_FLOW_ITEM(GRE, sizeof(struct rte_flow_item_gre)),
  55         MK_FLOW_ITEM(E_TAG, sizeof(struct rte_flow_item_e_tag)),
  56         MK_FLOW_ITEM(NVGRE, sizeof(struct rte_flow_item_nvgre)),
  57         MK_FLOW_ITEM(GENEVE, sizeof(struct rte_flow_item_geneve)),
  58         MK_FLOW_ITEM(VXLAN_GPE, sizeof(struct rte_flow_item_vxlan_gpe)),
  59         MK_FLOW_ITEM(ARP_ETH_IPV4, sizeof(struct rte_flow_item_arp_eth_ipv4)),
  60         MK_FLOW_ITEM(IPV6_EXT, sizeof(struct rte_flow_item_ipv6_ext)),
  61         MK_FLOW_ITEM(ICMP6, sizeof(struct rte_flow_item_icmp6)),
  62         MK_FLOW_ITEM(ICMP6_ND_NS, sizeof(struct rte_flow_item_icmp6_nd_ns)),
  63         MK_FLOW_ITEM(ICMP6_ND_NA, sizeof(struct rte_flow_item_icmp6_nd_na)),
  64         MK_FLOW_ITEM(ICMP6_ND_OPT, sizeof(struct rte_flow_item_icmp6_nd_opt)),
  65         MK_FLOW_ITEM(ICMP6_ND_OPT_SLA_ETH,
  66                      sizeof(struct rte_flow_item_icmp6_nd_opt_sla_eth)),
  67         MK_FLOW_ITEM(ICMP6_ND_OPT_TLA_ETH,
  68                      sizeof(struct rte_flow_item_icmp6_nd_opt_tla_eth)),
  69 };
  70
  71 /** Generate flow_action[] entry. */
  72 #define MK_FLOW_ACTION(t, s) \
  73         [RTE_FLOW_ACTION_TYPE_ ## t] = { \
  74                 .name = # t, \
  75                 .size = s, \
  76         }
  77
  78 /** Information about known flow actions. */
  79 static const struct rte_flow_desc_data rte_flow_desc_action[] = {
  80         MK_FLOW_ACTION(END, 0),
  81         MK_FLOW_ACTION(VOID, 0),
  82         MK_FLOW_ACTION(PASSTHRU, 0),
  83         MK_FLOW_ACTION(MARK, sizeof(struct rte_flow_action_mark)),
  84         MK_FLOW_ACTION(FLAG, 0),
  85         MK_FLOW_ACTION(QUEUE, sizeof(struct rte_flow_action_queue)),
  86         MK_FLOW_ACTION(DROP, 0),
  87         MK_FLOW_ACTION(COUNT, 0),
  88         MK_FLOW_ACTION(RSS, sizeof(struct rte_flow_action_rss)),
  89         MK_FLOW_ACTION(PF, 0),
  90         MK_FLOW_ACTION(VF, sizeof(struct rte_flow_action_vf)),
  91         MK_FLOW_ACTION(PHY_PORT, sizeof(struct rte_flow_action_phy_port)),
  92         MK_FLOW_ACTION(PORT_ID, sizeof(struct rte_flow_action_port_id)),
  93         MK_FLOW_ACTION(OF_SET_MPLS_TTL,
  94                        sizeof(struct rte_flow_action_of_set_mpls_ttl)),
  95         MK_FLOW_ACTION(OF_DEC_MPLS_TTL, 0),
  96         MK_FLOW_ACTION(OF_SET_NW_TTL,
  97                        sizeof(struct rte_flow_action_of_set_nw_ttl)),
  98         MK_FLOW_ACTION(OF_DEC_NW_TTL, 0),
  99         MK_FLOW_ACTION(OF_COPY_TTL_OUT, 0),
 100         MK_FLOW_ACTION(OF_COPY_TTL_IN, 0),
 101         MK_FLOW_ACTION(OF_POP_VLAN, 0),
 102         MK_FLOW_ACTION(OF_PUSH_VLAN,
 103                        sizeof(struct rte_flow_action_of_push_vlan)),
 104         MK_FLOW_ACTION(OF_SET_VLAN_VID,
 105                        sizeof(struct rte_flow_action_of_set_vlan_vid)),
 106         MK_FLOW_ACTION(OF_SET_VLAN_PCP,
 107                        sizeof(struct rte_flow_action_of_set_vlan_pcp)),
 108         MK_FLOW_ACTION(OF_POP_MPLS,
 109                        sizeof(struct rte_flow_action_of_pop_mpls)),
 110         MK_FLOW_ACTION(OF_PUSH_MPLS,
 111                        sizeof(struct rte_flow_action_of_push_mpls)),
 112 };
 113
 114 static int
 115 flow_err(uint16_t port_id, int ret, struct rte_flow_error *error)
 116 {
 117         if (ret == 0)
 118                 return 0;
 119         if (rte_eth_dev_is_removed(port_id))
 120                 return rte_flow_error_set(error, EIO,
 121                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 122                                           NULL, rte_strerror(EIO));
 123         return ret;
 124 }
 125
 126 /* Get generic flow operations structure from a port. */
 127 const struct rte_flow_ops *
 128 rte_flow_ops_get(uint16_t port_id, struct rte_flow_error *error)
 129 {
 130         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 131         const struct rte_flow_ops *ops;
 132         int code;
 133
 134         if (unlikely(!rte_eth_dev_is_valid_port(port_id)))
 135                 code = ENODEV;
 136         else if (unlikely(!dev->dev_ops->filter_ctrl ||
 137                           dev->dev_ops->filter_ctrl(dev,
 138                                                     RTE_ETH_FILTER_GENERIC,
 139                                                     RTE_ETH_FILTER_GET,
 140                                                     &ops) ||
 141                           !ops))
 142                 code = ENOSYS;
 143         else
 144                 return ops;
 145         rte_flow_error_set(error, code, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 146                            NULL, rte_strerror(code));
 147         return NULL;
 148 }
 149
 150 /* Check whether a flow rule can be created on a given port. */
 151 int
 152 rte_flow_validate(uint16_t port_id,
 153                   const struct rte_flow_attr *attr,
 154                   const struct rte_flow_item pattern[],
 155                   const struct rte_flow_action actions[],
 156                   struct rte_flow_error *error)
 157 {
 158         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 159         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 160
 161         if (unlikely(!ops))
 162                 return -rte_errno;
 163         if (likely(!!ops->validate))
 164                 return flow_err(port_id, ops->validate(dev, attr, pattern,
 165                                                        actions, error), error);
 166         return rte_flow_error_set(error, ENOSYS,
 167                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 168                                   NULL, rte_strerror(ENOSYS));
 169 }
 170
 171 /* Create a flow rule on a given port. */
 172 struct rte_flow *
 173 rte_flow_create(uint16_t port_id,
 174                 const struct rte_flow_attr *attr,
 175                 const struct rte_flow_item pattern[],
 176                 const struct rte_flow_action actions[],
 177                 struct rte_flow_error *error)
 178 {
 179         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 180         struct rte_flow *flow;
 181         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 182
 183         if (unlikely(!ops))
 184                 return NULL;
 185         if (likely(!!ops->create)) {
 186                 flow = ops->create(dev, attr, pattern, actions, error);
 187                 if (flow == NULL)
 188                         flow_err(port_id, -rte_errno, error);
 189                 return flow;
 190         }
 191         rte_flow_error_set(error, ENOSYS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 192                            NULL, rte_strerror(ENOSYS));
 193         return NULL;
 194 }
 195
 196 /* Destroy a flow rule on a given port. */
 197 int
 198 rte_flow_destroy(uint16_t port_id,
 199                  struct rte_flow *flow,
 200                  struct rte_flow_error *error)
 201 {
 202         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 203         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 204
 205         if (unlikely(!ops))
 206                 return -rte_errno;
 207         if (likely(!!ops->destroy))
 208                 return flow_err(port_id, ops->destroy(dev, flow, error),
 209                                 error);
 210         return rte_flow_error_set(error, ENOSYS,
 211                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 212                                   NULL, rte_strerror(ENOSYS));
 213 }
 214
 215 /* Destroy all flow rules associated with a port. */
 216 int
 217 rte_flow_flush(uint16_t port_id,
 218                struct rte_flow_error *error)
 219 {
 220         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 221         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 222
 223         if (unlikely(!ops))
 224                 return -rte_errno;
 225         if (likely(!!ops->flush))
 226                 return flow_err(port_id, ops->flush(dev, error), error);
 227         return rte_flow_error_set(error, ENOSYS,
 228                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 229                                   NULL, rte_strerror(ENOSYS));
 230 }
 231
 232 /* Query an existing flow rule. */
 233 int
 234 rte_flow_query(uint16_t port_id,
 235                struct rte_flow *flow,
 236                const struct rte_flow_action *action,
 237                void *data,
 238                struct rte_flow_error *error)
 239 {
 240         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 241         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 242
 243         if (!ops)
 244                 return -rte_errno;
 245         if (likely(!!ops->query))
 246                 return flow_err(port_id, ops->query(dev, flow, action, data,
 247                                                     error), error);
 248         return rte_flow_error_set(error, ENOSYS,
 249                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 250                                   NULL, rte_strerror(ENOSYS));
 251 }
 252
 253 /* Restrict ingress traffic to the defined flow rules. */
 254 int
 255 rte_flow_isolate(uint16_t port_id,
 256                  int set,
 257                  struct rte_flow_error *error)
 258 {
 259         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 260         const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
 261
 262         if (!ops)
 263                 return -rte_errno;
 264         if (likely(!!ops->isolate))
 265                 return flow_err(port_id, ops->isolate(dev, set, error), error);
 266         return rte_flow_error_set(error, ENOSYS,
 267                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 268                                   NULL, rte_strerror(ENOSYS));
 269 }
 270
 271 /* Initialize flow error structure. */
 272 int
 273 rte_flow_error_set(struct rte_flow_error *error,
 274                    int code,
 275                    enum rte_flow_error_type type,
 276                    const void *cause,
 277                    const char *message)
 278 {
 279         if (error) {
 280                 *error = (struct rte_flow_error){
 281                         .type = type,
 282                         .cause = cause,
 283                         .message = message,
 284                 };
 285         }
 286         rte_errno = code;
 287         return -code;
 288 }
 289
 290 /** Pattern item specification types. */
 291 enum item_spec_type {
 292         ITEM_SPEC,
 293         ITEM_LAST,
 294         ITEM_MASK,
 295 };
 296
 297 /** Compute storage space needed by item specification and copy it. */
 298 static size_t
 299 flow_item_spec_copy(void *buf, const struct rte_flow_item *item,
 300                     enum item_spec_type type)
 301 {
 302         size_t size = 0;
 303         const void *item_spec =
 304                 type == ITEM_SPEC ? item->spec :
 305                 type == ITEM_LAST ? item->last :
 306                 type == ITEM_MASK ? item->mask :
 307                 NULL;
 308
 309         if (!item_spec)
 310                 goto empty;
 311         switch (item->type) {
 312                 union {
 313                         const struct rte_flow_item_raw *raw;
 314                 } src;
 315                 union {
 316                         struct rte_flow_item_raw *raw;
 317                 } dst;
 318                 size_t off;
 319
 320         case RTE_FLOW_ITEM_TYPE_RAW:
 321                 src.raw = item_spec;
 322                 dst.raw = buf;
 323                 off = RTE_ALIGN_CEIL(sizeof(struct rte_flow_item_raw),
 324                                      sizeof(*src.raw->pattern));
 325                 size = off + src.raw->length * sizeof(*src.raw->pattern);
 326                 if (dst.raw) {
 327                         memcpy(dst.raw, src.raw, sizeof(*src.raw));
 328                         dst.raw->pattern = memcpy((uint8_t *)dst.raw + off,
 329                                                   src.raw->pattern,
 330                                                   size - off);
 331                 }
 332                 break;
 333         default:
 334                 size = rte_flow_desc_item[item->type].size;
 335                 if (buf)
 336                         memcpy(buf, item_spec, size);
 337                 break;
 338         }
 339 empty:
 340         return RTE_ALIGN_CEIL(size, sizeof(double));
 341 }
 342
 343 /** Compute storage space needed by action configuration and copy it. */
 344 static size_t
 345 flow_action_conf_copy(void *buf, const struct rte_flow_action *action)
 346 {
 347         size_t size = 0;
 348
 349         if (!action->conf)
 350                 goto empty;
 351         switch (action->type) {
 352                 union {
 353                         const struct rte_flow_action_rss *rss;
 354                 } src;
 355                 union {
 356                         struct rte_flow_action_rss *rss;
 357                 } dst;
 358                 size_t off;
 359
 360         case RTE_FLOW_ACTION_TYPE_RSS:
 361                 src.rss = action->conf;
 362                 dst.rss = buf;
 363                 off = 0;
 364                 if (dst.rss)
 365                         *dst.rss = (struct rte_flow_action_rss){
 366                                 .func = src.rss->func,
 367                                 .level = src.rss->level,
 368                                 .types = src.rss->types,
 369                                 .key_len = src.rss->key_len,
 370                                 .queue_num = src.rss->queue_num,
 371                         };
 372                 off += sizeof(*src.rss);
 373                 if (src.rss->key_len) {
 374                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 375                         size = sizeof(*src.rss->key) * src.rss->key_len;
 376                         if (dst.rss)
 377                                 dst.rss->key = memcpy
 378                                         ((void *)((uintptr_t)dst.rss + off),
 379                                          src.rss->key, size);
 380                         off += size;
 381                 }
 382                 if (src.rss->queue_num) {
 383                         off = RTE_ALIGN_CEIL(off, sizeof(double));
 384                         size = sizeof(*src.rss->queue) * src.rss->queue_num;
 385                         if (dst.rss)
 386                                 dst.rss->queue = memcpy
 387                                         ((void *)((uintptr_t)dst.rss + off),
 388                                          src.rss->queue, size);
 389                         off += size;
 390                 }
 391                 size = off;
 392                 break;
 393         default:
 394                 size = rte_flow_desc_action[action->type].size;
 395                 if (buf)
 396                         memcpy(buf, action->conf, size);
 397                 break;
 398         }
 399 empty:
 400         return RTE_ALIGN_CEIL(size, sizeof(double));
 401 }
 402
 403 /** Store a full rte_flow description. */
 404 size_t
 405 rte_flow_copy(struct rte_flow_desc *desc, size_t len,
 406               const struct rte_flow_attr *attr,
 407               const struct rte_flow_item *items,
 408               const struct rte_flow_action *actions)
 409 {
 410         struct rte_flow_desc *fd = NULL;
 411         size_t tmp;
 412         size_t off1 = 0;
 413         size_t off2 = 0;
 414         size_t size = 0;
 415
 416 store:
 417         if (items) {
 418                 const struct rte_flow_item *item;
 419
 420                 item = items;
 421                 if (fd)
 422                         fd->items = (void *)&fd->data[off1];
 423                 do {
 424                         struct rte_flow_item *dst = NULL;
 425
 426                         if ((size_t)item->type >=
 427                                 RTE_DIM(rte_flow_desc_item) ||
 428                             !rte_flow_desc_item[item->type].name) {
 429                                 rte_errno = ENOTSUP;
 430                                 return 0;
 431                         }
 432                         if (fd)
 433                                 dst = memcpy(fd->data + off1, item,
 434                                              sizeof(*item));
 435                         off1 += sizeof(*item);
 436                         if (item->spec) {
 437                                 if (fd)
 438                                         dst->spec = fd->data + off2;
 439                                 off2 += flow_item_spec_copy
 440                                         (fd ? fd->data + off2 : NULL, item,
 441                                          ITEM_SPEC);
 442                         }
 443                         if (item->last) {
 444                                 if (fd)
 445                                         dst->last = fd->data + off2;
 446                                 off2 += flow_item_spec_copy
 447                                         (fd ? fd->data + off2 : NULL, item,
 448                                          ITEM_LAST);
 449                         }
 450                         if (item->mask) {
 451                                 if (fd)
 452                                         dst->mask = fd->data + off2;
 453                                 off2 += flow_item_spec_copy
 454                                         (fd ? fd->data + off2 : NULL, item,
 455                                          ITEM_MASK);
 456                         }
 457                         off2 = RTE_ALIGN_CEIL(off2, sizeof(double));
 458                 } while ((item++)->type != RTE_FLOW_ITEM_TYPE_END);
 459                 off1 = RTE_ALIGN_CEIL(off1, sizeof(double));
 460         }
 461         if (actions) {
 462                 const struct rte_flow_action *action;
 463
 464                 action = actions;
 465                 if (fd)
 466                         fd->actions = (void *)&fd->data[off1];
 467                 do {
 468                         struct rte_flow_action *dst = NULL;
 469
 470                         if ((size_t)action->type >=
 471                                 RTE_DIM(rte_flow_desc_action) ||
 472                             !rte_flow_desc_action[action->type].name) {
 473                                 rte_errno = ENOTSUP;
 474                                 return 0;
 475                         }
 476                         if (fd)
 477                                 dst = memcpy(fd->data + off1, action,
 478                                              sizeof(*action));
 479                         off1 += sizeof(*action);
 480                         if (action->conf) {
 481                                 if (fd)
 482                                         dst->conf = fd->data + off2;
 483                                 off2 += flow_action_conf_copy
 484                                         (fd ? fd->data + off2 : NULL, action);
 485                         }
 486                         off2 = RTE_ALIGN_CEIL(off2, sizeof(double));
 487                 } while ((action++)->type != RTE_FLOW_ACTION_TYPE_END);
 488         }
 489         if (fd != NULL)
 490                 return size;
 491         off1 = RTE_ALIGN_CEIL(off1, sizeof(double));
 492         tmp = RTE_ALIGN_CEIL(offsetof(struct rte_flow_desc, data),
 493                              sizeof(double));
 494         size = tmp + off1 + off2;
 495         if (size > len)
 496                 return size;
 497         fd = desc;
 498         if (fd != NULL) {
 499                 *fd = (const struct rte_flow_desc) {
 500                         .size = size,
 501                         .attr = *attr,
 502                 };
 503                 tmp -= offsetof(struct rte_flow_desc, data);
 504                 off2 = tmp + off1;
 505                 off1 = tmp;
 506                 goto store;
 507         }
 508         return 0;
 509 }