drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <assert.h>
  35
  36 #include <rte_flow.h>
  37 #include <rte_flow_driver.h>
  38 #include <rte_malloc.h>
  39
  40 /* Generated configuration header. */
  41 #include "mlx4_autoconf.h"
  42
  43 /* PMD headers. */
  44 #include "mlx4.h"
  45 #include "mlx4_flow.h"
  46
  47 /** Static initializer for items. */
  48 #define ITEMS(...) \
  49         (const enum rte_flow_item_type []){ \
  50                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  51         }
  52
  53 /** Structure to generate a simple graph of layers supported by the NIC. */
  54 struct mlx4_flow_items {
  55         /** List of possible actions for these items. */
  56         const enum rte_flow_action_type *const actions;
  57         /** Bit-masks corresponding to the possibilities for the item. */
  58         const void *mask;
  59         /**
  60          * Default bit-masks to use when item->mask is not provided. When
  61          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  62          * used instead.
  63          */
  64         const void *default_mask;
  65         /** Bit-masks size in bytes. */
  66         const unsigned int mask_sz;
  67         /**
  68          * Check support for a given item.
  69          *
  70          * @param item[in]
  71          *   Item specification.
  72          * @param mask[in]
  73          *   Bit-masks covering supported fields to compare with spec,
  74          *   last and mask in
  75          *   \item.
  76          * @param size
  77          *   Bit-Mask size in bytes.
  78          *
  79          * @return
  80          *   0 on success, negative value otherwise.
  81          */
  82         int (*validate)(const struct rte_flow_item *item,
  83                         const uint8_t *mask, unsigned int size);
  84         /**
  85          * Conversion function from rte_flow to NIC specific flow.
  86          *
  87          * @param item
  88          *   rte_flow item to convert.
  89          * @param default_mask
  90          *   Default bit-masks to use when item->mask is not provided.
  91          * @param data
  92          *   Internal structure to store the conversion.
  93          *
  94          * @return
  95          *   0 on success, negative value otherwise.
  96          */
  97         int (*convert)(const struct rte_flow_item *item,
  98                        const void *default_mask,
  99                        void *data);
 100         /** Size in bytes of the destination structure. */
 101         const unsigned int dst_sz;
 102         /** List of possible following items.  */
 103         const enum rte_flow_item_type *const items;
 104 };
 105
 106 /** Valid action for this PMD. */
 107 static const enum rte_flow_action_type valid_actions[] = {
 108         RTE_FLOW_ACTION_TYPE_DROP,
 109         RTE_FLOW_ACTION_TYPE_QUEUE,
 110         RTE_FLOW_ACTION_TYPE_END,
 111 };
 112
 113 /**
 114  * Convert Ethernet item to Verbs specification.
 115  *
 116  * @param item[in]
 117  *   Item specification.
 118  * @param default_mask[in]
 119  *   Default bit-masks to use when item->mask is not provided.
 120  * @param data[in, out]
 121  *   User structure.
 122  */
 123 static int
 124 mlx4_flow_create_eth(const struct rte_flow_item *item,
 125                      const void *default_mask,
 126                      void *data)
 127 {
 128         const struct rte_flow_item_eth *spec = item->spec;
 129         const struct rte_flow_item_eth *mask = item->mask;
 130         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 131         struct ibv_flow_spec_eth *eth;
 132         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 133         unsigned int i;
 134
 135         ++flow->ibv_attr->num_of_specs;
 136         flow->ibv_attr->priority = 2;
 137         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 138         *eth = (struct ibv_flow_spec_eth) {
 139                 .type = IBV_FLOW_SPEC_ETH,
 140                 .size = eth_size,
 141         };
 142         if (!spec) {
 143                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 144                 return 0;
 145         }
 146         if (!mask)
 147                 mask = default_mask;
 148         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 149         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 150         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 151         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 152         /* Remove unwanted bits from values. */
 153         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 154                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 155                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 156         }
 157         return 0;
 158 }
 159
 160 /**
 161  * Convert VLAN item to Verbs specification.
 162  *
 163  * @param item[in]
 164  *   Item specification.
 165  * @param default_mask[in]
 166  *   Default bit-masks to use when item->mask is not provided.
 167  * @param data[in, out]
 168  *   User structure.
 169  */
 170 static int
 171 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 172                       const void *default_mask,
 173                       void *data)
 174 {
 175         const struct rte_flow_item_vlan *spec = item->spec;
 176         const struct rte_flow_item_vlan *mask = item->mask;
 177         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 178         struct ibv_flow_spec_eth *eth;
 179         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 180
 181         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 182         if (!spec)
 183                 return 0;
 184         if (!mask)
 185                 mask = default_mask;
 186         eth->val.vlan_tag = spec->tci;
 187         eth->mask.vlan_tag = mask->tci;
 188         eth->val.vlan_tag &= eth->mask.vlan_tag;
 189         return 0;
 190 }
 191
 192 /**
 193  * Convert IPv4 item to Verbs specification.
 194  *
 195  * @param item[in]
 196  *   Item specification.
 197  * @param default_mask[in]
 198  *   Default bit-masks to use when item->mask is not provided.
 199  * @param data[in, out]
 200  *   User structure.
 201  */
 202 static int
 203 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 204                       const void *default_mask,
 205                       void *data)
 206 {
 207         const struct rte_flow_item_ipv4 *spec = item->spec;
 208         const struct rte_flow_item_ipv4 *mask = item->mask;
 209         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 210         struct ibv_flow_spec_ipv4 *ipv4;
 211         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 212
 213         ++flow->ibv_attr->num_of_specs;
 214         flow->ibv_attr->priority = 1;
 215         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 216         *ipv4 = (struct ibv_flow_spec_ipv4) {
 217                 .type = IBV_FLOW_SPEC_IPV4,
 218                 .size = ipv4_size,
 219         };
 220         if (!spec)
 221                 return 0;
 222         ipv4->val = (struct ibv_flow_ipv4_filter) {
 223                 .src_ip = spec->hdr.src_addr,
 224                 .dst_ip = spec->hdr.dst_addr,
 225         };
 226         if (!mask)
 227                 mask = default_mask;
 228         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 229                 .src_ip = mask->hdr.src_addr,
 230                 .dst_ip = mask->hdr.dst_addr,
 231         };
 232         /* Remove unwanted bits from values. */
 233         ipv4->val.src_ip &= ipv4->mask.src_ip;
 234         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 235         return 0;
 236 }
 237
 238 /**
 239  * Convert UDP item to Verbs specification.
 240  *
 241  * @param item[in]
 242  *   Item specification.
 243  * @param default_mask[in]
 244  *   Default bit-masks to use when item->mask is not provided.
 245  * @param data[in, out]
 246  *   User structure.
 247  */
 248 static int
 249 mlx4_flow_create_udp(const struct rte_flow_item *item,
 250                      const void *default_mask,
 251                      void *data)
 252 {
 253         const struct rte_flow_item_udp *spec = item->spec;
 254         const struct rte_flow_item_udp *mask = item->mask;
 255         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 256         struct ibv_flow_spec_tcp_udp *udp;
 257         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 258
 259         ++flow->ibv_attr->num_of_specs;
 260         flow->ibv_attr->priority = 0;
 261         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 262         *udp = (struct ibv_flow_spec_tcp_udp) {
 263                 .type = IBV_FLOW_SPEC_UDP,
 264                 .size = udp_size,
 265         };
 266         if (!spec)
 267                 return 0;
 268         udp->val.dst_port = spec->hdr.dst_port;
 269         udp->val.src_port = spec->hdr.src_port;
 270         if (!mask)
 271                 mask = default_mask;
 272         udp->mask.dst_port = mask->hdr.dst_port;
 273         udp->mask.src_port = mask->hdr.src_port;
 274         /* Remove unwanted bits from values. */
 275         udp->val.src_port &= udp->mask.src_port;
 276         udp->val.dst_port &= udp->mask.dst_port;
 277         return 0;
 278 }
 279
 280 /**
 281  * Convert TCP item to Verbs specification.
 282  *
 283  * @param item[in]
 284  *   Item specification.
 285  * @param default_mask[in]
 286  *   Default bit-masks to use when item->mask is not provided.
 287  * @param data[in, out]
 288  *   User structure.
 289  */
 290 static int
 291 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 292                      const void *default_mask,
 293                      void *data)
 294 {
 295         const struct rte_flow_item_tcp *spec = item->spec;
 296         const struct rte_flow_item_tcp *mask = item->mask;
 297         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 298         struct ibv_flow_spec_tcp_udp *tcp;
 299         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 300
 301         ++flow->ibv_attr->num_of_specs;
 302         flow->ibv_attr->priority = 0;
 303         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 304         *tcp = (struct ibv_flow_spec_tcp_udp) {
 305                 .type = IBV_FLOW_SPEC_TCP,
 306                 .size = tcp_size,
 307         };
 308         if (!spec)
 309                 return 0;
 310         tcp->val.dst_port = spec->hdr.dst_port;
 311         tcp->val.src_port = spec->hdr.src_port;
 312         if (!mask)
 313                 mask = default_mask;
 314         tcp->mask.dst_port = mask->hdr.dst_port;
 315         tcp->mask.src_port = mask->hdr.src_port;
 316         /* Remove unwanted bits from values. */
 317         tcp->val.src_port &= tcp->mask.src_port;
 318         tcp->val.dst_port &= tcp->mask.dst_port;
 319         return 0;
 320 }
 321
 322 /**
 323  * Check support for a given item.
 324  *
 325  * @param item[in]
 326  *   Item specification.
 327  * @param mask[in]
 328  *   Bit-masks covering supported fields to compare with spec, last and mask in
 329  *   \item.
 330  * @param size
 331  *   Bit-Mask size in bytes.
 332  *
 333  * @return
 334  *   0 on success, negative value otherwise.
 335  */
 336 static int
 337 mlx4_flow_item_validate(const struct rte_flow_item *item,
 338                         const uint8_t *mask, unsigned int size)
 339 {
 340         int ret = 0;
 341
 342         if (!item->spec && (item->mask || item->last))
 343                 return -1;
 344         if (item->spec && !item->mask) {
 345                 unsigned int i;
 346                 const uint8_t *spec = item->spec;
 347
 348                 for (i = 0; i < size; ++i)
 349                         if ((spec[i] | mask[i]) != mask[i])
 350                                 return -1;
 351         }
 352         if (item->last && !item->mask) {
 353                 unsigned int i;
 354                 const uint8_t *spec = item->last;
 355
 356                 for (i = 0; i < size; ++i)
 357                         if ((spec[i] | mask[i]) != mask[i])
 358                                 return -1;
 359         }
 360         if (item->spec && item->last) {
 361                 uint8_t spec[size];
 362                 uint8_t last[size];
 363                 const uint8_t *apply = mask;
 364                 unsigned int i;
 365
 366                 if (item->mask)
 367                         apply = item->mask;
 368                 for (i = 0; i < size; ++i) {
 369                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 370                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 371                 }
 372                 ret = memcmp(spec, last, size);
 373         }
 374         return ret;
 375 }
 376
 377 static int
 378 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 379                        const uint8_t *mask, unsigned int size)
 380 {
 381         if (item->mask) {
 382                 const struct rte_flow_item_eth *mask = item->mask;
 383
 384                 if (mask->dst.addr_bytes[0] != 0xff ||
 385                                 mask->dst.addr_bytes[1] != 0xff ||
 386                                 mask->dst.addr_bytes[2] != 0xff ||
 387                                 mask->dst.addr_bytes[3] != 0xff ||
 388                                 mask->dst.addr_bytes[4] != 0xff ||
 389                                 mask->dst.addr_bytes[5] != 0xff)
 390                         return -1;
 391         }
 392         return mlx4_flow_item_validate(item, mask, size);
 393 }
 394
 395 static int
 396 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 397                         const uint8_t *mask, unsigned int size)
 398 {
 399         if (item->mask) {
 400                 const struct rte_flow_item_vlan *mask = item->mask;
 401
 402                 if (mask->tci != 0 &&
 403                     ntohs(mask->tci) != 0x0fff)
 404                         return -1;
 405         }
 406         return mlx4_flow_item_validate(item, mask, size);
 407 }
 408
 409 static int
 410 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 411                         const uint8_t *mask, unsigned int size)
 412 {
 413         if (item->mask) {
 414                 const struct rte_flow_item_ipv4 *mask = item->mask;
 415
 416                 if (mask->hdr.src_addr != 0 &&
 417                     mask->hdr.src_addr != 0xffffffff)
 418                         return -1;
 419                 if (mask->hdr.dst_addr != 0 &&
 420                     mask->hdr.dst_addr != 0xffffffff)
 421                         return -1;
 422         }
 423         return mlx4_flow_item_validate(item, mask, size);
 424 }
 425
 426 static int
 427 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 428                        const uint8_t *mask, unsigned int size)
 429 {
 430         if (item->mask) {
 431                 const struct rte_flow_item_udp *mask = item->mask;
 432
 433                 if (mask->hdr.src_port != 0 &&
 434                     mask->hdr.src_port != 0xffff)
 435                         return -1;
 436                 if (mask->hdr.dst_port != 0 &&
 437                     mask->hdr.dst_port != 0xffff)
 438                         return -1;
 439         }
 440         return mlx4_flow_item_validate(item, mask, size);
 441 }
 442
 443 static int
 444 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 445                        const uint8_t *mask, unsigned int size)
 446 {
 447         if (item->mask) {
 448                 const struct rte_flow_item_tcp *mask = item->mask;
 449
 450                 if (mask->hdr.src_port != 0 &&
 451                     mask->hdr.src_port != 0xffff)
 452                         return -1;
 453                 if (mask->hdr.dst_port != 0 &&
 454                     mask->hdr.dst_port != 0xffff)
 455                         return -1;
 456         }
 457         return mlx4_flow_item_validate(item, mask, size);
 458 }
 459
 460 /** Graph of supported items and associated actions. */
 461 static const struct mlx4_flow_items mlx4_flow_items[] = {
 462         [RTE_FLOW_ITEM_TYPE_END] = {
 463                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 464         },
 465         [RTE_FLOW_ITEM_TYPE_ETH] = {
 466                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 467                                RTE_FLOW_ITEM_TYPE_IPV4),
 468                 .actions = valid_actions,
 469                 .mask = &(const struct rte_flow_item_eth){
 470                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 471                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 472                 },
 473                 .default_mask = &rte_flow_item_eth_mask,
 474                 .mask_sz = sizeof(struct rte_flow_item_eth),
 475                 .validate = mlx4_flow_validate_eth,
 476                 .convert = mlx4_flow_create_eth,
 477                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 478         },
 479         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 480                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 481                 .actions = valid_actions,
 482                 .mask = &(const struct rte_flow_item_vlan){
 483                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 484 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 485                         .tci = 0x0fff,
 486 #else
 487                         .tci = 0xff0f,
 488 #endif
 489                 },
 490                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 491                 .validate = mlx4_flow_validate_vlan,
 492                 .convert = mlx4_flow_create_vlan,
 493                 .dst_sz = 0,
 494         },
 495         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 496                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 497                                RTE_FLOW_ITEM_TYPE_TCP),
 498                 .actions = valid_actions,
 499                 .mask = &(const struct rte_flow_item_ipv4){
 500                         .hdr = {
 501                                 .src_addr = -1,
 502                                 .dst_addr = -1,
 503                         },
 504                 },
 505                 .default_mask = &rte_flow_item_ipv4_mask,
 506                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 507                 .validate = mlx4_flow_validate_ipv4,
 508                 .convert = mlx4_flow_create_ipv4,
 509                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 510         },
 511         [RTE_FLOW_ITEM_TYPE_UDP] = {
 512                 .actions = valid_actions,
 513                 .mask = &(const struct rte_flow_item_udp){
 514                         .hdr = {
 515                                 .src_port = -1,
 516                                 .dst_port = -1,
 517                         },
 518                 },
 519                 .default_mask = &rte_flow_item_udp_mask,
 520                 .mask_sz = sizeof(struct rte_flow_item_udp),
 521                 .validate = mlx4_flow_validate_udp,
 522                 .convert = mlx4_flow_create_udp,
 523                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 524         },
 525         [RTE_FLOW_ITEM_TYPE_TCP] = {
 526                 .actions = valid_actions,
 527                 .mask = &(const struct rte_flow_item_tcp){
 528                         .hdr = {
 529                                 .src_port = -1,
 530                                 .dst_port = -1,
 531                         },
 532                 },
 533                 .default_mask = &rte_flow_item_tcp_mask,
 534                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 535                 .validate = mlx4_flow_validate_tcp,
 536                 .convert = mlx4_flow_create_tcp,
 537                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 538         },
 539 };
 540
 541 /**
 542  * Validate a flow supported by the NIC.
 543  *
 544  * @param priv
 545  *   Pointer to private structure.
 546  * @param[in] attr
 547  *   Flow rule attributes.
 548  * @param[in] items
 549  *   Pattern specification (list terminated by the END pattern item).
 550  * @param[in] actions
 551  *   Associated actions (list terminated by the END action).
 552  * @param[out] error
 553  *   Perform verbose error reporting if not NULL.
 554  * @param[in, out] flow
 555  *   Flow structure to update.
 556  *
 557  * @return
 558  *   0 on success, a negative errno value otherwise and rte_errno is set.
 559  */
 560 static int
 561 priv_flow_validate(struct priv *priv,
 562                    const struct rte_flow_attr *attr,
 563                    const struct rte_flow_item items[],
 564                    const struct rte_flow_action actions[],
 565                    struct rte_flow_error *error,
 566                    struct mlx4_flow *flow)
 567 {
 568         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 569         struct mlx4_flow_action action = {
 570                 .queue = 0,
 571                 .drop = 0,
 572         };
 573
 574         (void)priv;
 575         if (attr->group) {
 576                 rte_flow_error_set(error, ENOTSUP,
 577                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 578                                    NULL,
 579                                    "groups are not supported");
 580                 return -rte_errno;
 581         }
 582         if (attr->priority) {
 583                 rte_flow_error_set(error, ENOTSUP,
 584                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 585                                    NULL,
 586                                    "priorities are not supported");
 587                 return -rte_errno;
 588         }
 589         if (attr->egress) {
 590                 rte_flow_error_set(error, ENOTSUP,
 591                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 592                                    NULL,
 593                                    "egress is not supported");
 594                 return -rte_errno;
 595         }
 596         if (!attr->ingress) {
 597                 rte_flow_error_set(error, ENOTSUP,
 598                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 599                                    NULL,
 600                                    "only ingress is supported");
 601                 return -rte_errno;
 602         }
 603         /* Go over items list. */
 604         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 605                 const struct mlx4_flow_items *token = NULL;
 606                 unsigned int i;
 607                 int err;
 608
 609                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 610                         continue;
 611                 /*
 612                  * The nic can support patterns with NULL eth spec only
 613                  * if eth is a single item in a rule.
 614                  */
 615                 if (!items->spec &&
 616                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 617                         const struct rte_flow_item *next = items + 1;
 618
 619                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 620                                 rte_flow_error_set(error, ENOTSUP,
 621                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 622                                                    items,
 623                                                    "the rule requires"
 624                                                    " an Ethernet spec");
 625                                 return -rte_errno;
 626                         }
 627                 }
 628                 for (i = 0;
 629                      cur_item->items &&
 630                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 631                      ++i) {
 632                         if (cur_item->items[i] == items->type) {
 633                                 token = &mlx4_flow_items[items->type];
 634                                 break;
 635                         }
 636                 }
 637                 if (!token)
 638                         goto exit_item_not_supported;
 639                 cur_item = token;
 640                 err = cur_item->validate(items,
 641                                         (const uint8_t *)cur_item->mask,
 642                                          cur_item->mask_sz);
 643                 if (err)
 644                         goto exit_item_not_supported;
 645                 if (flow->ibv_attr && cur_item->convert) {
 646                         err = cur_item->convert(items,
 647                                                 (cur_item->default_mask ?
 648                                                  cur_item->default_mask :
 649                                                  cur_item->mask),
 650                                                  flow);
 651                         if (err)
 652                                 goto exit_item_not_supported;
 653                 }
 654                 flow->offset += cur_item->dst_sz;
 655         }
 656         /* Go over actions list */
 657         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 658                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 659                         continue;
 660                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 661                         action.drop = 1;
 662                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 663                         const struct rte_flow_action_queue *queue =
 664                                 (const struct rte_flow_action_queue *)
 665                                 actions->conf;
 666
 667                         if (!queue || (queue->index > (priv->rxqs_n - 1)))
 668                                 goto exit_action_not_supported;
 669                         action.queue = 1;
 670                 } else {
 671                         goto exit_action_not_supported;
 672                 }
 673         }
 674         if (!action.queue && !action.drop) {
 675                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 676                                    NULL, "no valid action");
 677                 return -rte_errno;
 678         }
 679         return 0;
 680 exit_item_not_supported:
 681         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 682                            items, "item not supported");
 683         return -rte_errno;
 684 exit_action_not_supported:
 685         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 686                            actions, "action not supported");
 687         return -rte_errno;
 688 }
 689
 690 /**
 691  * Validate a flow supported by the NIC.
 692  *
 693  * @see rte_flow_validate()
 694  * @see rte_flow_ops
 695  */
 696 int
 697 mlx4_flow_validate(struct rte_eth_dev *dev,
 698                    const struct rte_flow_attr *attr,
 699                    const struct rte_flow_item items[],
 700                    const struct rte_flow_action actions[],
 701                    struct rte_flow_error *error)
 702 {
 703         struct priv *priv = dev->data->dev_private;
 704         int ret;
 705         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 706
 707         priv_lock(priv);
 708         ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
 709         priv_unlock(priv);
 710         return ret;
 711 }
 712
 713 /**
 714  * Complete flow rule creation.
 715  *
 716  * @param priv
 717  *   Pointer to private structure.
 718  * @param ibv_attr
 719  *   Verbs flow attributes.
 720  * @param action
 721  *   Target action structure.
 722  * @param[out] error
 723  *   Perform verbose error reporting if not NULL.
 724  *
 725  * @return
 726  *   A flow if the rule could be created.
 727  */
 728 static struct rte_flow *
 729 priv_flow_create_action_queue(struct priv *priv,
 730                               struct ibv_flow_attr *ibv_attr,
 731                               struct mlx4_flow_action *action,
 732                               struct rte_flow_error *error)
 733 {
 734         struct rxq *rxq;
 735         struct ibv_qp *qp;
 736         struct rte_flow *rte_flow;
 737
 738         assert(priv->pd);
 739         assert(priv->ctx);
 740         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 741         if (!rte_flow) {
 742                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 743                                    NULL, "cannot allocate flow memory");
 744                 return NULL;
 745         }
 746         rxq = (*priv->rxqs)[action->queue_id];
 747         if (action->drop) {
 748                 rte_flow->cq =
 749                         ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
 750                                           &(struct ibv_exp_cq_init_attr){
 751                                                   .comp_mask = 0,
 752                                           });
 753                 if (!rte_flow->cq) {
 754                         rte_flow_error_set(error, ENOMEM,
 755                                            RTE_FLOW_ERROR_TYPE_HANDLE,
 756                                            NULL, "cannot allocate CQ");
 757                         goto error;
 758                 }
 759                 rte_flow->qp = ibv_exp_create_qp(
 760                         priv->ctx,
 761                         &(struct ibv_exp_qp_init_attr){
 762                                 .send_cq = rte_flow->cq,
 763                                 .recv_cq = rte_flow->cq,
 764                                 .cap = {
 765                                         .max_recv_wr = 1,
 766                                         .max_recv_sge = 1,
 767                                 },
 768                                 .qp_type = IBV_QPT_RAW_PACKET,
 769                                 .comp_mask =
 770                                         IBV_EXP_QP_INIT_ATTR_PD |
 771                                         IBV_EXP_QP_INIT_ATTR_PORT |
 772                                         IBV_EXP_QP_INIT_ATTR_RES_DOMAIN,
 773                                 .pd = priv->pd,
 774                                 .res_domain = rxq->rd,
 775                                 .port_num = priv->port,
 776                         });
 777                 if (!rte_flow->qp) {
 778                         rte_flow_error_set(error, ENOMEM,
 779                                            RTE_FLOW_ERROR_TYPE_HANDLE,
 780                                            NULL, "cannot allocate QP");
 781                         goto error;
 782                 }
 783                 qp = rte_flow->qp;
 784         } else {
 785                 rte_flow->rxq = rxq;
 786                 qp = rxq->qp;
 787         }
 788         rte_flow->ibv_attr = ibv_attr;
 789         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
 790         if (!rte_flow->ibv_flow) {
 791                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 792                                    NULL, "flow rule creation failure");
 793                 goto error;
 794         }
 795         return rte_flow;
 796
 797 error:
 798         assert(rte_flow);
 799         if (rte_flow->cq)
 800                 ibv_destroy_cq(rte_flow->cq);
 801         if (rte_flow->qp)
 802                 ibv_destroy_qp(rte_flow->qp);
 803         rte_free(rte_flow->ibv_attr);
 804         rte_free(rte_flow);
 805         return NULL;
 806 }
 807
 808 /**
 809  * Convert a flow.
 810  *
 811  * @param priv
 812  *   Pointer to private structure.
 813  * @param[in] attr
 814  *   Flow rule attributes.
 815  * @param[in] items
 816  *   Pattern specification (list terminated by the END pattern item).
 817  * @param[in] actions
 818  *   Associated actions (list terminated by the END action).
 819  * @param[out] error
 820  *   Perform verbose error reporting if not NULL.
 821  *
 822  * @return
 823  *   A flow on success, NULL otherwise.
 824  */
 825 static struct rte_flow *
 826 priv_flow_create(struct priv *priv,
 827                  const struct rte_flow_attr *attr,
 828                  const struct rte_flow_item items[],
 829                  const struct rte_flow_action actions[],
 830                  struct rte_flow_error *error)
 831 {
 832         struct rte_flow *rte_flow;
 833         struct mlx4_flow_action action;
 834         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
 835         int err;
 836
 837         err = priv_flow_validate(priv, attr, items, actions, error, &flow);
 838         if (err)
 839                 return NULL;
 840         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
 841         if (!flow.ibv_attr) {
 842                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 843                                    NULL, "cannot allocate ibv_attr memory");
 844                 return NULL;
 845         }
 846         flow.offset = sizeof(struct ibv_flow_attr);
 847         *flow.ibv_attr = (struct ibv_flow_attr){
 848                 .comp_mask = 0,
 849                 .type = IBV_FLOW_ATTR_NORMAL,
 850                 .size = sizeof(struct ibv_flow_attr),
 851                 .priority = attr->priority,
 852                 .num_of_specs = 0,
 853                 .port = priv->port,
 854                 .flags = 0,
 855         };
 856         claim_zero(priv_flow_validate(priv, attr, items, actions,
 857                                       error, &flow));
 858         action = (struct mlx4_flow_action){
 859                 .queue = 0,
 860                 .drop = 0,
 861         };
 862         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 863                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 864                         continue;
 865                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 866                         action.queue = 1;
 867                         action.queue_id =
 868                                 ((const struct rte_flow_action_queue *)
 869                                  actions->conf)->index;
 870                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 871                         action.drop = 1;
 872                 } else {
 873                         rte_flow_error_set(error, ENOTSUP,
 874                                            RTE_FLOW_ERROR_TYPE_ACTION,
 875                                            actions, "unsupported action");
 876                         goto exit;
 877                 }
 878         }
 879         rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
 880                                                  &action, error);
 881         return rte_flow;
 882 exit:
 883         rte_free(flow.ibv_attr);
 884         return NULL;
 885 }
 886
 887 /**
 888  * Create a flow.
 889  *
 890  * @see rte_flow_create()
 891  * @see rte_flow_ops
 892  */
 893 struct rte_flow *
 894 mlx4_flow_create(struct rte_eth_dev *dev,
 895                  const struct rte_flow_attr *attr,
 896                  const struct rte_flow_item items[],
 897                  const struct rte_flow_action actions[],
 898                  struct rte_flow_error *error)
 899 {
 900         struct priv *priv = dev->data->dev_private;
 901         struct rte_flow *flow;
 902
 903         priv_lock(priv);
 904         flow = priv_flow_create(priv, attr, items, actions, error);
 905         if (flow) {
 906                 LIST_INSERT_HEAD(&priv->flows, flow, next);
 907                 DEBUG("Flow created %p", (void *)flow);
 908         }
 909         priv_unlock(priv);
 910         return flow;
 911 }
 912
 913 /**
 914  * Destroy a flow.
 915  *
 916  * @param priv
 917  *   Pointer to private structure.
 918  * @param[in] flow
 919  *   Flow to destroy.
 920  */
 921 static void
 922 priv_flow_destroy(struct priv *priv, struct rte_flow *flow)
 923 {
 924         (void)priv;
 925         LIST_REMOVE(flow, next);
 926         if (flow->ibv_flow)
 927                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 928         if (flow->qp)
 929                 claim_zero(ibv_destroy_qp(flow->qp));
 930         if (flow->cq)
 931                 claim_zero(ibv_destroy_cq(flow->cq));
 932         rte_free(flow->ibv_attr);
 933         DEBUG("Flow destroyed %p", (void *)flow);
 934         rte_free(flow);
 935 }
 936
 937 /**
 938  * Destroy a flow.
 939  *
 940  * @see rte_flow_destroy()
 941  * @see rte_flow_ops
 942  */
 943 int
 944 mlx4_flow_destroy(struct rte_eth_dev *dev,
 945                   struct rte_flow *flow,
 946                   struct rte_flow_error *error)
 947 {
 948         struct priv *priv = dev->data->dev_private;
 949
 950         (void)error;
 951         priv_lock(priv);
 952         priv_flow_destroy(priv, flow);
 953         priv_unlock(priv);
 954         return 0;
 955 }
 956
 957 /**
 958  * Destroy all flows.
 959  *
 960  * @param priv
 961  *   Pointer to private structure.
 962  */
 963 static void
 964 priv_flow_flush(struct priv *priv)
 965 {
 966         while (!LIST_EMPTY(&priv->flows)) {
 967                 struct rte_flow *flow;
 968
 969                 flow = LIST_FIRST(&priv->flows);
 970                 priv_flow_destroy(priv, flow);
 971         }
 972 }
 973
 974 /**
 975  * Destroy all flows.
 976  *
 977  * @see rte_flow_flush()
 978  * @see rte_flow_ops
 979  */
 980 int
 981 mlx4_flow_flush(struct rte_eth_dev *dev,
 982                 struct rte_flow_error *error)
 983 {
 984         struct priv *priv = dev->data->dev_private;
 985
 986         (void)error;
 987         priv_lock(priv);
 988         priv_flow_flush(priv);
 989         priv_unlock(priv);
 990         return 0;
 991 }
 992
 993 /**
 994  * Remove all flows.
 995  *
 996  * Called by dev_stop() to remove all flows.
 997  *
 998  * @param priv
 999  *   Pointer to private structure.
1000  */
1001 void
1002 mlx4_priv_flow_stop(struct priv *priv)
1003 {
1004         struct rte_flow *flow;
1005
1006         for (flow = LIST_FIRST(&priv->flows);
1007              flow;
1008              flow = LIST_NEXT(flow, next)) {
1009                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1010                 flow->ibv_flow = NULL;
1011                 DEBUG("Flow %p removed", (void *)flow);
1012         }
1013 }
1014
1015 /**
1016  * Add all flows.
1017  *
1018  * @param priv
1019  *   Pointer to private structure.
1020  *
1021  * @return
1022  *   0 on success, a errno value otherwise and rte_errno is set.
1023  */
1024 int
1025 mlx4_priv_flow_start(struct priv *priv)
1026 {
1027         struct ibv_qp *qp;
1028         struct rte_flow *flow;
1029
1030         for (flow = LIST_FIRST(&priv->flows);
1031              flow;
1032              flow = LIST_NEXT(flow, next)) {
1033                 qp = flow->qp ? flow->qp : flow->rxq->qp;
1034                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1035                 if (!flow->ibv_flow) {
1036                         DEBUG("Flow %p cannot be applied", (void *)flow);
1037                         rte_errno = EINVAL;
1038                         return rte_errno;
1039                 }
1040                 DEBUG("Flow %p applied", (void *)flow);
1041         }
1042         return 0;
1043 }