drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <arpa/inet.h>
  35 #include <assert.h>
  36 #include <errno.h>
  37 #include <stddef.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <sys/queue.h>
  41
  42 /* Verbs headers do not support -pedantic. */
  43 #ifdef PEDANTIC
  44 #pragma GCC diagnostic ignored "-Wpedantic"
  45 #endif
  46 #include <infiniband/verbs.h>
  47 #ifdef PEDANTIC
  48 #pragma GCC diagnostic error "-Wpedantic"
  49 #endif
  50
  51 #include <rte_errno.h>
  52 #include <rte_eth_ctrl.h>
  53 #include <rte_ethdev.h>
  54 #include <rte_flow.h>
  55 #include <rte_flow_driver.h>
  56 #include <rte_malloc.h>
  57
  58 /* PMD headers. */
  59 #include "mlx4.h"
  60 #include "mlx4_flow.h"
  61 #include "mlx4_rxtx.h"
  62 #include "mlx4_utils.h"
  63
  64 /** Static initializer for items. */
  65 #define ITEMS(...) \
  66         (const enum rte_flow_item_type []){ \
  67                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  68         }
  69
  70 /** Structure to generate a simple graph of layers supported by the NIC. */
  71 struct mlx4_flow_items {
  72         /** List of possible actions for these items. */
  73         const enum rte_flow_action_type *const actions;
  74         /** Bit-masks corresponding to the possibilities for the item. */
  75         const void *mask;
  76         /**
  77          * Default bit-masks to use when item->mask is not provided. When
  78          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  79          * used instead.
  80          */
  81         const void *default_mask;
  82         /** Bit-masks size in bytes. */
  83         const unsigned int mask_sz;
  84         /**
  85          * Check support for a given item.
  86          *
  87          * @param item[in]
  88          *   Item specification.
  89          * @param mask[in]
  90          *   Bit-masks covering supported fields to compare with spec,
  91          *   last and mask in
  92          *   \item.
  93          * @param size
  94          *   Bit-Mask size in bytes.
  95          *
  96          * @return
  97          *   0 on success, negative value otherwise.
  98          */
  99         int (*validate)(const struct rte_flow_item *item,
 100                         const uint8_t *mask, unsigned int size);
 101         /**
 102          * Conversion function from rte_flow to NIC specific flow.
 103          *
 104          * @param item
 105          *   rte_flow item to convert.
 106          * @param default_mask
 107          *   Default bit-masks to use when item->mask is not provided.
 108          * @param data
 109          *   Internal structure to store the conversion.
 110          *
 111          * @return
 112          *   0 on success, negative value otherwise.
 113          */
 114         int (*convert)(const struct rte_flow_item *item,
 115                        const void *default_mask,
 116                        void *data);
 117         /** Size in bytes of the destination structure. */
 118         const unsigned int dst_sz;
 119         /** List of possible following items.  */
 120         const enum rte_flow_item_type *const items;
 121 };
 122
 123 struct rte_flow_drop {
 124         struct ibv_qp *qp; /**< Verbs queue pair. */
 125         struct ibv_cq *cq; /**< Verbs completion queue. */
 126 };
 127
 128 /** Valid action for this PMD. */
 129 static const enum rte_flow_action_type valid_actions[] = {
 130         RTE_FLOW_ACTION_TYPE_DROP,
 131         RTE_FLOW_ACTION_TYPE_QUEUE,
 132         RTE_FLOW_ACTION_TYPE_END,
 133 };
 134
 135 /**
 136  * Convert Ethernet item to Verbs specification.
 137  *
 138  * @param item[in]
 139  *   Item specification.
 140  * @param default_mask[in]
 141  *   Default bit-masks to use when item->mask is not provided.
 142  * @param data[in, out]
 143  *   User structure.
 144  */
 145 static int
 146 mlx4_flow_create_eth(const struct rte_flow_item *item,
 147                      const void *default_mask,
 148                      void *data)
 149 {
 150         const struct rte_flow_item_eth *spec = item->spec;
 151         const struct rte_flow_item_eth *mask = item->mask;
 152         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 153         struct ibv_flow_spec_eth *eth;
 154         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 155         unsigned int i;
 156
 157         ++flow->ibv_attr->num_of_specs;
 158         flow->ibv_attr->priority = 2;
 159         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 160         *eth = (struct ibv_flow_spec_eth) {
 161                 .type = IBV_FLOW_SPEC_ETH,
 162                 .size = eth_size,
 163         };
 164         if (!spec) {
 165                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 166                 return 0;
 167         }
 168         if (!mask)
 169                 mask = default_mask;
 170         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 171         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 172         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 173         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 174         /* Remove unwanted bits from values. */
 175         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 176                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 177                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 178         }
 179         return 0;
 180 }
 181
 182 /**
 183  * Convert VLAN item to Verbs specification.
 184  *
 185  * @param item[in]
 186  *   Item specification.
 187  * @param default_mask[in]
 188  *   Default bit-masks to use when item->mask is not provided.
 189  * @param data[in, out]
 190  *   User structure.
 191  */
 192 static int
 193 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 194                       const void *default_mask,
 195                       void *data)
 196 {
 197         const struct rte_flow_item_vlan *spec = item->spec;
 198         const struct rte_flow_item_vlan *mask = item->mask;
 199         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 200         struct ibv_flow_spec_eth *eth;
 201         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 202
 203         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 204         if (!spec)
 205                 return 0;
 206         if (!mask)
 207                 mask = default_mask;
 208         eth->val.vlan_tag = spec->tci;
 209         eth->mask.vlan_tag = mask->tci;
 210         eth->val.vlan_tag &= eth->mask.vlan_tag;
 211         return 0;
 212 }
 213
 214 /**
 215  * Convert IPv4 item to Verbs specification.
 216  *
 217  * @param item[in]
 218  *   Item specification.
 219  * @param default_mask[in]
 220  *   Default bit-masks to use when item->mask is not provided.
 221  * @param data[in, out]
 222  *   User structure.
 223  */
 224 static int
 225 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 226                       const void *default_mask,
 227                       void *data)
 228 {
 229         const struct rte_flow_item_ipv4 *spec = item->spec;
 230         const struct rte_flow_item_ipv4 *mask = item->mask;
 231         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 232         struct ibv_flow_spec_ipv4 *ipv4;
 233         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 234
 235         ++flow->ibv_attr->num_of_specs;
 236         flow->ibv_attr->priority = 1;
 237         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 238         *ipv4 = (struct ibv_flow_spec_ipv4) {
 239                 .type = IBV_FLOW_SPEC_IPV4,
 240                 .size = ipv4_size,
 241         };
 242         if (!spec)
 243                 return 0;
 244         ipv4->val = (struct ibv_flow_ipv4_filter) {
 245                 .src_ip = spec->hdr.src_addr,
 246                 .dst_ip = spec->hdr.dst_addr,
 247         };
 248         if (!mask)
 249                 mask = default_mask;
 250         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 251                 .src_ip = mask->hdr.src_addr,
 252                 .dst_ip = mask->hdr.dst_addr,
 253         };
 254         /* Remove unwanted bits from values. */
 255         ipv4->val.src_ip &= ipv4->mask.src_ip;
 256         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 257         return 0;
 258 }
 259
 260 /**
 261  * Convert UDP item to Verbs specification.
 262  *
 263  * @param item[in]
 264  *   Item specification.
 265  * @param default_mask[in]
 266  *   Default bit-masks to use when item->mask is not provided.
 267  * @param data[in, out]
 268  *   User structure.
 269  */
 270 static int
 271 mlx4_flow_create_udp(const struct rte_flow_item *item,
 272                      const void *default_mask,
 273                      void *data)
 274 {
 275         const struct rte_flow_item_udp *spec = item->spec;
 276         const struct rte_flow_item_udp *mask = item->mask;
 277         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 278         struct ibv_flow_spec_tcp_udp *udp;
 279         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 280
 281         ++flow->ibv_attr->num_of_specs;
 282         flow->ibv_attr->priority = 0;
 283         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 284         *udp = (struct ibv_flow_spec_tcp_udp) {
 285                 .type = IBV_FLOW_SPEC_UDP,
 286                 .size = udp_size,
 287         };
 288         if (!spec)
 289                 return 0;
 290         udp->val.dst_port = spec->hdr.dst_port;
 291         udp->val.src_port = spec->hdr.src_port;
 292         if (!mask)
 293                 mask = default_mask;
 294         udp->mask.dst_port = mask->hdr.dst_port;
 295         udp->mask.src_port = mask->hdr.src_port;
 296         /* Remove unwanted bits from values. */
 297         udp->val.src_port &= udp->mask.src_port;
 298         udp->val.dst_port &= udp->mask.dst_port;
 299         return 0;
 300 }
 301
 302 /**
 303  * Convert TCP item to Verbs specification.
 304  *
 305  * @param item[in]
 306  *   Item specification.
 307  * @param default_mask[in]
 308  *   Default bit-masks to use when item->mask is not provided.
 309  * @param data[in, out]
 310  *   User structure.
 311  */
 312 static int
 313 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 314                      const void *default_mask,
 315                      void *data)
 316 {
 317         const struct rte_flow_item_tcp *spec = item->spec;
 318         const struct rte_flow_item_tcp *mask = item->mask;
 319         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 320         struct ibv_flow_spec_tcp_udp *tcp;
 321         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 322
 323         ++flow->ibv_attr->num_of_specs;
 324         flow->ibv_attr->priority = 0;
 325         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 326         *tcp = (struct ibv_flow_spec_tcp_udp) {
 327                 .type = IBV_FLOW_SPEC_TCP,
 328                 .size = tcp_size,
 329         };
 330         if (!spec)
 331                 return 0;
 332         tcp->val.dst_port = spec->hdr.dst_port;
 333         tcp->val.src_port = spec->hdr.src_port;
 334         if (!mask)
 335                 mask = default_mask;
 336         tcp->mask.dst_port = mask->hdr.dst_port;
 337         tcp->mask.src_port = mask->hdr.src_port;
 338         /* Remove unwanted bits from values. */
 339         tcp->val.src_port &= tcp->mask.src_port;
 340         tcp->val.dst_port &= tcp->mask.dst_port;
 341         return 0;
 342 }
 343
 344 /**
 345  * Check support for a given item.
 346  *
 347  * @param item[in]
 348  *   Item specification.
 349  * @param mask[in]
 350  *   Bit-masks covering supported fields to compare with spec, last and mask in
 351  *   \item.
 352  * @param size
 353  *   Bit-Mask size in bytes.
 354  *
 355  * @return
 356  *   0 on success, negative value otherwise.
 357  */
 358 static int
 359 mlx4_flow_item_validate(const struct rte_flow_item *item,
 360                         const uint8_t *mask, unsigned int size)
 361 {
 362         int ret = 0;
 363
 364         if (!item->spec && (item->mask || item->last))
 365                 return -1;
 366         if (item->spec && !item->mask) {
 367                 unsigned int i;
 368                 const uint8_t *spec = item->spec;
 369
 370                 for (i = 0; i < size; ++i)
 371                         if ((spec[i] | mask[i]) != mask[i])
 372                                 return -1;
 373         }
 374         if (item->last && !item->mask) {
 375                 unsigned int i;
 376                 const uint8_t *spec = item->last;
 377
 378                 for (i = 0; i < size; ++i)
 379                         if ((spec[i] | mask[i]) != mask[i])
 380                                 return -1;
 381         }
 382         if (item->spec && item->last) {
 383                 uint8_t spec[size];
 384                 uint8_t last[size];
 385                 const uint8_t *apply = mask;
 386                 unsigned int i;
 387
 388                 if (item->mask)
 389                         apply = item->mask;
 390                 for (i = 0; i < size; ++i) {
 391                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 392                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 393                 }
 394                 ret = memcmp(spec, last, size);
 395         }
 396         return ret;
 397 }
 398
 399 static int
 400 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 401                        const uint8_t *mask, unsigned int size)
 402 {
 403         if (item->mask) {
 404                 const struct rte_flow_item_eth *mask = item->mask;
 405
 406                 if (mask->dst.addr_bytes[0] != 0xff ||
 407                                 mask->dst.addr_bytes[1] != 0xff ||
 408                                 mask->dst.addr_bytes[2] != 0xff ||
 409                                 mask->dst.addr_bytes[3] != 0xff ||
 410                                 mask->dst.addr_bytes[4] != 0xff ||
 411                                 mask->dst.addr_bytes[5] != 0xff)
 412                         return -1;
 413         }
 414         return mlx4_flow_item_validate(item, mask, size);
 415 }
 416
 417 static int
 418 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 419                         const uint8_t *mask, unsigned int size)
 420 {
 421         if (item->mask) {
 422                 const struct rte_flow_item_vlan *mask = item->mask;
 423
 424                 if (mask->tci != 0 &&
 425                     ntohs(mask->tci) != 0x0fff)
 426                         return -1;
 427         }
 428         return mlx4_flow_item_validate(item, mask, size);
 429 }
 430
 431 static int
 432 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 433                         const uint8_t *mask, unsigned int size)
 434 {
 435         if (item->mask) {
 436                 const struct rte_flow_item_ipv4 *mask = item->mask;
 437
 438                 if (mask->hdr.src_addr != 0 &&
 439                     mask->hdr.src_addr != 0xffffffff)
 440                         return -1;
 441                 if (mask->hdr.dst_addr != 0 &&
 442                     mask->hdr.dst_addr != 0xffffffff)
 443                         return -1;
 444         }
 445         return mlx4_flow_item_validate(item, mask, size);
 446 }
 447
 448 static int
 449 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 450                        const uint8_t *mask, unsigned int size)
 451 {
 452         if (item->mask) {
 453                 const struct rte_flow_item_udp *mask = item->mask;
 454
 455                 if (mask->hdr.src_port != 0 &&
 456                     mask->hdr.src_port != 0xffff)
 457                         return -1;
 458                 if (mask->hdr.dst_port != 0 &&
 459                     mask->hdr.dst_port != 0xffff)
 460                         return -1;
 461         }
 462         return mlx4_flow_item_validate(item, mask, size);
 463 }
 464
 465 static int
 466 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 467                        const uint8_t *mask, unsigned int size)
 468 {
 469         if (item->mask) {
 470                 const struct rte_flow_item_tcp *mask = item->mask;
 471
 472                 if (mask->hdr.src_port != 0 &&
 473                     mask->hdr.src_port != 0xffff)
 474                         return -1;
 475                 if (mask->hdr.dst_port != 0 &&
 476                     mask->hdr.dst_port != 0xffff)
 477                         return -1;
 478         }
 479         return mlx4_flow_item_validate(item, mask, size);
 480 }
 481
 482 /** Graph of supported items and associated actions. */
 483 static const struct mlx4_flow_items mlx4_flow_items[] = {
 484         [RTE_FLOW_ITEM_TYPE_END] = {
 485                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 486         },
 487         [RTE_FLOW_ITEM_TYPE_ETH] = {
 488                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 489                                RTE_FLOW_ITEM_TYPE_IPV4),
 490                 .actions = valid_actions,
 491                 .mask = &(const struct rte_flow_item_eth){
 492                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 493                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 494                 },
 495                 .default_mask = &rte_flow_item_eth_mask,
 496                 .mask_sz = sizeof(struct rte_flow_item_eth),
 497                 .validate = mlx4_flow_validate_eth,
 498                 .convert = mlx4_flow_create_eth,
 499                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 500         },
 501         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 502                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 503                 .actions = valid_actions,
 504                 .mask = &(const struct rte_flow_item_vlan){
 505                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 506 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 507                         .tci = 0x0fff,
 508 #else
 509                         .tci = 0xff0f,
 510 #endif
 511                 },
 512                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 513                 .validate = mlx4_flow_validate_vlan,
 514                 .convert = mlx4_flow_create_vlan,
 515                 .dst_sz = 0,
 516         },
 517         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 518                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 519                                RTE_FLOW_ITEM_TYPE_TCP),
 520                 .actions = valid_actions,
 521                 .mask = &(const struct rte_flow_item_ipv4){
 522                         .hdr = {
 523                                 .src_addr = -1,
 524                                 .dst_addr = -1,
 525                         },
 526                 },
 527                 .default_mask = &rte_flow_item_ipv4_mask,
 528                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 529                 .validate = mlx4_flow_validate_ipv4,
 530                 .convert = mlx4_flow_create_ipv4,
 531                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 532         },
 533         [RTE_FLOW_ITEM_TYPE_UDP] = {
 534                 .actions = valid_actions,
 535                 .mask = &(const struct rte_flow_item_udp){
 536                         .hdr = {
 537                                 .src_port = -1,
 538                                 .dst_port = -1,
 539                         },
 540                 },
 541                 .default_mask = &rte_flow_item_udp_mask,
 542                 .mask_sz = sizeof(struct rte_flow_item_udp),
 543                 .validate = mlx4_flow_validate_udp,
 544                 .convert = mlx4_flow_create_udp,
 545                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 546         },
 547         [RTE_FLOW_ITEM_TYPE_TCP] = {
 548                 .actions = valid_actions,
 549                 .mask = &(const struct rte_flow_item_tcp){
 550                         .hdr = {
 551                                 .src_port = -1,
 552                                 .dst_port = -1,
 553                         },
 554                 },
 555                 .default_mask = &rte_flow_item_tcp_mask,
 556                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 557                 .validate = mlx4_flow_validate_tcp,
 558                 .convert = mlx4_flow_create_tcp,
 559                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 560         },
 561 };
 562
 563 /**
 564  * Make sure a flow rule is supported and initialize associated structure.
 565  *
 566  * @param priv
 567  *   Pointer to private structure.
 568  * @param[in] attr
 569  *   Flow rule attributes.
 570  * @param[in] items
 571  *   Pattern specification (list terminated by the END pattern item).
 572  * @param[in] actions
 573  *   Associated actions (list terminated by the END action).
 574  * @param[out] error
 575  *   Perform verbose error reporting if not NULL.
 576  * @param[in, out] flow
 577  *   Flow structure to update.
 578  *
 579  * @return
 580  *   0 on success, a negative errno value otherwise and rte_errno is set.
 581  */
 582 static int
 583 mlx4_flow_prepare(struct priv *priv,
 584                   const struct rte_flow_attr *attr,
 585                   const struct rte_flow_item items[],
 586                   const struct rte_flow_action actions[],
 587                   struct rte_flow_error *error,
 588                   struct mlx4_flow *flow)
 589 {
 590         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 591         struct mlx4_flow_action action = {
 592                 .queue = 0,
 593                 .drop = 0,
 594         };
 595
 596         (void)priv;
 597         if (attr->group) {
 598                 rte_flow_error_set(error, ENOTSUP,
 599                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 600                                    NULL,
 601                                    "groups are not supported");
 602                 return -rte_errno;
 603         }
 604         if (attr->priority) {
 605                 rte_flow_error_set(error, ENOTSUP,
 606                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 607                                    NULL,
 608                                    "priorities are not supported");
 609                 return -rte_errno;
 610         }
 611         if (attr->egress) {
 612                 rte_flow_error_set(error, ENOTSUP,
 613                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 614                                    NULL,
 615                                    "egress is not supported");
 616                 return -rte_errno;
 617         }
 618         if (!attr->ingress) {
 619                 rte_flow_error_set(error, ENOTSUP,
 620                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 621                                    NULL,
 622                                    "only ingress is supported");
 623                 return -rte_errno;
 624         }
 625         /* Go over items list. */
 626         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 627                 const struct mlx4_flow_items *token = NULL;
 628                 unsigned int i;
 629                 int err;
 630
 631                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 632                         continue;
 633                 /*
 634                  * The nic can support patterns with NULL eth spec only
 635                  * if eth is a single item in a rule.
 636                  */
 637                 if (!items->spec &&
 638                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 639                         const struct rte_flow_item *next = items + 1;
 640
 641                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 642                                 rte_flow_error_set(error, ENOTSUP,
 643                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 644                                                    items,
 645                                                    "the rule requires"
 646                                                    " an Ethernet spec");
 647                                 return -rte_errno;
 648                         }
 649                 }
 650                 for (i = 0;
 651                      cur_item->items &&
 652                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 653                      ++i) {
 654                         if (cur_item->items[i] == items->type) {
 655                                 token = &mlx4_flow_items[items->type];
 656                                 break;
 657                         }
 658                 }
 659                 if (!token)
 660                         goto exit_item_not_supported;
 661                 cur_item = token;
 662                 err = cur_item->validate(items,
 663                                         (const uint8_t *)cur_item->mask,
 664                                          cur_item->mask_sz);
 665                 if (err)
 666                         goto exit_item_not_supported;
 667                 if (flow->ibv_attr && cur_item->convert) {
 668                         err = cur_item->convert(items,
 669                                                 (cur_item->default_mask ?
 670                                                  cur_item->default_mask :
 671                                                  cur_item->mask),
 672                                                  flow);
 673                         if (err)
 674                                 goto exit_item_not_supported;
 675                 }
 676                 flow->offset += cur_item->dst_sz;
 677         }
 678         /* Go over actions list */
 679         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 680                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 681                         continue;
 682                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 683                         action.drop = 1;
 684                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 685                         const struct rte_flow_action_queue *queue =
 686                                 (const struct rte_flow_action_queue *)
 687                                 actions->conf;
 688
 689                         if (!queue || (queue->index > (priv->rxqs_n - 1)))
 690                                 goto exit_action_not_supported;
 691                         action.queue = 1;
 692                 } else {
 693                         goto exit_action_not_supported;
 694                 }
 695         }
 696         if (!action.queue && !action.drop) {
 697                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 698                                    NULL, "no valid action");
 699                 return -rte_errno;
 700         }
 701         return 0;
 702 exit_item_not_supported:
 703         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 704                            items, "item not supported");
 705         return -rte_errno;
 706 exit_action_not_supported:
 707         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 708                            actions, "action not supported");
 709         return -rte_errno;
 710 }
 711
 712 /**
 713  * Validate a flow supported by the NIC.
 714  *
 715  * @see rte_flow_validate()
 716  * @see rte_flow_ops
 717  */
 718 static int
 719 mlx4_flow_validate(struct rte_eth_dev *dev,
 720                    const struct rte_flow_attr *attr,
 721                    const struct rte_flow_item items[],
 722                    const struct rte_flow_action actions[],
 723                    struct rte_flow_error *error)
 724 {
 725         struct priv *priv = dev->data->dev_private;
 726         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 727
 728         return mlx4_flow_prepare(priv, attr, items, actions, error, &flow);
 729 }
 730
 731 /**
 732  * Destroy a drop queue.
 733  *
 734  * @param priv
 735  *   Pointer to private structure.
 736  */
 737 static void
 738 mlx4_flow_destroy_drop_queue(struct priv *priv)
 739 {
 740         if (priv->flow_drop_queue) {
 741                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 742
 743                 priv->flow_drop_queue = NULL;
 744                 claim_zero(ibv_destroy_qp(fdq->qp));
 745                 claim_zero(ibv_destroy_cq(fdq->cq));
 746                 rte_free(fdq);
 747         }
 748 }
 749
 750 /**
 751  * Create a single drop queue for all drop flows.
 752  *
 753  * @param priv
 754  *   Pointer to private structure.
 755  *
 756  * @return
 757  *   0 on success, negative value otherwise.
 758  */
 759 static int
 760 mlx4_flow_create_drop_queue(struct priv *priv)
 761 {
 762         struct ibv_qp *qp;
 763         struct ibv_cq *cq;
 764         struct rte_flow_drop *fdq;
 765
 766         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 767         if (!fdq) {
 768                 ERROR("Cannot allocate memory for drop struct");
 769                 goto err;
 770         }
 771         cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 772         if (!cq) {
 773                 ERROR("Cannot create drop CQ");
 774                 goto err_create_cq;
 775         }
 776         qp = ibv_create_qp(priv->pd,
 777                            &(struct ibv_qp_init_attr){
 778                                 .send_cq = cq,
 779                                 .recv_cq = cq,
 780                                 .cap = {
 781                                         .max_recv_wr = 1,
 782                                         .max_recv_sge = 1,
 783                                 },
 784                                 .qp_type = IBV_QPT_RAW_PACKET,
 785                            });
 786         if (!qp) {
 787                 ERROR("Cannot create drop QP");
 788                 goto err_create_qp;
 789         }
 790         *fdq = (struct rte_flow_drop){
 791                 .qp = qp,
 792                 .cq = cq,
 793         };
 794         priv->flow_drop_queue = fdq;
 795         return 0;
 796 err_create_qp:
 797         claim_zero(ibv_destroy_cq(cq));
 798 err_create_cq:
 799         rte_free(fdq);
 800 err:
 801         return -1;
 802 }
 803
 804 /**
 805  * Complete flow rule creation.
 806  *
 807  * @param priv
 808  *   Pointer to private structure.
 809  * @param ibv_attr
 810  *   Verbs flow attributes.
 811  * @param action
 812  *   Target action structure.
 813  * @param[out] error
 814  *   Perform verbose error reporting if not NULL.
 815  *
 816  * @return
 817  *   A flow if the rule could be created.
 818  */
 819 static struct rte_flow *
 820 mlx4_flow_create_action_queue(struct priv *priv,
 821                               struct ibv_flow_attr *ibv_attr,
 822                               struct mlx4_flow_action *action,
 823                               struct rte_flow_error *error)
 824 {
 825         struct ibv_qp *qp;
 826         struct rte_flow *rte_flow;
 827
 828         assert(priv->pd);
 829         assert(priv->ctx);
 830         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 831         if (!rte_flow) {
 832                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 833                                    NULL, "cannot allocate flow memory");
 834                 return NULL;
 835         }
 836         if (action->drop) {
 837                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 838         } else {
 839                 struct rxq *rxq = (*priv->rxqs)[action->queue_id];
 840
 841                 qp = rxq->qp;
 842                 rte_flow->qp = qp;
 843         }
 844         rte_flow->ibv_attr = ibv_attr;
 845         if (!priv->started)
 846                 return rte_flow;
 847         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
 848         if (!rte_flow->ibv_flow) {
 849                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 850                                    NULL, "flow rule creation failure");
 851                 goto error;
 852         }
 853         return rte_flow;
 854 error:
 855         rte_free(rte_flow);
 856         return NULL;
 857 }
 858
 859 /**
 860  * Create a flow.
 861  *
 862  * @see rte_flow_create()
 863  * @see rte_flow_ops
 864  */
 865 static struct rte_flow *
 866 mlx4_flow_create(struct rte_eth_dev *dev,
 867                  const struct rte_flow_attr *attr,
 868                  const struct rte_flow_item items[],
 869                  const struct rte_flow_action actions[],
 870                  struct rte_flow_error *error)
 871 {
 872         struct priv *priv = dev->data->dev_private;
 873         struct rte_flow *rte_flow;
 874         struct mlx4_flow_action action;
 875         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
 876         int err;
 877
 878         err = mlx4_flow_prepare(priv, attr, items, actions, error, &flow);
 879         if (err)
 880                 return NULL;
 881         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
 882         if (!flow.ibv_attr) {
 883                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 884                                    NULL, "cannot allocate ibv_attr memory");
 885                 return NULL;
 886         }
 887         flow.offset = sizeof(struct ibv_flow_attr);
 888         *flow.ibv_attr = (struct ibv_flow_attr){
 889                 .comp_mask = 0,
 890                 .type = IBV_FLOW_ATTR_NORMAL,
 891                 .size = sizeof(struct ibv_flow_attr),
 892                 .priority = attr->priority,
 893                 .num_of_specs = 0,
 894                 .port = priv->port,
 895                 .flags = 0,
 896         };
 897         claim_zero(mlx4_flow_prepare(priv, attr, items, actions,
 898                                      error, &flow));
 899         action = (struct mlx4_flow_action){
 900                 .queue = 0,
 901                 .drop = 0,
 902         };
 903         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 904                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 905                         continue;
 906                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 907                         action.queue = 1;
 908                         action.queue_id =
 909                                 ((const struct rte_flow_action_queue *)
 910                                  actions->conf)->index;
 911                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 912                         action.drop = 1;
 913                 } else {
 914                         rte_flow_error_set(error, ENOTSUP,
 915                                            RTE_FLOW_ERROR_TYPE_ACTION,
 916                                            actions, "unsupported action");
 917                         goto exit;
 918                 }
 919         }
 920         rte_flow = mlx4_flow_create_action_queue(priv, flow.ibv_attr,
 921                                                  &action, error);
 922         if (rte_flow) {
 923                 LIST_INSERT_HEAD(&priv->flows, rte_flow, next);
 924                 DEBUG("Flow created %p", (void *)rte_flow);
 925                 return rte_flow;
 926         }
 927 exit:
 928         rte_free(flow.ibv_attr);
 929         return NULL;
 930 }
 931
 932 /**
 933  * @see rte_flow_isolate()
 934  *
 935  * Must be done before calling dev_configure().
 936  *
 937  * @param dev
 938  *   Pointer to the ethernet device structure.
 939  * @param enable
 940  *   Nonzero to enter isolated mode, attempt to leave it otherwise.
 941  * @param[out] error
 942  *   Perform verbose error reporting if not NULL. PMDs initialize this
 943  *   structure in case of error only.
 944  *
 945  * @return
 946  *   0 on success, a negative value on error.
 947  */
 948 static int
 949 mlx4_flow_isolate(struct rte_eth_dev *dev,
 950                   int enable,
 951                   struct rte_flow_error *error)
 952 {
 953         struct priv *priv = dev->data->dev_private;
 954
 955         if (priv->rxqs) {
 956                 rte_flow_error_set(error, ENOTSUP,
 957                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 958                                    NULL, "isolated mode must be set"
 959                                    " before configuring the device");
 960                 return -rte_errno;
 961         }
 962         priv->isolated = !!enable;
 963         return 0;
 964 }
 965
 966 /**
 967  * Destroy a flow.
 968  *
 969  * @see rte_flow_destroy()
 970  * @see rte_flow_ops
 971  */
 972 static int
 973 mlx4_flow_destroy(struct rte_eth_dev *dev,
 974                   struct rte_flow *flow,
 975                   struct rte_flow_error *error)
 976 {
 977         (void)dev;
 978         (void)error;
 979         LIST_REMOVE(flow, next);
 980         if (flow->ibv_flow)
 981                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 982         rte_free(flow->ibv_attr);
 983         DEBUG("Flow destroyed %p", (void *)flow);
 984         rte_free(flow);
 985         return 0;
 986 }
 987
 988 /**
 989  * Destroy all flows.
 990  *
 991  * @see rte_flow_flush()
 992  * @see rte_flow_ops
 993  */
 994 static int
 995 mlx4_flow_flush(struct rte_eth_dev *dev,
 996                 struct rte_flow_error *error)
 997 {
 998         struct priv *priv = dev->data->dev_private;
 999
1000         while (!LIST_EMPTY(&priv->flows)) {
1001                 struct rte_flow *flow;
1002
1003                 flow = LIST_FIRST(&priv->flows);
1004                 mlx4_flow_destroy(dev, flow, error);
1005         }
1006         return 0;
1007 }
1008
1009 /**
1010  * Remove all flows.
1011  *
1012  * Called by dev_stop() to remove all flows.
1013  *
1014  * @param priv
1015  *   Pointer to private structure.
1016  */
1017 void
1018 mlx4_flow_stop(struct priv *priv)
1019 {
1020         struct rte_flow *flow;
1021
1022         for (flow = LIST_FIRST(&priv->flows);
1023              flow;
1024              flow = LIST_NEXT(flow, next)) {
1025                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1026                 flow->ibv_flow = NULL;
1027                 DEBUG("Flow %p removed", (void *)flow);
1028         }
1029         mlx4_flow_destroy_drop_queue(priv);
1030 }
1031
1032 /**
1033  * Add all flows.
1034  *
1035  * @param priv
1036  *   Pointer to private structure.
1037  *
1038  * @return
1039  *   0 on success, a errno value otherwise and rte_errno is set.
1040  */
1041 int
1042 mlx4_flow_start(struct priv *priv)
1043 {
1044         int ret;
1045         struct ibv_qp *qp;
1046         struct rte_flow *flow;
1047
1048         ret = mlx4_flow_create_drop_queue(priv);
1049         if (ret)
1050                 return -1;
1051         for (flow = LIST_FIRST(&priv->flows);
1052              flow;
1053              flow = LIST_NEXT(flow, next)) {
1054                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1055                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1056                 if (!flow->ibv_flow) {
1057                         DEBUG("Flow %p cannot be applied", (void *)flow);
1058                         rte_errno = EINVAL;
1059                         return rte_errno;
1060                 }
1061                 DEBUG("Flow %p applied", (void *)flow);
1062         }
1063         return 0;
1064 }
1065
1066 static const struct rte_flow_ops mlx4_flow_ops = {
1067         .validate = mlx4_flow_validate,
1068         .create = mlx4_flow_create,
1069         .destroy = mlx4_flow_destroy,
1070         .flush = mlx4_flow_flush,
1071         .isolate = mlx4_flow_isolate,
1072 };
1073
1074 /**
1075  * Manage filter operations.
1076  *
1077  * @param dev
1078  *   Pointer to Ethernet device structure.
1079  * @param filter_type
1080  *   Filter type.
1081  * @param filter_op
1082  *   Operation to perform.
1083  * @param arg
1084  *   Pointer to operation-specific structure.
1085  *
1086  * @return
1087  *   0 on success, negative errno value otherwise and rte_errno is set.
1088  */
1089 int
1090 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1091                  enum rte_filter_type filter_type,
1092                  enum rte_filter_op filter_op,
1093                  void *arg)
1094 {
1095         switch (filter_type) {
1096         case RTE_ETH_FILTER_GENERIC:
1097                 if (filter_op != RTE_ETH_FILTER_GET)
1098                         break;
1099                 *(const void **)arg = &mlx4_flow_ops;
1100                 return 0;
1101         default:
1102                 ERROR("%p: filter type (%d) not supported",
1103                       (void *)dev, filter_type);
1104                 break;
1105         }
1106         rte_errno = ENOTSUP;
1107         return -rte_errno;
1108 }