drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Flow API operations for mlx4 driver.
  37  */
  38
  39 #include <arpa/inet.h>
  40 #include <assert.h>
  41 #include <errno.h>
  42 #include <stddef.h>
  43 #include <stdint.h>
  44 #include <string.h>
  45 #include <sys/queue.h>
  46
  47 /* Verbs headers do not support -pedantic. */
  48 #ifdef PEDANTIC
  49 #pragma GCC diagnostic ignored "-Wpedantic"
  50 #endif
  51 #include <infiniband/verbs.h>
  52 #ifdef PEDANTIC
  53 #pragma GCC diagnostic error "-Wpedantic"
  54 #endif
  55
  56 #include <rte_errno.h>
  57 #include <rte_eth_ctrl.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_flow.h>
  60 #include <rte_flow_driver.h>
  61 #include <rte_malloc.h>
  62
  63 /* PMD headers. */
  64 #include "mlx4.h"
  65 #include "mlx4_flow.h"
  66 #include "mlx4_rxtx.h"
  67 #include "mlx4_utils.h"
  68
  69 /** Static initializer for items. */
  70 #define ITEMS(...) \
  71         (const enum rte_flow_item_type []){ \
  72                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  73         }
  74
  75 /** Structure to generate a simple graph of layers supported by the NIC. */
  76 struct mlx4_flow_items {
  77         /** List of possible actions for these items. */
  78         const enum rte_flow_action_type *const actions;
  79         /** Bit-masks corresponding to the possibilities for the item. */
  80         const void *mask;
  81         /**
  82          * Default bit-masks to use when item->mask is not provided. When
  83          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  84          * used instead.
  85          */
  86         const void *default_mask;
  87         /** Bit-masks size in bytes. */
  88         const unsigned int mask_sz;
  89         /**
  90          * Check support for a given item.
  91          *
  92          * @param item[in]
  93          *   Item specification.
  94          * @param mask[in]
  95          *   Bit-masks covering supported fields to compare with spec,
  96          *   last and mask in
  97          *   \item.
  98          * @param size
  99          *   Bit-Mask size in bytes.
 100          *
 101          * @return
 102          *   0 on success, negative value otherwise.
 103          */
 104         int (*validate)(const struct rte_flow_item *item,
 105                         const uint8_t *mask, unsigned int size);
 106         /**
 107          * Conversion function from rte_flow to NIC specific flow.
 108          *
 109          * @param item
 110          *   rte_flow item to convert.
 111          * @param default_mask
 112          *   Default bit-masks to use when item->mask is not provided.
 113          * @param data
 114          *   Internal structure to store the conversion.
 115          *
 116          * @return
 117          *   0 on success, negative value otherwise.
 118          */
 119         int (*convert)(const struct rte_flow_item *item,
 120                        const void *default_mask,
 121                        void *data);
 122         /** Size in bytes of the destination structure. */
 123         const unsigned int dst_sz;
 124         /** List of possible following items.  */
 125         const enum rte_flow_item_type *const items;
 126 };
 127
 128 struct rte_flow_drop {
 129         struct ibv_qp *qp; /**< Verbs queue pair. */
 130         struct ibv_cq *cq; /**< Verbs completion queue. */
 131 };
 132
 133 /** Valid action for this PMD. */
 134 static const enum rte_flow_action_type valid_actions[] = {
 135         RTE_FLOW_ACTION_TYPE_DROP,
 136         RTE_FLOW_ACTION_TYPE_QUEUE,
 137         RTE_FLOW_ACTION_TYPE_END,
 138 };
 139
 140 /**
 141  * Convert Ethernet item to Verbs specification.
 142  *
 143  * @param item[in]
 144  *   Item specification.
 145  * @param default_mask[in]
 146  *   Default bit-masks to use when item->mask is not provided.
 147  * @param data[in, out]
 148  *   User structure.
 149  */
 150 static int
 151 mlx4_flow_create_eth(const struct rte_flow_item *item,
 152                      const void *default_mask,
 153                      void *data)
 154 {
 155         const struct rte_flow_item_eth *spec = item->spec;
 156         const struct rte_flow_item_eth *mask = item->mask;
 157         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 158         struct ibv_flow_spec_eth *eth;
 159         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 160         unsigned int i;
 161
 162         ++flow->ibv_attr->num_of_specs;
 163         flow->ibv_attr->priority = 2;
 164         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 165         *eth = (struct ibv_flow_spec_eth) {
 166                 .type = IBV_FLOW_SPEC_ETH,
 167                 .size = eth_size,
 168         };
 169         if (!spec) {
 170                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 171                 return 0;
 172         }
 173         if (!mask)
 174                 mask = default_mask;
 175         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 176         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 177         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 178         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 179         /* Remove unwanted bits from values. */
 180         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 181                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 182                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 183         }
 184         return 0;
 185 }
 186
 187 /**
 188  * Convert VLAN item to Verbs specification.
 189  *
 190  * @param item[in]
 191  *   Item specification.
 192  * @param default_mask[in]
 193  *   Default bit-masks to use when item->mask is not provided.
 194  * @param data[in, out]
 195  *   User structure.
 196  */
 197 static int
 198 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 199                       const void *default_mask,
 200                       void *data)
 201 {
 202         const struct rte_flow_item_vlan *spec = item->spec;
 203         const struct rte_flow_item_vlan *mask = item->mask;
 204         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 205         struct ibv_flow_spec_eth *eth;
 206         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 207
 208         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 209         if (!spec)
 210                 return 0;
 211         if (!mask)
 212                 mask = default_mask;
 213         eth->val.vlan_tag = spec->tci;
 214         eth->mask.vlan_tag = mask->tci;
 215         eth->val.vlan_tag &= eth->mask.vlan_tag;
 216         return 0;
 217 }
 218
 219 /**
 220  * Convert IPv4 item to Verbs specification.
 221  *
 222  * @param item[in]
 223  *   Item specification.
 224  * @param default_mask[in]
 225  *   Default bit-masks to use when item->mask is not provided.
 226  * @param data[in, out]
 227  *   User structure.
 228  */
 229 static int
 230 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 231                       const void *default_mask,
 232                       void *data)
 233 {
 234         const struct rte_flow_item_ipv4 *spec = item->spec;
 235         const struct rte_flow_item_ipv4 *mask = item->mask;
 236         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 237         struct ibv_flow_spec_ipv4 *ipv4;
 238         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 239
 240         ++flow->ibv_attr->num_of_specs;
 241         flow->ibv_attr->priority = 1;
 242         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 243         *ipv4 = (struct ibv_flow_spec_ipv4) {
 244                 .type = IBV_FLOW_SPEC_IPV4,
 245                 .size = ipv4_size,
 246         };
 247         if (!spec)
 248                 return 0;
 249         ipv4->val = (struct ibv_flow_ipv4_filter) {
 250                 .src_ip = spec->hdr.src_addr,
 251                 .dst_ip = spec->hdr.dst_addr,
 252         };
 253         if (!mask)
 254                 mask = default_mask;
 255         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 256                 .src_ip = mask->hdr.src_addr,
 257                 .dst_ip = mask->hdr.dst_addr,
 258         };
 259         /* Remove unwanted bits from values. */
 260         ipv4->val.src_ip &= ipv4->mask.src_ip;
 261         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 262         return 0;
 263 }
 264
 265 /**
 266  * Convert UDP item to Verbs specification.
 267  *
 268  * @param item[in]
 269  *   Item specification.
 270  * @param default_mask[in]
 271  *   Default bit-masks to use when item->mask is not provided.
 272  * @param data[in, out]
 273  *   User structure.
 274  */
 275 static int
 276 mlx4_flow_create_udp(const struct rte_flow_item *item,
 277                      const void *default_mask,
 278                      void *data)
 279 {
 280         const struct rte_flow_item_udp *spec = item->spec;
 281         const struct rte_flow_item_udp *mask = item->mask;
 282         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 283         struct ibv_flow_spec_tcp_udp *udp;
 284         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 285
 286         ++flow->ibv_attr->num_of_specs;
 287         flow->ibv_attr->priority = 0;
 288         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 289         *udp = (struct ibv_flow_spec_tcp_udp) {
 290                 .type = IBV_FLOW_SPEC_UDP,
 291                 .size = udp_size,
 292         };
 293         if (!spec)
 294                 return 0;
 295         udp->val.dst_port = spec->hdr.dst_port;
 296         udp->val.src_port = spec->hdr.src_port;
 297         if (!mask)
 298                 mask = default_mask;
 299         udp->mask.dst_port = mask->hdr.dst_port;
 300         udp->mask.src_port = mask->hdr.src_port;
 301         /* Remove unwanted bits from values. */
 302         udp->val.src_port &= udp->mask.src_port;
 303         udp->val.dst_port &= udp->mask.dst_port;
 304         return 0;
 305 }
 306
 307 /**
 308  * Convert TCP item to Verbs specification.
 309  *
 310  * @param item[in]
 311  *   Item specification.
 312  * @param default_mask[in]
 313  *   Default bit-masks to use when item->mask is not provided.
 314  * @param data[in, out]
 315  *   User structure.
 316  */
 317 static int
 318 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 319                      const void *default_mask,
 320                      void *data)
 321 {
 322         const struct rte_flow_item_tcp *spec = item->spec;
 323         const struct rte_flow_item_tcp *mask = item->mask;
 324         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 325         struct ibv_flow_spec_tcp_udp *tcp;
 326         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 327
 328         ++flow->ibv_attr->num_of_specs;
 329         flow->ibv_attr->priority = 0;
 330         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 331         *tcp = (struct ibv_flow_spec_tcp_udp) {
 332                 .type = IBV_FLOW_SPEC_TCP,
 333                 .size = tcp_size,
 334         };
 335         if (!spec)
 336                 return 0;
 337         tcp->val.dst_port = spec->hdr.dst_port;
 338         tcp->val.src_port = spec->hdr.src_port;
 339         if (!mask)
 340                 mask = default_mask;
 341         tcp->mask.dst_port = mask->hdr.dst_port;
 342         tcp->mask.src_port = mask->hdr.src_port;
 343         /* Remove unwanted bits from values. */
 344         tcp->val.src_port &= tcp->mask.src_port;
 345         tcp->val.dst_port &= tcp->mask.dst_port;
 346         return 0;
 347 }
 348
 349 /**
 350  * Check support for a given item.
 351  *
 352  * @param item[in]
 353  *   Item specification.
 354  * @param mask[in]
 355  *   Bit-masks covering supported fields to compare with spec, last and mask in
 356  *   \item.
 357  * @param size
 358  *   Bit-Mask size in bytes.
 359  *
 360  * @return
 361  *   0 on success, negative value otherwise.
 362  */
 363 static int
 364 mlx4_flow_item_validate(const struct rte_flow_item *item,
 365                         const uint8_t *mask, unsigned int size)
 366 {
 367         int ret = 0;
 368
 369         if (!item->spec && (item->mask || item->last))
 370                 return -1;
 371         if (item->spec && !item->mask) {
 372                 unsigned int i;
 373                 const uint8_t *spec = item->spec;
 374
 375                 for (i = 0; i < size; ++i)
 376                         if ((spec[i] | mask[i]) != mask[i])
 377                                 return -1;
 378         }
 379         if (item->last && !item->mask) {
 380                 unsigned int i;
 381                 const uint8_t *spec = item->last;
 382
 383                 for (i = 0; i < size; ++i)
 384                         if ((spec[i] | mask[i]) != mask[i])
 385                                 return -1;
 386         }
 387         if (item->spec && item->last) {
 388                 uint8_t spec[size];
 389                 uint8_t last[size];
 390                 const uint8_t *apply = mask;
 391                 unsigned int i;
 392
 393                 if (item->mask)
 394                         apply = item->mask;
 395                 for (i = 0; i < size; ++i) {
 396                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 397                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 398                 }
 399                 ret = memcmp(spec, last, size);
 400         }
 401         return ret;
 402 }
 403
 404 static int
 405 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 406                        const uint8_t *mask, unsigned int size)
 407 {
 408         if (item->mask) {
 409                 const struct rte_flow_item_eth *mask = item->mask;
 410
 411                 if (mask->dst.addr_bytes[0] != 0xff ||
 412                                 mask->dst.addr_bytes[1] != 0xff ||
 413                                 mask->dst.addr_bytes[2] != 0xff ||
 414                                 mask->dst.addr_bytes[3] != 0xff ||
 415                                 mask->dst.addr_bytes[4] != 0xff ||
 416                                 mask->dst.addr_bytes[5] != 0xff)
 417                         return -1;
 418         }
 419         return mlx4_flow_item_validate(item, mask, size);
 420 }
 421
 422 static int
 423 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 424                         const uint8_t *mask, unsigned int size)
 425 {
 426         if (item->mask) {
 427                 const struct rte_flow_item_vlan *mask = item->mask;
 428
 429                 if (mask->tci != 0 &&
 430                     ntohs(mask->tci) != 0x0fff)
 431                         return -1;
 432         }
 433         return mlx4_flow_item_validate(item, mask, size);
 434 }
 435
 436 static int
 437 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 438                         const uint8_t *mask, unsigned int size)
 439 {
 440         if (item->mask) {
 441                 const struct rte_flow_item_ipv4 *mask = item->mask;
 442
 443                 if (mask->hdr.src_addr != 0 &&
 444                     mask->hdr.src_addr != 0xffffffff)
 445                         return -1;
 446                 if (mask->hdr.dst_addr != 0 &&
 447                     mask->hdr.dst_addr != 0xffffffff)
 448                         return -1;
 449         }
 450         return mlx4_flow_item_validate(item, mask, size);
 451 }
 452
 453 static int
 454 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 455                        const uint8_t *mask, unsigned int size)
 456 {
 457         if (item->mask) {
 458                 const struct rte_flow_item_udp *mask = item->mask;
 459
 460                 if (mask->hdr.src_port != 0 &&
 461                     mask->hdr.src_port != 0xffff)
 462                         return -1;
 463                 if (mask->hdr.dst_port != 0 &&
 464                     mask->hdr.dst_port != 0xffff)
 465                         return -1;
 466         }
 467         return mlx4_flow_item_validate(item, mask, size);
 468 }
 469
 470 static int
 471 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 472                        const uint8_t *mask, unsigned int size)
 473 {
 474         if (item->mask) {
 475                 const struct rte_flow_item_tcp *mask = item->mask;
 476
 477                 if (mask->hdr.src_port != 0 &&
 478                     mask->hdr.src_port != 0xffff)
 479                         return -1;
 480                 if (mask->hdr.dst_port != 0 &&
 481                     mask->hdr.dst_port != 0xffff)
 482                         return -1;
 483         }
 484         return mlx4_flow_item_validate(item, mask, size);
 485 }
 486
 487 /** Graph of supported items and associated actions. */
 488 static const struct mlx4_flow_items mlx4_flow_items[] = {
 489         [RTE_FLOW_ITEM_TYPE_END] = {
 490                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 491         },
 492         [RTE_FLOW_ITEM_TYPE_ETH] = {
 493                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 494                                RTE_FLOW_ITEM_TYPE_IPV4),
 495                 .actions = valid_actions,
 496                 .mask = &(const struct rte_flow_item_eth){
 497                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 498                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 499                 },
 500                 .default_mask = &rte_flow_item_eth_mask,
 501                 .mask_sz = sizeof(struct rte_flow_item_eth),
 502                 .validate = mlx4_flow_validate_eth,
 503                 .convert = mlx4_flow_create_eth,
 504                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 505         },
 506         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 507                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 508                 .actions = valid_actions,
 509                 .mask = &(const struct rte_flow_item_vlan){
 510                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 511 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 512                         .tci = 0x0fff,
 513 #else
 514                         .tci = 0xff0f,
 515 #endif
 516                 },
 517                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 518                 .validate = mlx4_flow_validate_vlan,
 519                 .convert = mlx4_flow_create_vlan,
 520                 .dst_sz = 0,
 521         },
 522         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 523                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 524                                RTE_FLOW_ITEM_TYPE_TCP),
 525                 .actions = valid_actions,
 526                 .mask = &(const struct rte_flow_item_ipv4){
 527                         .hdr = {
 528                                 .src_addr = -1,
 529                                 .dst_addr = -1,
 530                         },
 531                 },
 532                 .default_mask = &rte_flow_item_ipv4_mask,
 533                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 534                 .validate = mlx4_flow_validate_ipv4,
 535                 .convert = mlx4_flow_create_ipv4,
 536                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 537         },
 538         [RTE_FLOW_ITEM_TYPE_UDP] = {
 539                 .actions = valid_actions,
 540                 .mask = &(const struct rte_flow_item_udp){
 541                         .hdr = {
 542                                 .src_port = -1,
 543                                 .dst_port = -1,
 544                         },
 545                 },
 546                 .default_mask = &rte_flow_item_udp_mask,
 547                 .mask_sz = sizeof(struct rte_flow_item_udp),
 548                 .validate = mlx4_flow_validate_udp,
 549                 .convert = mlx4_flow_create_udp,
 550                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 551         },
 552         [RTE_FLOW_ITEM_TYPE_TCP] = {
 553                 .actions = valid_actions,
 554                 .mask = &(const struct rte_flow_item_tcp){
 555                         .hdr = {
 556                                 .src_port = -1,
 557                                 .dst_port = -1,
 558                         },
 559                 },
 560                 .default_mask = &rte_flow_item_tcp_mask,
 561                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 562                 .validate = mlx4_flow_validate_tcp,
 563                 .convert = mlx4_flow_create_tcp,
 564                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 565         },
 566 };
 567
 568 /**
 569  * Make sure a flow rule is supported and initialize associated structure.
 570  *
 571  * @param priv
 572  *   Pointer to private structure.
 573  * @param[in] attr
 574  *   Flow rule attributes.
 575  * @param[in] items
 576  *   Pattern specification (list terminated by the END pattern item).
 577  * @param[in] actions
 578  *   Associated actions (list terminated by the END action).
 579  * @param[out] error
 580  *   Perform verbose error reporting if not NULL.
 581  * @param[in, out] flow
 582  *   Flow structure to update.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 mlx4_flow_prepare(struct priv *priv,
 589                   const struct rte_flow_attr *attr,
 590                   const struct rte_flow_item items[],
 591                   const struct rte_flow_action actions[],
 592                   struct rte_flow_error *error,
 593                   struct mlx4_flow *flow)
 594 {
 595         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 596         struct mlx4_flow_action action = {
 597                 .queue = 0,
 598                 .drop = 0,
 599         };
 600         uint32_t priority_override = 0;
 601
 602         if (attr->group) {
 603                 rte_flow_error_set(error, ENOTSUP,
 604                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 605                                    NULL,
 606                                    "groups are not supported");
 607                 return -rte_errno;
 608         }
 609         if (priv->isolated) {
 610                 priority_override = attr->priority;
 611         } else if (attr->priority) {
 612                 rte_flow_error_set(error, ENOTSUP,
 613                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 614                                    NULL,
 615                                    "priorities are not supported outside"
 616                                    " isolated mode");
 617                 return -rte_errno;
 618         }
 619         if (attr->priority > MLX4_FLOW_PRIORITY_LAST) {
 620                 rte_flow_error_set(error, ENOTSUP,
 621                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 622                                    NULL,
 623                                    "maximum priority level is "
 624                                    MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST));
 625                 return -rte_errno;
 626         }
 627         if (attr->egress) {
 628                 rte_flow_error_set(error, ENOTSUP,
 629                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 630                                    NULL,
 631                                    "egress is not supported");
 632                 return -rte_errno;
 633         }
 634         if (!attr->ingress) {
 635                 rte_flow_error_set(error, ENOTSUP,
 636                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 637                                    NULL,
 638                                    "only ingress is supported");
 639                 return -rte_errno;
 640         }
 641         /* Go over items list. */
 642         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 643                 const struct mlx4_flow_items *token = NULL;
 644                 unsigned int i;
 645                 int err;
 646
 647                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 648                         continue;
 649                 /*
 650                  * The nic can support patterns with NULL eth spec only
 651                  * if eth is a single item in a rule.
 652                  */
 653                 if (!items->spec &&
 654                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 655                         const struct rte_flow_item *next = items + 1;
 656
 657                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 658                                 rte_flow_error_set(error, ENOTSUP,
 659                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 660                                                    items,
 661                                                    "the rule requires"
 662                                                    " an Ethernet spec");
 663                                 return -rte_errno;
 664                         }
 665                 }
 666                 for (i = 0;
 667                      cur_item->items &&
 668                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 669                      ++i) {
 670                         if (cur_item->items[i] == items->type) {
 671                                 token = &mlx4_flow_items[items->type];
 672                                 break;
 673                         }
 674                 }
 675                 if (!token)
 676                         goto exit_item_not_supported;
 677                 cur_item = token;
 678                 err = cur_item->validate(items,
 679                                         (const uint8_t *)cur_item->mask,
 680                                          cur_item->mask_sz);
 681                 if (err)
 682                         goto exit_item_not_supported;
 683                 if (flow->ibv_attr && cur_item->convert) {
 684                         err = cur_item->convert(items,
 685                                                 (cur_item->default_mask ?
 686                                                  cur_item->default_mask :
 687                                                  cur_item->mask),
 688                                                  flow);
 689                         if (err)
 690                                 goto exit_item_not_supported;
 691                 }
 692                 flow->offset += cur_item->dst_sz;
 693         }
 694         /* Use specified priority level when in isolated mode. */
 695         if (priv->isolated && flow->ibv_attr)
 696                 flow->ibv_attr->priority = priority_override;
 697         /* Go over actions list */
 698         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 699                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 700                         continue;
 701                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 702                         action.drop = 1;
 703                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 704                         const struct rte_flow_action_queue *queue =
 705                                 (const struct rte_flow_action_queue *)
 706                                 actions->conf;
 707
 708                         if (!queue || (queue->index >
 709                                        (priv->dev->data->nb_rx_queues - 1)))
 710                                 goto exit_action_not_supported;
 711                         action.queue = 1;
 712                 } else {
 713                         goto exit_action_not_supported;
 714                 }
 715         }
 716         if (!action.queue && !action.drop) {
 717                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 718                                    NULL, "no valid action");
 719                 return -rte_errno;
 720         }
 721         return 0;
 722 exit_item_not_supported:
 723         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 724                            items, "item not supported");
 725         return -rte_errno;
 726 exit_action_not_supported:
 727         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 728                            actions, "action not supported");
 729         return -rte_errno;
 730 }
 731
 732 /**
 733  * Validate a flow supported by the NIC.
 734  *
 735  * @see rte_flow_validate()
 736  * @see rte_flow_ops
 737  */
 738 static int
 739 mlx4_flow_validate(struct rte_eth_dev *dev,
 740                    const struct rte_flow_attr *attr,
 741                    const struct rte_flow_item items[],
 742                    const struct rte_flow_action actions[],
 743                    struct rte_flow_error *error)
 744 {
 745         struct priv *priv = dev->data->dev_private;
 746         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 747
 748         return mlx4_flow_prepare(priv, attr, items, actions, error, &flow);
 749 }
 750
 751 /**
 752  * Destroy a drop queue.
 753  *
 754  * @param priv
 755  *   Pointer to private structure.
 756  */
 757 static void
 758 mlx4_flow_destroy_drop_queue(struct priv *priv)
 759 {
 760         if (priv->flow_drop_queue) {
 761                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 762
 763                 priv->flow_drop_queue = NULL;
 764                 claim_zero(ibv_destroy_qp(fdq->qp));
 765                 claim_zero(ibv_destroy_cq(fdq->cq));
 766                 rte_free(fdq);
 767         }
 768 }
 769
 770 /**
 771  * Create a single drop queue for all drop flows.
 772  *
 773  * @param priv
 774  *   Pointer to private structure.
 775  *
 776  * @return
 777  *   0 on success, negative value otherwise.
 778  */
 779 static int
 780 mlx4_flow_create_drop_queue(struct priv *priv)
 781 {
 782         struct ibv_qp *qp;
 783         struct ibv_cq *cq;
 784         struct rte_flow_drop *fdq;
 785
 786         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 787         if (!fdq) {
 788                 ERROR("Cannot allocate memory for drop struct");
 789                 goto err;
 790         }
 791         cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 792         if (!cq) {
 793                 ERROR("Cannot create drop CQ");
 794                 goto err_create_cq;
 795         }
 796         qp = ibv_create_qp(priv->pd,
 797                            &(struct ibv_qp_init_attr){
 798                                 .send_cq = cq,
 799                                 .recv_cq = cq,
 800                                 .cap = {
 801                                         .max_recv_wr = 1,
 802                                         .max_recv_sge = 1,
 803                                 },
 804                                 .qp_type = IBV_QPT_RAW_PACKET,
 805                            });
 806         if (!qp) {
 807                 ERROR("Cannot create drop QP");
 808                 goto err_create_qp;
 809         }
 810         *fdq = (struct rte_flow_drop){
 811                 .qp = qp,
 812                 .cq = cq,
 813         };
 814         priv->flow_drop_queue = fdq;
 815         return 0;
 816 err_create_qp:
 817         claim_zero(ibv_destroy_cq(cq));
 818 err_create_cq:
 819         rte_free(fdq);
 820 err:
 821         return -1;
 822 }
 823
 824 /**
 825  * Complete flow rule creation.
 826  *
 827  * @param priv
 828  *   Pointer to private structure.
 829  * @param ibv_attr
 830  *   Verbs flow attributes.
 831  * @param action
 832  *   Target action structure.
 833  * @param[out] error
 834  *   Perform verbose error reporting if not NULL.
 835  *
 836  * @return
 837  *   A flow if the rule could be created.
 838  */
 839 static struct rte_flow *
 840 mlx4_flow_create_action_queue(struct priv *priv,
 841                               struct ibv_flow_attr *ibv_attr,
 842                               struct mlx4_flow_action *action,
 843                               struct rte_flow_error *error)
 844 {
 845         struct ibv_qp *qp;
 846         struct rte_flow *rte_flow;
 847
 848         assert(priv->pd);
 849         assert(priv->ctx);
 850         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 851         if (!rte_flow) {
 852                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 853                                    NULL, "cannot allocate flow memory");
 854                 return NULL;
 855         }
 856         if (action->drop) {
 857                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 858         } else {
 859                 struct rxq *rxq = priv->dev->data->rx_queues[action->queue_id];
 860
 861                 qp = rxq->qp;
 862                 rte_flow->qp = qp;
 863         }
 864         rte_flow->ibv_attr = ibv_attr;
 865         if (!priv->started)
 866                 return rte_flow;
 867         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
 868         if (!rte_flow->ibv_flow) {
 869                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 870                                    NULL, "flow rule creation failure");
 871                 goto error;
 872         }
 873         return rte_flow;
 874 error:
 875         rte_free(rte_flow);
 876         return NULL;
 877 }
 878
 879 /**
 880  * Create a flow.
 881  *
 882  * @see rte_flow_create()
 883  * @see rte_flow_ops
 884  */
 885 static struct rte_flow *
 886 mlx4_flow_create(struct rte_eth_dev *dev,
 887                  const struct rte_flow_attr *attr,
 888                  const struct rte_flow_item items[],
 889                  const struct rte_flow_action actions[],
 890                  struct rte_flow_error *error)
 891 {
 892         struct priv *priv = dev->data->dev_private;
 893         struct rte_flow *rte_flow;
 894         struct mlx4_flow_action action;
 895         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
 896         int err;
 897
 898         err = mlx4_flow_prepare(priv, attr, items, actions, error, &flow);
 899         if (err)
 900                 return NULL;
 901         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
 902         if (!flow.ibv_attr) {
 903                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 904                                    NULL, "cannot allocate ibv_attr memory");
 905                 return NULL;
 906         }
 907         flow.offset = sizeof(struct ibv_flow_attr);
 908         *flow.ibv_attr = (struct ibv_flow_attr){
 909                 .comp_mask = 0,
 910                 .type = IBV_FLOW_ATTR_NORMAL,
 911                 .size = sizeof(struct ibv_flow_attr),
 912                 .priority = attr->priority,
 913                 .num_of_specs = 0,
 914                 .port = priv->port,
 915                 .flags = 0,
 916         };
 917         claim_zero(mlx4_flow_prepare(priv, attr, items, actions,
 918                                      error, &flow));
 919         action = (struct mlx4_flow_action){
 920                 .queue = 0,
 921                 .drop = 0,
 922         };
 923         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 924                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 925                         continue;
 926                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 927                         action.queue = 1;
 928                         action.queue_id =
 929                                 ((const struct rte_flow_action_queue *)
 930                                  actions->conf)->index;
 931                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 932                         action.drop = 1;
 933                 } else {
 934                         rte_flow_error_set(error, ENOTSUP,
 935                                            RTE_FLOW_ERROR_TYPE_ACTION,
 936                                            actions, "unsupported action");
 937                         goto exit;
 938                 }
 939         }
 940         rte_flow = mlx4_flow_create_action_queue(priv, flow.ibv_attr,
 941                                                  &action, error);
 942         if (rte_flow) {
 943                 LIST_INSERT_HEAD(&priv->flows, rte_flow, next);
 944                 DEBUG("Flow created %p", (void *)rte_flow);
 945                 return rte_flow;
 946         }
 947 exit:
 948         rte_free(flow.ibv_attr);
 949         return NULL;
 950 }
 951
 952 /**
 953  * Configure isolated mode.
 954  *
 955  * @see rte_flow_isolate()
 956  * @see rte_flow_ops
 957  */
 958 static int
 959 mlx4_flow_isolate(struct rte_eth_dev *dev,
 960                   int enable,
 961                   struct rte_flow_error *error)
 962 {
 963         struct priv *priv = dev->data->dev_private;
 964
 965         if (!!enable == !!priv->isolated)
 966                 return 0;
 967         priv->isolated = !!enable;
 968         if (enable) {
 969                 mlx4_mac_addr_del(priv);
 970         } else if (mlx4_mac_addr_add(priv) < 0) {
 971                 priv->isolated = 1;
 972                 return rte_flow_error_set(error, rte_errno,
 973                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 974                                           NULL, "cannot leave isolated mode");
 975         }
 976         return 0;
 977 }
 978
 979 /**
 980  * Destroy a flow.
 981  *
 982  * @see rte_flow_destroy()
 983  * @see rte_flow_ops
 984  */
 985 static int
 986 mlx4_flow_destroy(struct rte_eth_dev *dev,
 987                   struct rte_flow *flow,
 988                   struct rte_flow_error *error)
 989 {
 990         (void)dev;
 991         (void)error;
 992         LIST_REMOVE(flow, next);
 993         if (flow->ibv_flow)
 994                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 995         rte_free(flow->ibv_attr);
 996         DEBUG("Flow destroyed %p", (void *)flow);
 997         rte_free(flow);
 998         return 0;
 999 }
1000
1001 /**
1002  * Destroy all flows.
1003  *
1004  * @see rte_flow_flush()
1005  * @see rte_flow_ops
1006  */
1007 static int
1008 mlx4_flow_flush(struct rte_eth_dev *dev,
1009                 struct rte_flow_error *error)
1010 {
1011         struct priv *priv = dev->data->dev_private;
1012
1013         while (!LIST_EMPTY(&priv->flows)) {
1014                 struct rte_flow *flow;
1015
1016                 flow = LIST_FIRST(&priv->flows);
1017                 mlx4_flow_destroy(dev, flow, error);
1018         }
1019         return 0;
1020 }
1021
1022 /**
1023  * Remove all flows.
1024  *
1025  * Called by dev_stop() to remove all flows.
1026  *
1027  * @param priv
1028  *   Pointer to private structure.
1029  */
1030 void
1031 mlx4_flow_stop(struct priv *priv)
1032 {
1033         struct rte_flow *flow;
1034
1035         for (flow = LIST_FIRST(&priv->flows);
1036              flow;
1037              flow = LIST_NEXT(flow, next)) {
1038                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1039                 flow->ibv_flow = NULL;
1040                 DEBUG("Flow %p removed", (void *)flow);
1041         }
1042         mlx4_flow_destroy_drop_queue(priv);
1043 }
1044
1045 /**
1046  * Add all flows.
1047  *
1048  * @param priv
1049  *   Pointer to private structure.
1050  *
1051  * @return
1052  *   0 on success, a errno value otherwise and rte_errno is set.
1053  */
1054 int
1055 mlx4_flow_start(struct priv *priv)
1056 {
1057         int ret;
1058         struct ibv_qp *qp;
1059         struct rte_flow *flow;
1060
1061         ret = mlx4_flow_create_drop_queue(priv);
1062         if (ret)
1063                 return -1;
1064         for (flow = LIST_FIRST(&priv->flows);
1065              flow;
1066              flow = LIST_NEXT(flow, next)) {
1067                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1068                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1069                 if (!flow->ibv_flow) {
1070                         DEBUG("Flow %p cannot be applied", (void *)flow);
1071                         rte_errno = EINVAL;
1072                         return rte_errno;
1073                 }
1074                 DEBUG("Flow %p applied", (void *)flow);
1075         }
1076         return 0;
1077 }
1078
1079 static const struct rte_flow_ops mlx4_flow_ops = {
1080         .validate = mlx4_flow_validate,
1081         .create = mlx4_flow_create,
1082         .destroy = mlx4_flow_destroy,
1083         .flush = mlx4_flow_flush,
1084         .isolate = mlx4_flow_isolate,
1085 };
1086
1087 /**
1088  * Manage filter operations.
1089  *
1090  * @param dev
1091  *   Pointer to Ethernet device structure.
1092  * @param filter_type
1093  *   Filter type.
1094  * @param filter_op
1095  *   Operation to perform.
1096  * @param arg
1097  *   Pointer to operation-specific structure.
1098  *
1099  * @return
1100  *   0 on success, negative errno value otherwise and rte_errno is set.
1101  */
1102 int
1103 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1104                  enum rte_filter_type filter_type,
1105                  enum rte_filter_op filter_op,
1106                  void *arg)
1107 {
1108         switch (filter_type) {
1109         case RTE_ETH_FILTER_GENERIC:
1110                 if (filter_op != RTE_ETH_FILTER_GET)
1111                         break;
1112                 *(const void **)arg = &mlx4_flow_ops;
1113                 return 0;
1114         default:
1115                 ERROR("%p: filter type (%d) not supported",
1116                       (void *)dev, filter_type);
1117                 break;
1118         }
1119         rte_errno = ENOTSUP;
1120         return -rte_errno;
1121 }