drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Flow API operations for mlx4 driver.
  37  */
  38
  39 #include <arpa/inet.h>
  40 #include <assert.h>
  41 #include <errno.h>
  42 #include <stdalign.h>
  43 #include <stddef.h>
  44 #include <stdint.h>
  45 #include <string.h>
  46 #include <sys/queue.h>
  47
  48 /* Verbs headers do not support -pedantic. */
  49 #ifdef PEDANTIC
  50 #pragma GCC diagnostic ignored "-Wpedantic"
  51 #endif
  52 #include <infiniband/verbs.h>
  53 #ifdef PEDANTIC
  54 #pragma GCC diagnostic error "-Wpedantic"
  55 #endif
  56
  57 #include <rte_byteorder.h>
  58 #include <rte_errno.h>
  59 #include <rte_eth_ctrl.h>
  60 #include <rte_ethdev.h>
  61 #include <rte_flow.h>
  62 #include <rte_flow_driver.h>
  63 #include <rte_malloc.h>
  64
  65 /* PMD headers. */
  66 #include "mlx4.h"
  67 #include "mlx4_flow.h"
  68 #include "mlx4_rxtx.h"
  69 #include "mlx4_utils.h"
  70
  71 /** Static initializer for a list of subsequent item types. */
  72 #define NEXT_ITEM(...) \
  73         (const enum rte_flow_item_type []){ \
  74                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  75         }
  76
  77 /** Processor structure associated with a flow item. */
  78 struct mlx4_flow_proc_item {
  79         /** Bit-masks corresponding to the possibilities for the item. */
  80         const void *mask;
  81         /**
  82          * Default bit-masks to use when item->mask is not provided. When
  83          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  84          * used instead.
  85          */
  86         const void *default_mask;
  87         /** Bit-masks size in bytes. */
  88         const unsigned int mask_sz;
  89         /**
  90          * Check support for a given item.
  91          *
  92          * @param item[in]
  93          *   Item specification.
  94          * @param mask[in]
  95          *   Bit-masks covering supported fields to compare with spec,
  96          *   last and mask in
  97          *   \item.
  98          * @param size
  99          *   Bit-Mask size in bytes.
 100          *
 101          * @return
 102          *   0 on success, negative value otherwise.
 103          */
 104         int (*validate)(const struct rte_flow_item *item,
 105                         const uint8_t *mask, unsigned int size);
 106         /**
 107          * Conversion function from rte_flow to NIC specific flow.
 108          *
 109          * @param item
 110          *   rte_flow item to convert.
 111          * @param default_mask
 112          *   Default bit-masks to use when item->mask is not provided.
 113          * @param flow
 114          *   Flow rule handle to update.
 115          *
 116          * @return
 117          *   0 on success, negative value otherwise.
 118          */
 119         int (*convert)(const struct rte_flow_item *item,
 120                        const void *default_mask,
 121                        struct rte_flow *flow);
 122         /** Size in bytes of the destination structure. */
 123         const unsigned int dst_sz;
 124         /** List of possible subsequent items. */
 125         const enum rte_flow_item_type *const next_item;
 126 };
 127
 128 /** Shared resources for drop flow rules. */
 129 struct mlx4_drop {
 130         struct ibv_qp *qp; /**< QP target. */
 131         struct ibv_cq *cq; /**< CQ associated with above QP. */
 132         struct priv *priv; /**< Back pointer to private data. */
 133         uint32_t refcnt; /**< Reference count. */
 134 };
 135
 136 /**
 137  * Convert Ethernet item to Verbs specification.
 138  *
 139  * @param item[in]
 140  *   Item specification.
 141  * @param default_mask[in]
 142  *   Default bit-masks to use when item->mask is not provided.
 143  * @param flow[in, out]
 144  *   Flow rule handle to update.
 145  */
 146 static int
 147 mlx4_flow_create_eth(const struct rte_flow_item *item,
 148                      const void *default_mask,
 149                      struct rte_flow *flow)
 150 {
 151         const struct rte_flow_item_eth *spec = item->spec;
 152         const struct rte_flow_item_eth *mask = item->mask;
 153         struct ibv_flow_spec_eth *eth;
 154         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 155         unsigned int i;
 156
 157         ++flow->ibv_attr->num_of_specs;
 158         flow->ibv_attr->priority = 2;
 159         eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 160         *eth = (struct ibv_flow_spec_eth) {
 161                 .type = IBV_FLOW_SPEC_ETH,
 162                 .size = eth_size,
 163         };
 164         if (!spec) {
 165                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 166                 return 0;
 167         }
 168         if (!mask)
 169                 mask = default_mask;
 170         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 171         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 172         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 173         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 174         /* Remove unwanted bits from values. */
 175         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 176                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 177                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 178         }
 179         return 0;
 180 }
 181
 182 /**
 183  * Convert VLAN item to Verbs specification.
 184  *
 185  * @param item[in]
 186  *   Item specification.
 187  * @param default_mask[in]
 188  *   Default bit-masks to use when item->mask is not provided.
 189  * @param flow[in, out]
 190  *   Flow rule handle to update.
 191  */
 192 static int
 193 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 194                       const void *default_mask,
 195                       struct rte_flow *flow)
 196 {
 197         const struct rte_flow_item_vlan *spec = item->spec;
 198         const struct rte_flow_item_vlan *mask = item->mask;
 199         struct ibv_flow_spec_eth *eth;
 200         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 201
 202         eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size -
 203                        eth_size);
 204         if (!spec)
 205                 return 0;
 206         if (!mask)
 207                 mask = default_mask;
 208         eth->val.vlan_tag = spec->tci;
 209         eth->mask.vlan_tag = mask->tci;
 210         eth->val.vlan_tag &= eth->mask.vlan_tag;
 211         return 0;
 212 }
 213
 214 /**
 215  * Convert IPv4 item to Verbs specification.
 216  *
 217  * @param item[in]
 218  *   Item specification.
 219  * @param default_mask[in]
 220  *   Default bit-masks to use when item->mask is not provided.
 221  * @param flow[in, out]
 222  *   Flow rule handle to update.
 223  */
 224 static int
 225 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 226                       const void *default_mask,
 227                       struct rte_flow *flow)
 228 {
 229         const struct rte_flow_item_ipv4 *spec = item->spec;
 230         const struct rte_flow_item_ipv4 *mask = item->mask;
 231         struct ibv_flow_spec_ipv4 *ipv4;
 232         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 233
 234         ++flow->ibv_attr->num_of_specs;
 235         flow->ibv_attr->priority = 1;
 236         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 237         *ipv4 = (struct ibv_flow_spec_ipv4) {
 238                 .type = IBV_FLOW_SPEC_IPV4,
 239                 .size = ipv4_size,
 240         };
 241         if (!spec)
 242                 return 0;
 243         ipv4->val = (struct ibv_flow_ipv4_filter) {
 244                 .src_ip = spec->hdr.src_addr,
 245                 .dst_ip = spec->hdr.dst_addr,
 246         };
 247         if (!mask)
 248                 mask = default_mask;
 249         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 250                 .src_ip = mask->hdr.src_addr,
 251                 .dst_ip = mask->hdr.dst_addr,
 252         };
 253         /* Remove unwanted bits from values. */
 254         ipv4->val.src_ip &= ipv4->mask.src_ip;
 255         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 256         return 0;
 257 }
 258
 259 /**
 260  * Convert UDP item to Verbs specification.
 261  *
 262  * @param item[in]
 263  *   Item specification.
 264  * @param default_mask[in]
 265  *   Default bit-masks to use when item->mask is not provided.
 266  * @param flow[in, out]
 267  *   Flow rule handle to update.
 268  */
 269 static int
 270 mlx4_flow_create_udp(const struct rte_flow_item *item,
 271                      const void *default_mask,
 272                      struct rte_flow *flow)
 273 {
 274         const struct rte_flow_item_udp *spec = item->spec;
 275         const struct rte_flow_item_udp *mask = item->mask;
 276         struct ibv_flow_spec_tcp_udp *udp;
 277         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 278
 279         ++flow->ibv_attr->num_of_specs;
 280         flow->ibv_attr->priority = 0;
 281         udp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 282         *udp = (struct ibv_flow_spec_tcp_udp) {
 283                 .type = IBV_FLOW_SPEC_UDP,
 284                 .size = udp_size,
 285         };
 286         if (!spec)
 287                 return 0;
 288         udp->val.dst_port = spec->hdr.dst_port;
 289         udp->val.src_port = spec->hdr.src_port;
 290         if (!mask)
 291                 mask = default_mask;
 292         udp->mask.dst_port = mask->hdr.dst_port;
 293         udp->mask.src_port = mask->hdr.src_port;
 294         /* Remove unwanted bits from values. */
 295         udp->val.src_port &= udp->mask.src_port;
 296         udp->val.dst_port &= udp->mask.dst_port;
 297         return 0;
 298 }
 299
 300 /**
 301  * Convert TCP item to Verbs specification.
 302  *
 303  * @param item[in]
 304  *   Item specification.
 305  * @param default_mask[in]
 306  *   Default bit-masks to use when item->mask is not provided.
 307  * @param flow[in, out]
 308  *   Flow rule handle to update.
 309  */
 310 static int
 311 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 312                      const void *default_mask,
 313                      struct rte_flow *flow)
 314 {
 315         const struct rte_flow_item_tcp *spec = item->spec;
 316         const struct rte_flow_item_tcp *mask = item->mask;
 317         struct ibv_flow_spec_tcp_udp *tcp;
 318         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 319
 320         ++flow->ibv_attr->num_of_specs;
 321         flow->ibv_attr->priority = 0;
 322         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 323         *tcp = (struct ibv_flow_spec_tcp_udp) {
 324                 .type = IBV_FLOW_SPEC_TCP,
 325                 .size = tcp_size,
 326         };
 327         if (!spec)
 328                 return 0;
 329         tcp->val.dst_port = spec->hdr.dst_port;
 330         tcp->val.src_port = spec->hdr.src_port;
 331         if (!mask)
 332                 mask = default_mask;
 333         tcp->mask.dst_port = mask->hdr.dst_port;
 334         tcp->mask.src_port = mask->hdr.src_port;
 335         /* Remove unwanted bits from values. */
 336         tcp->val.src_port &= tcp->mask.src_port;
 337         tcp->val.dst_port &= tcp->mask.dst_port;
 338         return 0;
 339 }
 340
 341 /**
 342  * Check support for a given item.
 343  *
 344  * @param item[in]
 345  *   Item specification.
 346  * @param mask[in]
 347  *   Bit-masks covering supported fields to compare with spec, last and mask in
 348  *   \item.
 349  * @param size
 350  *   Bit-Mask size in bytes.
 351  *
 352  * @return
 353  *   0 on success, negative value otherwise.
 354  */
 355 static int
 356 mlx4_flow_item_validate(const struct rte_flow_item *item,
 357                         const uint8_t *mask, unsigned int size)
 358 {
 359         int ret = 0;
 360
 361         if (!item->spec && (item->mask || item->last))
 362                 return -1;
 363         if (item->spec && !item->mask) {
 364                 unsigned int i;
 365                 const uint8_t *spec = item->spec;
 366
 367                 for (i = 0; i < size; ++i)
 368                         if ((spec[i] | mask[i]) != mask[i])
 369                                 return -1;
 370         }
 371         if (item->last && !item->mask) {
 372                 unsigned int i;
 373                 const uint8_t *spec = item->last;
 374
 375                 for (i = 0; i < size; ++i)
 376                         if ((spec[i] | mask[i]) != mask[i])
 377                                 return -1;
 378         }
 379         if (item->spec && item->last) {
 380                 uint8_t spec[size];
 381                 uint8_t last[size];
 382                 const uint8_t *apply = mask;
 383                 unsigned int i;
 384
 385                 if (item->mask)
 386                         apply = item->mask;
 387                 for (i = 0; i < size; ++i) {
 388                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 389                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 390                 }
 391                 ret = memcmp(spec, last, size);
 392         }
 393         return ret;
 394 }
 395
 396 static int
 397 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 398                        const uint8_t *mask, unsigned int size)
 399 {
 400         if (item->mask) {
 401                 const struct rte_flow_item_eth *mask = item->mask;
 402
 403                 if (mask->dst.addr_bytes[0] != 0xff ||
 404                                 mask->dst.addr_bytes[1] != 0xff ||
 405                                 mask->dst.addr_bytes[2] != 0xff ||
 406                                 mask->dst.addr_bytes[3] != 0xff ||
 407                                 mask->dst.addr_bytes[4] != 0xff ||
 408                                 mask->dst.addr_bytes[5] != 0xff)
 409                         return -1;
 410         }
 411         return mlx4_flow_item_validate(item, mask, size);
 412 }
 413
 414 static int
 415 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 416                         const uint8_t *mask, unsigned int size)
 417 {
 418         if (item->mask) {
 419                 const struct rte_flow_item_vlan *mask = item->mask;
 420
 421                 if (mask->tci != 0 &&
 422                     ntohs(mask->tci) != 0x0fff)
 423                         return -1;
 424         }
 425         return mlx4_flow_item_validate(item, mask, size);
 426 }
 427
 428 static int
 429 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 430                         const uint8_t *mask, unsigned int size)
 431 {
 432         if (item->mask) {
 433                 const struct rte_flow_item_ipv4 *mask = item->mask;
 434
 435                 if (mask->hdr.src_addr != 0 &&
 436                     mask->hdr.src_addr != 0xffffffff)
 437                         return -1;
 438                 if (mask->hdr.dst_addr != 0 &&
 439                     mask->hdr.dst_addr != 0xffffffff)
 440                         return -1;
 441         }
 442         return mlx4_flow_item_validate(item, mask, size);
 443 }
 444
 445 static int
 446 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 447                        const uint8_t *mask, unsigned int size)
 448 {
 449         if (item->mask) {
 450                 const struct rte_flow_item_udp *mask = item->mask;
 451
 452                 if (mask->hdr.src_port != 0 &&
 453                     mask->hdr.src_port != 0xffff)
 454                         return -1;
 455                 if (mask->hdr.dst_port != 0 &&
 456                     mask->hdr.dst_port != 0xffff)
 457                         return -1;
 458         }
 459         return mlx4_flow_item_validate(item, mask, size);
 460 }
 461
 462 static int
 463 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 464                        const uint8_t *mask, unsigned int size)
 465 {
 466         if (item->mask) {
 467                 const struct rte_flow_item_tcp *mask = item->mask;
 468
 469                 if (mask->hdr.src_port != 0 &&
 470                     mask->hdr.src_port != 0xffff)
 471                         return -1;
 472                 if (mask->hdr.dst_port != 0 &&
 473                     mask->hdr.dst_port != 0xffff)
 474                         return -1;
 475         }
 476         return mlx4_flow_item_validate(item, mask, size);
 477 }
 478
 479 /** Graph of supported items and associated actions. */
 480 static const struct mlx4_flow_proc_item mlx4_flow_proc_item_list[] = {
 481         [RTE_FLOW_ITEM_TYPE_END] = {
 482                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_ETH),
 483         },
 484         [RTE_FLOW_ITEM_TYPE_ETH] = {
 485                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_VLAN,
 486                                        RTE_FLOW_ITEM_TYPE_IPV4),
 487                 .mask = &(const struct rte_flow_item_eth){
 488                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 489                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 490                 },
 491                 .default_mask = &rte_flow_item_eth_mask,
 492                 .mask_sz = sizeof(struct rte_flow_item_eth),
 493                 .validate = mlx4_flow_validate_eth,
 494                 .convert = mlx4_flow_create_eth,
 495                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 496         },
 497         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 498                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_IPV4),
 499                 .mask = &(const struct rte_flow_item_vlan){
 500                         /* Only TCI VID matching is supported. */
 501                         .tci = RTE_BE16(0x0fff),
 502                 },
 503                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 504                 .validate = mlx4_flow_validate_vlan,
 505                 .convert = mlx4_flow_create_vlan,
 506                 .dst_sz = 0,
 507         },
 508         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 509                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_UDP,
 510                                        RTE_FLOW_ITEM_TYPE_TCP),
 511                 .mask = &(const struct rte_flow_item_ipv4){
 512                         .hdr = {
 513                                 .src_addr = RTE_BE32(0xffffffff),
 514                                 .dst_addr = RTE_BE32(0xffffffff),
 515                         },
 516                 },
 517                 .default_mask = &rte_flow_item_ipv4_mask,
 518                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 519                 .validate = mlx4_flow_validate_ipv4,
 520                 .convert = mlx4_flow_create_ipv4,
 521                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 522         },
 523         [RTE_FLOW_ITEM_TYPE_UDP] = {
 524                 .mask = &(const struct rte_flow_item_udp){
 525                         .hdr = {
 526                                 .src_port = RTE_BE16(0xffff),
 527                                 .dst_port = RTE_BE16(0xffff),
 528                         },
 529                 },
 530                 .default_mask = &rte_flow_item_udp_mask,
 531                 .mask_sz = sizeof(struct rte_flow_item_udp),
 532                 .validate = mlx4_flow_validate_udp,
 533                 .convert = mlx4_flow_create_udp,
 534                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 535         },
 536         [RTE_FLOW_ITEM_TYPE_TCP] = {
 537                 .mask = &(const struct rte_flow_item_tcp){
 538                         .hdr = {
 539                                 .src_port = RTE_BE16(0xffff),
 540                                 .dst_port = RTE_BE16(0xffff),
 541                         },
 542                 },
 543                 .default_mask = &rte_flow_item_tcp_mask,
 544                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 545                 .validate = mlx4_flow_validate_tcp,
 546                 .convert = mlx4_flow_create_tcp,
 547                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 548         },
 549 };
 550
 551 /**
 552  * Make sure a flow rule is supported and initialize associated structure.
 553  *
 554  * @param priv
 555  *   Pointer to private structure.
 556  * @param[in] attr
 557  *   Flow rule attributes.
 558  * @param[in] pattern
 559  *   Pattern specification (list terminated by the END pattern item).
 560  * @param[in] actions
 561  *   Associated actions (list terminated by the END action).
 562  * @param[out] error
 563  *   Perform verbose error reporting if not NULL.
 564  * @param[in, out] addr
 565  *   Buffer where the resulting flow rule handle pointer must be stored.
 566  *   If NULL, stop processing after validation stage.
 567  *
 568  * @return
 569  *   0 on success, a negative errno value otherwise and rte_errno is set.
 570  */
 571 static int
 572 mlx4_flow_prepare(struct priv *priv,
 573                   const struct rte_flow_attr *attr,
 574                   const struct rte_flow_item pattern[],
 575                   const struct rte_flow_action actions[],
 576                   struct rte_flow_error *error,
 577                   struct rte_flow **addr)
 578 {
 579         const struct rte_flow_item *item;
 580         const struct rte_flow_action *action;
 581         const struct mlx4_flow_proc_item *proc;
 582         struct rte_flow temp = { .ibv_attr_size = sizeof(*temp.ibv_attr) };
 583         struct rte_flow *flow = &temp;
 584         uint32_t priority_override = 0;
 585
 586         if (attr->group)
 587                 return rte_flow_error_set
 588                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 589                          NULL, "groups are not supported");
 590         if (priv->isolated)
 591                 priority_override = attr->priority;
 592         else if (attr->priority)
 593                 return rte_flow_error_set
 594                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 595                          NULL,
 596                          "priorities are not supported outside isolated mode");
 597         if (attr->priority > MLX4_FLOW_PRIORITY_LAST)
 598                 return rte_flow_error_set
 599                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 600                          NULL, "maximum priority level is "
 601                          MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST));
 602         if (attr->egress)
 603                 return rte_flow_error_set
 604                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 605                          NULL, "egress is not supported");
 606         if (!attr->ingress)
 607                 return rte_flow_error_set
 608                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 609                          NULL, "only ingress is supported");
 610 fill:
 611         proc = mlx4_flow_proc_item_list;
 612         /* Go over pattern. */
 613         for (item = pattern; item->type; ++item) {
 614                 const struct mlx4_flow_proc_item *next = NULL;
 615                 unsigned int i;
 616                 int err;
 617
 618                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 619                         continue;
 620                 if (item->type == MLX4_FLOW_ITEM_TYPE_INTERNAL) {
 621                         flow->internal = 1;
 622                         continue;
 623                 }
 624                 /*
 625                  * The nic can support patterns with NULL eth spec only
 626                  * if eth is a single item in a rule.
 627                  */
 628                 if (!item->spec && item->type == RTE_FLOW_ITEM_TYPE_ETH) {
 629                         const struct rte_flow_item *next = item + 1;
 630
 631                         if (next->type)
 632                                 return rte_flow_error_set
 633                                         (error, ENOTSUP,
 634                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
 635                                          "the rule requires an Ethernet spec");
 636                 }
 637                 for (i = 0; proc->next_item && proc->next_item[i]; ++i) {
 638                         if (proc->next_item[i] == item->type) {
 639                                 next = &mlx4_flow_proc_item_list[item->type];
 640                                 break;
 641                         }
 642                 }
 643                 if (!next)
 644                         goto exit_item_not_supported;
 645                 proc = next;
 646                 /* Perform validation once, while handle is not allocated. */
 647                 if (flow == &temp) {
 648                         err = proc->validate(item, proc->mask, proc->mask_sz);
 649                         if (err)
 650                                 goto exit_item_not_supported;
 651                 } else if (proc->convert) {
 652                         err = proc->convert(item,
 653                                             (proc->default_mask ?
 654                                              proc->default_mask :
 655                                              proc->mask),
 656                                             flow);
 657                         if (err)
 658                                 goto exit_item_not_supported;
 659                 }
 660                 flow->ibv_attr_size += proc->dst_sz;
 661         }
 662         /* Use specified priority level when in isolated mode. */
 663         if (priv->isolated && flow != &temp)
 664                 flow->ibv_attr->priority = priority_override;
 665         /* Go over actions list. */
 666         for (action = actions; action->type; ++action) {
 667                 switch (action->type) {
 668                         const struct rte_flow_action_queue *queue;
 669
 670                 case RTE_FLOW_ACTION_TYPE_VOID:
 671                         continue;
 672                 case RTE_FLOW_ACTION_TYPE_DROP:
 673                         flow->drop = 1;
 674                         break;
 675                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 676                         queue = action->conf;
 677                         if (queue->index >= priv->dev->data->nb_rx_queues)
 678                                 goto exit_action_not_supported;
 679                         flow->queue = 1;
 680                         flow->queue_id = queue->index;
 681                         break;
 682                 default:
 683                         goto exit_action_not_supported;
 684                 }
 685         }
 686         if (!flow->queue && !flow->drop)
 687                 return rte_flow_error_set
 688                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 689                          NULL, "no valid action");
 690         /* Validation ends here. */
 691         if (!addr)
 692                 return 0;
 693         if (flow == &temp) {
 694                 /* Allocate proper handle based on collected data. */
 695                 const struct mlx4_malloc_vec vec[] = {
 696                         {
 697                                 .align = alignof(struct rte_flow),
 698                                 .size = sizeof(*flow),
 699                                 .addr = (void **)&flow,
 700                         },
 701                         {
 702                                 .align = alignof(struct ibv_flow_attr),
 703                                 .size = temp.ibv_attr_size,
 704                                 .addr = (void **)&temp.ibv_attr,
 705                         },
 706                 };
 707
 708                 if (!mlx4_zmallocv(__func__, vec, RTE_DIM(vec)))
 709                         return rte_flow_error_set
 710                                 (error, -rte_errno,
 711                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 712                                  "flow rule handle allocation failure");
 713                 /* Most fields will be updated by second pass. */
 714                 *flow = (struct rte_flow){
 715                         .ibv_attr = temp.ibv_attr,
 716                         .ibv_attr_size = sizeof(*flow->ibv_attr),
 717                 };
 718                 *flow->ibv_attr = (struct ibv_flow_attr){
 719                         .type = IBV_FLOW_ATTR_NORMAL,
 720                         .size = sizeof(*flow->ibv_attr),
 721                         .port = priv->port,
 722                 };
 723                 goto fill;
 724         }
 725         *addr = flow;
 726         return 0;
 727 exit_item_not_supported:
 728         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 729                                   item, "item not supported");
 730 exit_action_not_supported:
 731         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 732                                   action, "action not supported");
 733 }
 734
 735 /**
 736  * Validate a flow supported by the NIC.
 737  *
 738  * @see rte_flow_validate()
 739  * @see rte_flow_ops
 740  */
 741 static int
 742 mlx4_flow_validate(struct rte_eth_dev *dev,
 743                    const struct rte_flow_attr *attr,
 744                    const struct rte_flow_item pattern[],
 745                    const struct rte_flow_action actions[],
 746                    struct rte_flow_error *error)
 747 {
 748         struct priv *priv = dev->data->dev_private;
 749
 750         return mlx4_flow_prepare(priv, attr, pattern, actions, error, NULL);
 751 }
 752
 753 /**
 754  * Get a drop flow rule resources instance.
 755  *
 756  * @param priv
 757  *   Pointer to private structure.
 758  *
 759  * @return
 760  *   Pointer to drop flow resources on success, NULL otherwise and rte_errno
 761  *   is set.
 762  */
 763 static struct mlx4_drop *
 764 mlx4_drop_get(struct priv *priv)
 765 {
 766         struct mlx4_drop *drop = priv->drop;
 767
 768         if (drop) {
 769                 assert(drop->refcnt);
 770                 assert(drop->priv == priv);
 771                 ++drop->refcnt;
 772                 return drop;
 773         }
 774         drop = rte_malloc(__func__, sizeof(*drop), 0);
 775         if (!drop)
 776                 goto error;
 777         *drop = (struct mlx4_drop){
 778                 .priv = priv,
 779                 .refcnt = 1,
 780         };
 781         drop->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 782         if (!drop->cq)
 783                 goto error;
 784         drop->qp = ibv_create_qp(priv->pd,
 785                                  &(struct ibv_qp_init_attr){
 786                                         .send_cq = drop->cq,
 787                                         .recv_cq = drop->cq,
 788                                         .qp_type = IBV_QPT_RAW_PACKET,
 789                                  });
 790         if (!drop->qp)
 791                 goto error;
 792         priv->drop = drop;
 793         return drop;
 794 error:
 795         if (drop->qp)
 796                 claim_zero(ibv_destroy_qp(drop->qp));
 797         if (drop->cq)
 798                 claim_zero(ibv_destroy_cq(drop->cq));
 799         if (drop)
 800                 rte_free(drop);
 801         rte_errno = ENOMEM;
 802         return NULL;
 803 }
 804
 805 /**
 806  * Give back a drop flow rule resources instance.
 807  *
 808  * @param drop
 809  *   Pointer to drop flow rule resources.
 810  */
 811 static void
 812 mlx4_drop_put(struct mlx4_drop *drop)
 813 {
 814         assert(drop->refcnt);
 815         if (--drop->refcnt)
 816                 return;
 817         drop->priv->drop = NULL;
 818         claim_zero(ibv_destroy_qp(drop->qp));
 819         claim_zero(ibv_destroy_cq(drop->cq));
 820         rte_free(drop);
 821 }
 822
 823 /**
 824  * Toggle a configured flow rule.
 825  *
 826  * @param priv
 827  *   Pointer to private structure.
 828  * @param flow
 829  *   Flow rule handle to toggle.
 830  * @param enable
 831  *   Whether associated Verbs flow must be created or removed.
 832  * @param[out] error
 833  *   Perform verbose error reporting if not NULL.
 834  *
 835  * @return
 836  *   0 on success, a negative errno value otherwise and rte_errno is set.
 837  */
 838 static int
 839 mlx4_flow_toggle(struct priv *priv,
 840                  struct rte_flow *flow,
 841                  int enable,
 842                  struct rte_flow_error *error)
 843 {
 844         struct ibv_qp *qp = NULL;
 845         const char *msg;
 846         int err;
 847
 848         if (!enable) {
 849                 if (!flow->ibv_flow)
 850                         return 0;
 851                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 852                 flow->ibv_flow = NULL;
 853                 if (flow->drop)
 854                         mlx4_drop_put(priv->drop);
 855                 return 0;
 856         }
 857         if (flow->queue) {
 858                 struct rxq *rxq = NULL;
 859
 860                 if (flow->queue_id < priv->dev->data->nb_rx_queues)
 861                         rxq = priv->dev->data->rx_queues[flow->queue_id];
 862                 if (flow->ibv_flow) {
 863                         if (!rxq ^ !flow->drop)
 864                                 return 0;
 865                         /* Verbs flow needs updating. */
 866                         claim_zero(ibv_destroy_flow(flow->ibv_flow));
 867                         flow->ibv_flow = NULL;
 868                         if (flow->drop)
 869                                 mlx4_drop_put(priv->drop);
 870                 }
 871                 if (rxq)
 872                         qp = rxq->qp;
 873                 /* A missing target queue drops traffic implicitly. */
 874                 flow->drop = !rxq;
 875         }
 876         if (flow->drop) {
 877                 mlx4_drop_get(priv);
 878                 if (!priv->drop) {
 879                         err = rte_errno;
 880                         msg = "resources for drop flow rule cannot be created";
 881                         goto error;
 882                 }
 883                 qp = priv->drop->qp;
 884         }
 885         assert(qp);
 886         assert(flow->ibv_attr);
 887         if (flow->ibv_flow)
 888                 return 0;
 889         flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
 890         if (flow->ibv_flow)
 891                 return 0;
 892         if (flow->drop)
 893                 mlx4_drop_put(priv->drop);
 894         err = errno;
 895         msg = "flow rule rejected by device";
 896 error:
 897         return rte_flow_error_set
 898                 (error, err, RTE_FLOW_ERROR_TYPE_HANDLE, flow, msg);
 899 }
 900
 901 /**
 902  * Create a flow.
 903  *
 904  * @see rte_flow_create()
 905  * @see rte_flow_ops
 906  */
 907 static struct rte_flow *
 908 mlx4_flow_create(struct rte_eth_dev *dev,
 909                  const struct rte_flow_attr *attr,
 910                  const struct rte_flow_item pattern[],
 911                  const struct rte_flow_action actions[],
 912                  struct rte_flow_error *error)
 913 {
 914         struct priv *priv = dev->data->dev_private;
 915         struct rte_flow *flow;
 916         int err;
 917
 918         err = mlx4_flow_prepare(priv, attr, pattern, actions, error, &flow);
 919         if (err)
 920                 return NULL;
 921         err = mlx4_flow_toggle(priv, flow, priv->started, error);
 922         if (!err) {
 923                 struct rte_flow *curr = LIST_FIRST(&priv->flows);
 924
 925                 /* New rules are inserted after internal ones. */
 926                 if (!curr || !curr->internal) {
 927                         LIST_INSERT_HEAD(&priv->flows, flow, next);
 928                 } else {
 929                         while (LIST_NEXT(curr, next) &&
 930                                LIST_NEXT(curr, next)->internal)
 931                                 curr = LIST_NEXT(curr, next);
 932                         LIST_INSERT_AFTER(curr, flow, next);
 933                 }
 934                 return flow;
 935         }
 936         rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 937                            error->message);
 938         rte_free(flow);
 939         return NULL;
 940 }
 941
 942 /**
 943  * Configure isolated mode.
 944  *
 945  * @see rte_flow_isolate()
 946  * @see rte_flow_ops
 947  */
 948 static int
 949 mlx4_flow_isolate(struct rte_eth_dev *dev,
 950                   int enable,
 951                   struct rte_flow_error *error)
 952 {
 953         struct priv *priv = dev->data->dev_private;
 954
 955         if (!!enable == !!priv->isolated)
 956                 return 0;
 957         priv->isolated = !!enable;
 958         if (mlx4_flow_sync(priv)) {
 959                 priv->isolated = !enable;
 960                 return rte_flow_error_set(error, rte_errno,
 961                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 962                                           NULL,
 963                                           enable ?
 964                                           "cannot enter isolated mode" :
 965                                           "cannot leave isolated mode");
 966         }
 967         return 0;
 968 }
 969
 970 /**
 971  * Destroy a flow rule.
 972  *
 973  * @see rte_flow_destroy()
 974  * @see rte_flow_ops
 975  */
 976 static int
 977 mlx4_flow_destroy(struct rte_eth_dev *dev,
 978                   struct rte_flow *flow,
 979                   struct rte_flow_error *error)
 980 {
 981         struct priv *priv = dev->data->dev_private;
 982         int err = mlx4_flow_toggle(priv, flow, 0, error);
 983
 984         if (err)
 985                 return err;
 986         LIST_REMOVE(flow, next);
 987         rte_free(flow);
 988         return 0;
 989 }
 990
 991 /**
 992  * Destroy user-configured flow rules.
 993  *
 994  * This function skips internal flows rules.
 995  *
 996  * @see rte_flow_flush()
 997  * @see rte_flow_ops
 998  */
 999 static int
1000 mlx4_flow_flush(struct rte_eth_dev *dev,
1001                 struct rte_flow_error *error)
1002 {
1003         struct priv *priv = dev->data->dev_private;
1004         struct rte_flow *flow = LIST_FIRST(&priv->flows);
1005
1006         while (flow) {
1007                 struct rte_flow *next = LIST_NEXT(flow, next);
1008
1009                 if (!flow->internal)
1010                         mlx4_flow_destroy(dev, flow, error);
1011                 flow = next;
1012         }
1013         return 0;
1014 }
1015
1016 /**
1017  * Generate internal flow rules.
1018  *
1019  * @param priv
1020  *   Pointer to private structure.
1021  * @param[out] error
1022  *   Perform verbose error reporting if not NULL.
1023  *
1024  * @return
1025  *   0 on success, a negative errno value otherwise and rte_errno is set.
1026  */
1027 static int
1028 mlx4_flow_internal(struct priv *priv, struct rte_flow_error *error)
1029 {
1030         struct rte_flow_attr attr = {
1031                 .ingress = 1,
1032         };
1033         struct rte_flow_item pattern[] = {
1034                 {
1035                         .type = MLX4_FLOW_ITEM_TYPE_INTERNAL,
1036                 },
1037                 {
1038                         .type = RTE_FLOW_ITEM_TYPE_ETH,
1039                         .spec = &(struct rte_flow_item_eth){
1040                                 .dst = priv->mac,
1041                         },
1042                         .mask = &(struct rte_flow_item_eth){
1043                                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1044                         },
1045                 },
1046                 {
1047                         .type = RTE_FLOW_ITEM_TYPE_END,
1048                 },
1049         };
1050         struct rte_flow_action actions[] = {
1051                 {
1052                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
1053                         .conf = &(struct rte_flow_action_queue){
1054                                 .index = 0,
1055                         },
1056                 },
1057                 {
1058                         .type = RTE_FLOW_ACTION_TYPE_END,
1059                 },
1060         };
1061
1062         if (!mlx4_flow_create(priv->dev, &attr, pattern, actions, error))
1063                 return -rte_errno;
1064         return 0;
1065 }
1066
1067 /**
1068  * Synchronize flow rules.
1069  *
1070  * This function synchronizes flow rules with the state of the device by
1071  * taking into account isolated mode and whether target queues are
1072  * configured.
1073  *
1074  * @param priv
1075  *   Pointer to private structure.
1076  *
1077  * @return
1078  *   0 on success, a negative errno value otherwise and rte_errno is set.
1079  */
1080 int
1081 mlx4_flow_sync(struct priv *priv)
1082 {
1083         struct rte_flow *flow;
1084         int ret;
1085
1086         /* Internal flow rules are guaranteed to come first in the list. */
1087         if (priv->isolated) {
1088                 /*
1089                  * Get rid of them in isolated mode, stop at the first
1090                  * non-internal rule found.
1091                  */
1092                 for (flow = LIST_FIRST(&priv->flows);
1093                      flow && flow->internal;
1094                      flow = LIST_FIRST(&priv->flows))
1095                         claim_zero(mlx4_flow_destroy(priv->dev, flow, NULL));
1096         } else if (!LIST_FIRST(&priv->flows) ||
1097                    !LIST_FIRST(&priv->flows)->internal) {
1098                 /*
1099                  * If the first rule is not internal outside isolated mode,
1100                  * they must be added back.
1101                  */
1102                 ret = mlx4_flow_internal(priv, NULL);
1103                 if (ret)
1104                         return ret;
1105         }
1106         if (priv->started)
1107                 return mlx4_flow_start(priv);
1108         mlx4_flow_stop(priv);
1109         return 0;
1110 }
1111
1112 /**
1113  * Clean up all flow rules.
1114  *
1115  * Unlike mlx4_flow_flush(), this function takes care of all remaining flow
1116  * rules regardless of whether they are internal or user-configured.
1117  *
1118  * @param priv
1119  *   Pointer to private structure.
1120  */
1121 void
1122 mlx4_flow_clean(struct priv *priv)
1123 {
1124         struct rte_flow *flow;
1125
1126         while ((flow = LIST_FIRST(&priv->flows)))
1127                 mlx4_flow_destroy(priv->dev, flow, NULL);
1128 }
1129
1130 /**
1131  * Disable flow rules.
1132  *
1133  * @param priv
1134  *   Pointer to private structure.
1135  */
1136 void
1137 mlx4_flow_stop(struct priv *priv)
1138 {
1139         struct rte_flow *flow;
1140
1141         for (flow = LIST_FIRST(&priv->flows);
1142              flow;
1143              flow = LIST_NEXT(flow, next)) {
1144                 claim_zero(mlx4_flow_toggle(priv, flow, 0, NULL));
1145         }
1146         assert(!priv->drop);
1147 }
1148
1149 /**
1150  * Enable flow rules.
1151  *
1152  * @param priv
1153  *   Pointer to private structure.
1154  *
1155  * @return
1156  *   0 on success, a negative errno value otherwise and rte_errno is set.
1157  */
1158 int
1159 mlx4_flow_start(struct priv *priv)
1160 {
1161         int ret;
1162         struct rte_flow *flow;
1163
1164         for (flow = LIST_FIRST(&priv->flows);
1165              flow;
1166              flow = LIST_NEXT(flow, next)) {
1167                 ret = mlx4_flow_toggle(priv, flow, 1, NULL);
1168                 if (unlikely(ret)) {
1169                         mlx4_flow_stop(priv);
1170                         return ret;
1171                 }
1172         }
1173         return 0;
1174 }
1175
1176 static const struct rte_flow_ops mlx4_flow_ops = {
1177         .validate = mlx4_flow_validate,
1178         .create = mlx4_flow_create,
1179         .destroy = mlx4_flow_destroy,
1180         .flush = mlx4_flow_flush,
1181         .isolate = mlx4_flow_isolate,
1182 };
1183
1184 /**
1185  * Manage filter operations.
1186  *
1187  * @param dev
1188  *   Pointer to Ethernet device structure.
1189  * @param filter_type
1190  *   Filter type.
1191  * @param filter_op
1192  *   Operation to perform.
1193  * @param arg
1194  *   Pointer to operation-specific structure.
1195  *
1196  * @return
1197  *   0 on success, negative errno value otherwise and rte_errno is set.
1198  */
1199 int
1200 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1201                  enum rte_filter_type filter_type,
1202                  enum rte_filter_op filter_op,
1203                  void *arg)
1204 {
1205         switch (filter_type) {
1206         case RTE_ETH_FILTER_GENERIC:
1207                 if (filter_op != RTE_ETH_FILTER_GET)
1208                         break;
1209                 *(const void **)arg = &mlx4_flow_ops;
1210                 return 0;
1211         default:
1212                 ERROR("%p: filter type (%d) not supported",
1213                       (void *)dev, filter_type);
1214                 break;
1215         }
1216         rte_errno = ENOTSUP;
1217         return -rte_errno;
1218 }