drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Flow API operations for mlx4 driver.
  37  */
  38
  39 #include <arpa/inet.h>
  40 #include <assert.h>
  41 #include <errno.h>
  42 #include <stdalign.h>
  43 #include <stddef.h>
  44 #include <stdint.h>
  45 #include <string.h>
  46 #include <sys/queue.h>
  47
  48 /* Verbs headers do not support -pedantic. */
  49 #ifdef PEDANTIC
  50 #pragma GCC diagnostic ignored "-Wpedantic"
  51 #endif
  52 #include <infiniband/verbs.h>
  53 #ifdef PEDANTIC
  54 #pragma GCC diagnostic error "-Wpedantic"
  55 #endif
  56
  57 #include <rte_byteorder.h>
  58 #include <rte_errno.h>
  59 #include <rte_eth_ctrl.h>
  60 #include <rte_ethdev.h>
  61 #include <rte_ether.h>
  62 #include <rte_flow.h>
  63 #include <rte_flow_driver.h>
  64 #include <rte_malloc.h>
  65
  66 /* PMD headers. */
  67 #include "mlx4.h"
  68 #include "mlx4_flow.h"
  69 #include "mlx4_rxtx.h"
  70 #include "mlx4_utils.h"
  71
  72 /** Static initializer for a list of subsequent item types. */
  73 #define NEXT_ITEM(...) \
  74         (const enum rte_flow_item_type []){ \
  75                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  76         }
  77
  78 /** Processor structure associated with a flow item. */
  79 struct mlx4_flow_proc_item {
  80         /** Bit-mask for fields supported by this PMD. */
  81         const void *mask_support;
  82         /** Bit-mask to use when @p item->mask is not provided. */
  83         const void *mask_default;
  84         /** Size in bytes for @p mask_support and @p mask_default. */
  85         const unsigned int mask_sz;
  86         /** Merge a pattern item into a flow rule handle. */
  87         int (*merge)(struct rte_flow *flow,
  88                      const struct rte_flow_item *item,
  89                      const struct mlx4_flow_proc_item *proc,
  90                      struct rte_flow_error *error);
  91         /** Size in bytes of the destination structure. */
  92         const unsigned int dst_sz;
  93         /** List of possible subsequent items. */
  94         const enum rte_flow_item_type *const next_item;
  95 };
  96
  97 /** Shared resources for drop flow rules. */
  98 struct mlx4_drop {
  99         struct ibv_qp *qp; /**< QP target. */
 100         struct ibv_cq *cq; /**< CQ associated with above QP. */
 101         struct priv *priv; /**< Back pointer to private data. */
 102         uint32_t refcnt; /**< Reference count. */
 103 };
 104
 105 /**
 106  * Merge Ethernet pattern item into flow rule handle.
 107  *
 108  * Additional mlx4-specific constraints on supported fields:
 109  *
 110  * - No support for partial masks, except in the specific case of matching
 111  *   all multicast traffic (@p spec->dst and @p mask->dst equal to
 112  *   01:00:00:00:00:00).
 113  * - Not providing @p item->spec or providing an empty @p mask->dst is
 114  *   *only* supported if the rule doesn't specify additional matching
 115  *   criteria (i.e. rule is promiscuous-like).
 116  *
 117  * @param[in, out] flow
 118  *   Flow rule handle to update.
 119  * @param[in] item
 120  *   Pattern item to merge.
 121  * @param[in] proc
 122  *   Associated item-processing object.
 123  * @param[out] error
 124  *   Perform verbose error reporting if not NULL.
 125  *
 126  * @return
 127  *   0 on success, a negative errno value otherwise and rte_errno is set.
 128  */
 129 static int
 130 mlx4_flow_merge_eth(struct rte_flow *flow,
 131                     const struct rte_flow_item *item,
 132                     const struct mlx4_flow_proc_item *proc,
 133                     struct rte_flow_error *error)
 134 {
 135         const struct rte_flow_item_eth *spec = item->spec;
 136         const struct rte_flow_item_eth *mask =
 137                 spec ? (item->mask ? item->mask : proc->mask_default) : NULL;
 138         struct ibv_flow_spec_eth *eth;
 139         const char *msg;
 140         unsigned int i;
 141
 142         if (!mask) {
 143                 flow->promisc = 1;
 144         } else {
 145                 uint32_t sum_dst = 0;
 146                 uint32_t sum_src = 0;
 147
 148                 for (i = 0; i != sizeof(mask->dst.addr_bytes); ++i) {
 149                         sum_dst += mask->dst.addr_bytes[i];
 150                         sum_src += mask->src.addr_bytes[i];
 151                 }
 152                 if (sum_src) {
 153                         msg = "mlx4 does not support source MAC matching";
 154                         goto error;
 155                 } else if (!sum_dst) {
 156                         flow->promisc = 1;
 157                 } else if (sum_dst == 1 && mask->dst.addr_bytes[0] == 1) {
 158                         if (!(spec->dst.addr_bytes[0] & 1)) {
 159                                 msg = "mlx4 does not support the explicit"
 160                                         " exclusion of all multicast traffic";
 161                                 goto error;
 162                         }
 163                         flow->allmulti = 1;
 164                 } else if (sum_dst != (UINT8_C(0xff) * ETHER_ADDR_LEN)) {
 165                         msg = "mlx4 does not support matching partial"
 166                                 " Ethernet fields";
 167                         goto error;
 168                 }
 169         }
 170         if (!flow->ibv_attr)
 171                 return 0;
 172         if (flow->promisc) {
 173                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 174                 return 0;
 175         }
 176         if (flow->allmulti) {
 177                 flow->ibv_attr->type = IBV_FLOW_ATTR_MC_DEFAULT;
 178                 return 0;
 179         }
 180         ++flow->ibv_attr->num_of_specs;
 181         eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 182         *eth = (struct ibv_flow_spec_eth) {
 183                 .type = IBV_FLOW_SPEC_ETH,
 184                 .size = sizeof(*eth),
 185         };
 186         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 187         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 188         /* Remove unwanted bits from values. */
 189         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 190                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 191         }
 192         return 0;
 193 error:
 194         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 195                                   item, msg);
 196 }
 197
 198 /**
 199  * Merge VLAN pattern item into flow rule handle.
 200  *
 201  * Additional mlx4-specific constraints on supported fields:
 202  *
 203  * - Matching *all* VLAN traffic by omitting @p item->spec or providing an
 204  *   empty @p item->mask would also include non-VLAN traffic. Doing so is
 205  *   therefore unsupported.
 206  * - No support for partial masks.
 207  *
 208  * @param[in, out] flow
 209  *   Flow rule handle to update.
 210  * @param[in] item
 211  *   Pattern item to merge.
 212  * @param[in] proc
 213  *   Associated item-processing object.
 214  * @param[out] error
 215  *   Perform verbose error reporting if not NULL.
 216  *
 217  * @return
 218  *   0 on success, a negative errno value otherwise and rte_errno is set.
 219  */
 220 static int
 221 mlx4_flow_merge_vlan(struct rte_flow *flow,
 222                      const struct rte_flow_item *item,
 223                      const struct mlx4_flow_proc_item *proc,
 224                      struct rte_flow_error *error)
 225 {
 226         const struct rte_flow_item_vlan *spec = item->spec;
 227         const struct rte_flow_item_vlan *mask =
 228                 spec ? (item->mask ? item->mask : proc->mask_default) : NULL;
 229         struct ibv_flow_spec_eth *eth;
 230         const char *msg;
 231
 232         if (!mask || !mask->tci) {
 233                 msg = "mlx4 cannot match all VLAN traffic while excluding"
 234                         " non-VLAN traffic, TCI VID must be specified";
 235                 goto error;
 236         }
 237         if (mask->tci != RTE_BE16(0x0fff)) {
 238                 msg = "mlx4 does not support partial TCI VID matching";
 239                 goto error;
 240         }
 241         if (!flow->ibv_attr)
 242                 return 0;
 243         eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size -
 244                        sizeof(*eth));
 245         eth->val.vlan_tag = spec->tci;
 246         eth->mask.vlan_tag = mask->tci;
 247         eth->val.vlan_tag &= eth->mask.vlan_tag;
 248         return 0;
 249 error:
 250         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 251                                   item, msg);
 252 }
 253
 254 /**
 255  * Merge IPv4 pattern item into flow rule handle.
 256  *
 257  * Additional mlx4-specific constraints on supported fields:
 258  *
 259  * - No support for partial masks.
 260  *
 261  * @param[in, out] flow
 262  *   Flow rule handle to update.
 263  * @param[in] item
 264  *   Pattern item to merge.
 265  * @param[in] proc
 266  *   Associated item-processing object.
 267  * @param[out] error
 268  *   Perform verbose error reporting if not NULL.
 269  *
 270  * @return
 271  *   0 on success, a negative errno value otherwise and rte_errno is set.
 272  */
 273 static int
 274 mlx4_flow_merge_ipv4(struct rte_flow *flow,
 275                      const struct rte_flow_item *item,
 276                      const struct mlx4_flow_proc_item *proc,
 277                      struct rte_flow_error *error)
 278 {
 279         const struct rte_flow_item_ipv4 *spec = item->spec;
 280         const struct rte_flow_item_ipv4 *mask =
 281                 spec ? (item->mask ? item->mask : proc->mask_default) : NULL;
 282         struct ibv_flow_spec_ipv4 *ipv4;
 283         const char *msg;
 284
 285         if (mask &&
 286             ((uint32_t)(mask->hdr.src_addr + 1) > UINT32_C(1) ||
 287              (uint32_t)(mask->hdr.dst_addr + 1) > UINT32_C(1))) {
 288                 msg = "mlx4 does not support matching partial IPv4 fields";
 289                 goto error;
 290         }
 291         if (!flow->ibv_attr)
 292                 return 0;
 293         ++flow->ibv_attr->num_of_specs;
 294         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 295         *ipv4 = (struct ibv_flow_spec_ipv4) {
 296                 .type = IBV_FLOW_SPEC_IPV4,
 297                 .size = sizeof(*ipv4),
 298         };
 299         if (!spec)
 300                 return 0;
 301         ipv4->val = (struct ibv_flow_ipv4_filter) {
 302                 .src_ip = spec->hdr.src_addr,
 303                 .dst_ip = spec->hdr.dst_addr,
 304         };
 305         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 306                 .src_ip = mask->hdr.src_addr,
 307                 .dst_ip = mask->hdr.dst_addr,
 308         };
 309         /* Remove unwanted bits from values. */
 310         ipv4->val.src_ip &= ipv4->mask.src_ip;
 311         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 312         return 0;
 313 error:
 314         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 315                                   item, msg);
 316 }
 317
 318 /**
 319  * Merge UDP pattern item into flow rule handle.
 320  *
 321  * Additional mlx4-specific constraints on supported fields:
 322  *
 323  * - No support for partial masks.
 324  *
 325  * @param[in, out] flow
 326  *   Flow rule handle to update.
 327  * @param[in] item
 328  *   Pattern item to merge.
 329  * @param[in] proc
 330  *   Associated item-processing object.
 331  * @param[out] error
 332  *   Perform verbose error reporting if not NULL.
 333  *
 334  * @return
 335  *   0 on success, a negative errno value otherwise and rte_errno is set.
 336  */
 337 static int
 338 mlx4_flow_merge_udp(struct rte_flow *flow,
 339                     const struct rte_flow_item *item,
 340                     const struct mlx4_flow_proc_item *proc,
 341                     struct rte_flow_error *error)
 342 {
 343         const struct rte_flow_item_udp *spec = item->spec;
 344         const struct rte_flow_item_udp *mask =
 345                 spec ? (item->mask ? item->mask : proc->mask_default) : NULL;
 346         struct ibv_flow_spec_tcp_udp *udp;
 347         const char *msg;
 348
 349         if (!mask ||
 350             ((uint16_t)(mask->hdr.src_port + 1) > UINT16_C(1) ||
 351              (uint16_t)(mask->hdr.dst_port + 1) > UINT16_C(1))) {
 352                 msg = "mlx4 does not support matching partial UDP fields";
 353                 goto error;
 354         }
 355         if (!flow->ibv_attr)
 356                 return 0;
 357         ++flow->ibv_attr->num_of_specs;
 358         udp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 359         *udp = (struct ibv_flow_spec_tcp_udp) {
 360                 .type = IBV_FLOW_SPEC_UDP,
 361                 .size = sizeof(*udp),
 362         };
 363         if (!spec)
 364                 return 0;
 365         udp->val.dst_port = spec->hdr.dst_port;
 366         udp->val.src_port = spec->hdr.src_port;
 367         udp->mask.dst_port = mask->hdr.dst_port;
 368         udp->mask.src_port = mask->hdr.src_port;
 369         /* Remove unwanted bits from values. */
 370         udp->val.src_port &= udp->mask.src_port;
 371         udp->val.dst_port &= udp->mask.dst_port;
 372         return 0;
 373 error:
 374         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 375                                   item, msg);
 376 }
 377
 378 /**
 379  * Merge TCP pattern item into flow rule handle.
 380  *
 381  * Additional mlx4-specific constraints on supported fields:
 382  *
 383  * - No support for partial masks.
 384  *
 385  * @param[in, out] flow
 386  *   Flow rule handle to update.
 387  * @param[in] item
 388  *   Pattern item to merge.
 389  * @param[in] proc
 390  *   Associated item-processing object.
 391  * @param[out] error
 392  *   Perform verbose error reporting if not NULL.
 393  *
 394  * @return
 395  *   0 on success, a negative errno value otherwise and rte_errno is set.
 396  */
 397 static int
 398 mlx4_flow_merge_tcp(struct rte_flow *flow,
 399                     const struct rte_flow_item *item,
 400                     const struct mlx4_flow_proc_item *proc,
 401                     struct rte_flow_error *error)
 402 {
 403         const struct rte_flow_item_tcp *spec = item->spec;
 404         const struct rte_flow_item_tcp *mask =
 405                 spec ? (item->mask ? item->mask : proc->mask_default) : NULL;
 406         struct ibv_flow_spec_tcp_udp *tcp;
 407         const char *msg;
 408
 409         if (!mask ||
 410             ((uint16_t)(mask->hdr.src_port + 1) > UINT16_C(1) ||
 411              (uint16_t)(mask->hdr.dst_port + 1) > UINT16_C(1))) {
 412                 msg = "mlx4 does not support matching partial TCP fields";
 413                 goto error;
 414         }
 415         if (!flow->ibv_attr)
 416                 return 0;
 417         ++flow->ibv_attr->num_of_specs;
 418         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 419         *tcp = (struct ibv_flow_spec_tcp_udp) {
 420                 .type = IBV_FLOW_SPEC_TCP,
 421                 .size = sizeof(*tcp),
 422         };
 423         if (!spec)
 424                 return 0;
 425         tcp->val.dst_port = spec->hdr.dst_port;
 426         tcp->val.src_port = spec->hdr.src_port;
 427         tcp->mask.dst_port = mask->hdr.dst_port;
 428         tcp->mask.src_port = mask->hdr.src_port;
 429         /* Remove unwanted bits from values. */
 430         tcp->val.src_port &= tcp->mask.src_port;
 431         tcp->val.dst_port &= tcp->mask.dst_port;
 432         return 0;
 433 error:
 434         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 435                                   item, msg);
 436 }
 437
 438 /**
 439  * Perform basic sanity checks on a pattern item.
 440  *
 441  * @param[in] item
 442  *   Item specification.
 443  * @param[in] proc
 444  *   Associated item-processing object.
 445  * @param[out] error
 446  *   Perform verbose error reporting if not NULL.
 447  *
 448  * @return
 449  *   0 on success, a negative errno value otherwise and rte_errno is set.
 450  */
 451 static int
 452 mlx4_flow_item_check(const struct rte_flow_item *item,
 453                      const struct mlx4_flow_proc_item *proc,
 454                      struct rte_flow_error *error)
 455 {
 456         const uint8_t *mask;
 457         unsigned int i;
 458
 459         /* item->last and item->mask cannot exist without item->spec. */
 460         if (!item->spec && (item->mask || item->last))
 461                 return rte_flow_error_set
 462                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
 463                          "\"mask\" or \"last\" field provided without a"
 464                          " corresponding \"spec\"");
 465         /* No spec, no mask, no problem. */
 466         if (!item->spec)
 467                 return 0;
 468         mask = item->mask ?
 469                 (const uint8_t *)item->mask :
 470                 (const uint8_t *)proc->mask_default;
 471         assert(mask);
 472         /*
 473          * Single-pass check to make sure that:
 474          * - Mask is supported, no bits are set outside proc->mask_support.
 475          * - Both item->spec and item->last are included in mask.
 476          */
 477         for (i = 0; i != proc->mask_sz; ++i) {
 478                 if (!mask[i])
 479                         continue;
 480                 if ((mask[i] | ((const uint8_t *)proc->mask_support)[i]) !=
 481                     ((const uint8_t *)proc->mask_support)[i])
 482                         return rte_flow_error_set
 483                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 484                                  item, "unsupported field found in \"mask\"");
 485                 if (item->last &&
 486                     (((const uint8_t *)item->spec)[i] & mask[i]) !=
 487                     (((const uint8_t *)item->last)[i] & mask[i]))
 488                         return rte_flow_error_set
 489                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 490                                  item,
 491                                  "range between \"spec\" and \"last\""
 492                                  " is larger than \"mask\"");
 493         }
 494         return 0;
 495 }
 496
 497 /** Graph of supported items and associated actions. */
 498 static const struct mlx4_flow_proc_item mlx4_flow_proc_item_list[] = {
 499         [RTE_FLOW_ITEM_TYPE_END] = {
 500                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_ETH),
 501         },
 502         [RTE_FLOW_ITEM_TYPE_ETH] = {
 503                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_VLAN,
 504                                        RTE_FLOW_ITEM_TYPE_IPV4),
 505                 .mask_support = &(const struct rte_flow_item_eth){
 506                         /* Only destination MAC can be matched. */
 507                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 508                 },
 509                 .mask_default = &rte_flow_item_eth_mask,
 510                 .mask_sz = sizeof(struct rte_flow_item_eth),
 511                 .merge = mlx4_flow_merge_eth,
 512                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 513         },
 514         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 515                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_IPV4),
 516                 .mask_support = &(const struct rte_flow_item_vlan){
 517                         /* Only TCI VID matching is supported. */
 518                         .tci = RTE_BE16(0x0fff),
 519                 },
 520                 .mask_default = &rte_flow_item_vlan_mask,
 521                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 522                 .merge = mlx4_flow_merge_vlan,
 523                 .dst_sz = 0,
 524         },
 525         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 526                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_UDP,
 527                                        RTE_FLOW_ITEM_TYPE_TCP),
 528                 .mask_support = &(const struct rte_flow_item_ipv4){
 529                         .hdr = {
 530                                 .src_addr = RTE_BE32(0xffffffff),
 531                                 .dst_addr = RTE_BE32(0xffffffff),
 532                         },
 533                 },
 534                 .mask_default = &rte_flow_item_ipv4_mask,
 535                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 536                 .merge = mlx4_flow_merge_ipv4,
 537                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 538         },
 539         [RTE_FLOW_ITEM_TYPE_UDP] = {
 540                 .mask_support = &(const struct rte_flow_item_udp){
 541                         .hdr = {
 542                                 .src_port = RTE_BE16(0xffff),
 543                                 .dst_port = RTE_BE16(0xffff),
 544                         },
 545                 },
 546                 .mask_default = &rte_flow_item_udp_mask,
 547                 .mask_sz = sizeof(struct rte_flow_item_udp),
 548                 .merge = mlx4_flow_merge_udp,
 549                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 550         },
 551         [RTE_FLOW_ITEM_TYPE_TCP] = {
 552                 .mask_support = &(const struct rte_flow_item_tcp){
 553                         .hdr = {
 554                                 .src_port = RTE_BE16(0xffff),
 555                                 .dst_port = RTE_BE16(0xffff),
 556                         },
 557                 },
 558                 .mask_default = &rte_flow_item_tcp_mask,
 559                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 560                 .merge = mlx4_flow_merge_tcp,
 561                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 562         },
 563 };
 564
 565 /**
 566  * Make sure a flow rule is supported and initialize associated structure.
 567  *
 568  * @param priv
 569  *   Pointer to private structure.
 570  * @param[in] attr
 571  *   Flow rule attributes.
 572  * @param[in] pattern
 573  *   Pattern specification (list terminated by the END pattern item).
 574  * @param[in] actions
 575  *   Associated actions (list terminated by the END action).
 576  * @param[out] error
 577  *   Perform verbose error reporting if not NULL.
 578  * @param[in, out] addr
 579  *   Buffer where the resulting flow rule handle pointer must be stored.
 580  *   If NULL, stop processing after validation stage.
 581  *
 582  * @return
 583  *   0 on success, a negative errno value otherwise and rte_errno is set.
 584  */
 585 static int
 586 mlx4_flow_prepare(struct priv *priv,
 587                   const struct rte_flow_attr *attr,
 588                   const struct rte_flow_item pattern[],
 589                   const struct rte_flow_action actions[],
 590                   struct rte_flow_error *error,
 591                   struct rte_flow **addr)
 592 {
 593         const struct rte_flow_item *item;
 594         const struct rte_flow_action *action;
 595         const struct mlx4_flow_proc_item *proc;
 596         struct rte_flow temp = { .ibv_attr_size = sizeof(*temp.ibv_attr) };
 597         struct rte_flow *flow = &temp;
 598         const char *msg = NULL;
 599
 600         if (attr->group)
 601                 return rte_flow_error_set
 602                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 603                          NULL, "groups are not supported");
 604         if (attr->priority > MLX4_FLOW_PRIORITY_LAST)
 605                 return rte_flow_error_set
 606                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 607                          NULL, "maximum priority level is "
 608                          MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST));
 609         if (attr->egress)
 610                 return rte_flow_error_set
 611                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 612                          NULL, "egress is not supported");
 613         if (!attr->ingress)
 614                 return rte_flow_error_set
 615                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 616                          NULL, "only ingress is supported");
 617 fill:
 618         proc = mlx4_flow_proc_item_list;
 619         /* Go over pattern. */
 620         for (item = pattern; item->type; ++item) {
 621                 const struct mlx4_flow_proc_item *next = NULL;
 622                 unsigned int i;
 623                 int err;
 624
 625                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 626                         continue;
 627                 if (item->type == MLX4_FLOW_ITEM_TYPE_INTERNAL) {
 628                         flow->internal = 1;
 629                         continue;
 630                 }
 631                 if (flow->promisc || flow->allmulti) {
 632                         msg = "mlx4 does not support additional matching"
 633                                 " criteria combined with indiscriminate"
 634                                 " matching on Ethernet headers";
 635                         goto exit_item_not_supported;
 636                 }
 637                 for (i = 0; proc->next_item && proc->next_item[i]; ++i) {
 638                         if (proc->next_item[i] == item->type) {
 639                                 next = &mlx4_flow_proc_item_list[item->type];
 640                                 break;
 641                         }
 642                 }
 643                 if (!next)
 644                         goto exit_item_not_supported;
 645                 proc = next;
 646                 /*
 647                  * Perform basic sanity checks only once, while handle is
 648                  * not allocated.
 649                  */
 650                 if (flow == &temp) {
 651                         err = mlx4_flow_item_check(item, proc, error);
 652                         if (err)
 653                                 return err;
 654                 }
 655                 if (proc->merge) {
 656                         err = proc->merge(flow, item, proc, error);
 657                         if (err)
 658                                 return err;
 659                 }
 660                 flow->ibv_attr_size += proc->dst_sz;
 661         }
 662         /* Go over actions list. */
 663         for (action = actions; action->type; ++action) {
 664                 switch (action->type) {
 665                         const struct rte_flow_action_queue *queue;
 666
 667                 case RTE_FLOW_ACTION_TYPE_VOID:
 668                         continue;
 669                 case RTE_FLOW_ACTION_TYPE_DROP:
 670                         flow->drop = 1;
 671                         break;
 672                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 673                         queue = action->conf;
 674                         if (queue->index >= priv->dev->data->nb_rx_queues)
 675                                 goto exit_action_not_supported;
 676                         flow->queue = 1;
 677                         flow->queue_id = queue->index;
 678                         break;
 679                 default:
 680                         goto exit_action_not_supported;
 681                 }
 682         }
 683         if (!flow->queue && !flow->drop)
 684                 return rte_flow_error_set
 685                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 686                          NULL, "no valid action");
 687         /* Validation ends here. */
 688         if (!addr)
 689                 return 0;
 690         if (flow == &temp) {
 691                 /* Allocate proper handle based on collected data. */
 692                 const struct mlx4_malloc_vec vec[] = {
 693                         {
 694                                 .align = alignof(struct rte_flow),
 695                                 .size = sizeof(*flow),
 696                                 .addr = (void **)&flow,
 697                         },
 698                         {
 699                                 .align = alignof(struct ibv_flow_attr),
 700                                 .size = temp.ibv_attr_size,
 701                                 .addr = (void **)&temp.ibv_attr,
 702                         },
 703                 };
 704
 705                 if (!mlx4_zmallocv(__func__, vec, RTE_DIM(vec)))
 706                         return rte_flow_error_set
 707                                 (error, -rte_errno,
 708                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 709                                  "flow rule handle allocation failure");
 710                 /* Most fields will be updated by second pass. */
 711                 *flow = (struct rte_flow){
 712                         .ibv_attr = temp.ibv_attr,
 713                         .ibv_attr_size = sizeof(*flow->ibv_attr),
 714                 };
 715                 *flow->ibv_attr = (struct ibv_flow_attr){
 716                         .type = IBV_FLOW_ATTR_NORMAL,
 717                         .size = sizeof(*flow->ibv_attr),
 718                         .priority = attr->priority,
 719                         .port = priv->port,
 720                 };
 721                 goto fill;
 722         }
 723         *addr = flow;
 724         return 0;
 725 exit_item_not_supported:
 726         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 727                                   item, msg ? msg : "item not supported");
 728 exit_action_not_supported:
 729         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 730                                   action, "action not supported");
 731 }
 732
 733 /**
 734  * Validate a flow supported by the NIC.
 735  *
 736  * @see rte_flow_validate()
 737  * @see rte_flow_ops
 738  */
 739 static int
 740 mlx4_flow_validate(struct rte_eth_dev *dev,
 741                    const struct rte_flow_attr *attr,
 742                    const struct rte_flow_item pattern[],
 743                    const struct rte_flow_action actions[],
 744                    struct rte_flow_error *error)
 745 {
 746         struct priv *priv = dev->data->dev_private;
 747
 748         return mlx4_flow_prepare(priv, attr, pattern, actions, error, NULL);
 749 }
 750
 751 /**
 752  * Get a drop flow rule resources instance.
 753  *
 754  * @param priv
 755  *   Pointer to private structure.
 756  *
 757  * @return
 758  *   Pointer to drop flow resources on success, NULL otherwise and rte_errno
 759  *   is set.
 760  */
 761 static struct mlx4_drop *
 762 mlx4_drop_get(struct priv *priv)
 763 {
 764         struct mlx4_drop *drop = priv->drop;
 765
 766         if (drop) {
 767                 assert(drop->refcnt);
 768                 assert(drop->priv == priv);
 769                 ++drop->refcnt;
 770                 return drop;
 771         }
 772         drop = rte_malloc(__func__, sizeof(*drop), 0);
 773         if (!drop)
 774                 goto error;
 775         *drop = (struct mlx4_drop){
 776                 .priv = priv,
 777                 .refcnt = 1,
 778         };
 779         drop->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 780         if (!drop->cq)
 781                 goto error;
 782         drop->qp = ibv_create_qp(priv->pd,
 783                                  &(struct ibv_qp_init_attr){
 784                                         .send_cq = drop->cq,
 785                                         .recv_cq = drop->cq,
 786                                         .qp_type = IBV_QPT_RAW_PACKET,
 787                                  });
 788         if (!drop->qp)
 789                 goto error;
 790         priv->drop = drop;
 791         return drop;
 792 error:
 793         if (drop->qp)
 794                 claim_zero(ibv_destroy_qp(drop->qp));
 795         if (drop->cq)
 796                 claim_zero(ibv_destroy_cq(drop->cq));
 797         if (drop)
 798                 rte_free(drop);
 799         rte_errno = ENOMEM;
 800         return NULL;
 801 }
 802
 803 /**
 804  * Give back a drop flow rule resources instance.
 805  *
 806  * @param drop
 807  *   Pointer to drop flow rule resources.
 808  */
 809 static void
 810 mlx4_drop_put(struct mlx4_drop *drop)
 811 {
 812         assert(drop->refcnt);
 813         if (--drop->refcnt)
 814                 return;
 815         drop->priv->drop = NULL;
 816         claim_zero(ibv_destroy_qp(drop->qp));
 817         claim_zero(ibv_destroy_cq(drop->cq));
 818         rte_free(drop);
 819 }
 820
 821 /**
 822  * Toggle a configured flow rule.
 823  *
 824  * @param priv
 825  *   Pointer to private structure.
 826  * @param flow
 827  *   Flow rule handle to toggle.
 828  * @param enable
 829  *   Whether associated Verbs flow must be created or removed.
 830  * @param[out] error
 831  *   Perform verbose error reporting if not NULL.
 832  *
 833  * @return
 834  *   0 on success, a negative errno value otherwise and rte_errno is set.
 835  */
 836 static int
 837 mlx4_flow_toggle(struct priv *priv,
 838                  struct rte_flow *flow,
 839                  int enable,
 840                  struct rte_flow_error *error)
 841 {
 842         struct ibv_qp *qp = NULL;
 843         const char *msg;
 844         int err;
 845
 846         if (!enable) {
 847                 if (!flow->ibv_flow)
 848                         return 0;
 849                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 850                 flow->ibv_flow = NULL;
 851                 if (flow->drop)
 852                         mlx4_drop_put(priv->drop);
 853                 return 0;
 854         }
 855         assert(flow->ibv_attr);
 856         if (!flow->internal &&
 857             !priv->isolated &&
 858             flow->ibv_attr->priority == MLX4_FLOW_PRIORITY_LAST) {
 859                 if (flow->ibv_flow) {
 860                         claim_zero(ibv_destroy_flow(flow->ibv_flow));
 861                         flow->ibv_flow = NULL;
 862                         if (flow->drop)
 863                                 mlx4_drop_put(priv->drop);
 864                 }
 865                 err = EACCES;
 866                 msg = ("priority level "
 867                        MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST)
 868                        " is reserved when not in isolated mode");
 869                 goto error;
 870         }
 871         if (flow->queue) {
 872                 struct rxq *rxq = NULL;
 873
 874                 if (flow->queue_id < priv->dev->data->nb_rx_queues)
 875                         rxq = priv->dev->data->rx_queues[flow->queue_id];
 876                 if (flow->ibv_flow) {
 877                         if (!rxq ^ !flow->drop)
 878                                 return 0;
 879                         /* Verbs flow needs updating. */
 880                         claim_zero(ibv_destroy_flow(flow->ibv_flow));
 881                         flow->ibv_flow = NULL;
 882                         if (flow->drop)
 883                                 mlx4_drop_put(priv->drop);
 884                 }
 885                 if (rxq)
 886                         qp = rxq->qp;
 887                 /* A missing target queue drops traffic implicitly. */
 888                 flow->drop = !rxq;
 889         }
 890         if (flow->drop) {
 891                 mlx4_drop_get(priv);
 892                 if (!priv->drop) {
 893                         err = rte_errno;
 894                         msg = "resources for drop flow rule cannot be created";
 895                         goto error;
 896                 }
 897                 qp = priv->drop->qp;
 898         }
 899         assert(qp);
 900         if (flow->ibv_flow)
 901                 return 0;
 902         flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
 903         if (flow->ibv_flow)
 904                 return 0;
 905         if (flow->drop)
 906                 mlx4_drop_put(priv->drop);
 907         err = errno;
 908         msg = "flow rule rejected by device";
 909 error:
 910         return rte_flow_error_set
 911                 (error, err, RTE_FLOW_ERROR_TYPE_HANDLE, flow, msg);
 912 }
 913
 914 /**
 915  * Create a flow.
 916  *
 917  * @see rte_flow_create()
 918  * @see rte_flow_ops
 919  */
 920 static struct rte_flow *
 921 mlx4_flow_create(struct rte_eth_dev *dev,
 922                  const struct rte_flow_attr *attr,
 923                  const struct rte_flow_item pattern[],
 924                  const struct rte_flow_action actions[],
 925                  struct rte_flow_error *error)
 926 {
 927         struct priv *priv = dev->data->dev_private;
 928         struct rte_flow *flow;
 929         int err;
 930
 931         err = mlx4_flow_prepare(priv, attr, pattern, actions, error, &flow);
 932         if (err)
 933                 return NULL;
 934         err = mlx4_flow_toggle(priv, flow, priv->started, error);
 935         if (!err) {
 936                 struct rte_flow *curr = LIST_FIRST(&priv->flows);
 937
 938                 /* New rules are inserted after internal ones. */
 939                 if (!curr || !curr->internal) {
 940                         LIST_INSERT_HEAD(&priv->flows, flow, next);
 941                 } else {
 942                         while (LIST_NEXT(curr, next) &&
 943                                LIST_NEXT(curr, next)->internal)
 944                                 curr = LIST_NEXT(curr, next);
 945                         LIST_INSERT_AFTER(curr, flow, next);
 946                 }
 947                 return flow;
 948         }
 949         rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 950                            error->message);
 951         rte_free(flow);
 952         return NULL;
 953 }
 954
 955 /**
 956  * Configure isolated mode.
 957  *
 958  * @see rte_flow_isolate()
 959  * @see rte_flow_ops
 960  */
 961 static int
 962 mlx4_flow_isolate(struct rte_eth_dev *dev,
 963                   int enable,
 964                   struct rte_flow_error *error)
 965 {
 966         struct priv *priv = dev->data->dev_private;
 967
 968         if (!!enable == !!priv->isolated)
 969                 return 0;
 970         priv->isolated = !!enable;
 971         if (mlx4_flow_sync(priv, error)) {
 972                 priv->isolated = !enable;
 973                 return -rte_errno;
 974         }
 975         return 0;
 976 }
 977
 978 /**
 979  * Destroy a flow rule.
 980  *
 981  * @see rte_flow_destroy()
 982  * @see rte_flow_ops
 983  */
 984 static int
 985 mlx4_flow_destroy(struct rte_eth_dev *dev,
 986                   struct rte_flow *flow,
 987                   struct rte_flow_error *error)
 988 {
 989         struct priv *priv = dev->data->dev_private;
 990         int err = mlx4_flow_toggle(priv, flow, 0, error);
 991
 992         if (err)
 993                 return err;
 994         LIST_REMOVE(flow, next);
 995         rte_free(flow);
 996         return 0;
 997 }
 998
 999 /**
1000  * Destroy user-configured flow rules.
1001  *
1002  * This function skips internal flows rules.
1003  *
1004  * @see rte_flow_flush()
1005  * @see rte_flow_ops
1006  */
1007 static int
1008 mlx4_flow_flush(struct rte_eth_dev *dev,
1009                 struct rte_flow_error *error)
1010 {
1011         struct priv *priv = dev->data->dev_private;
1012         struct rte_flow *flow = LIST_FIRST(&priv->flows);
1013
1014         while (flow) {
1015                 struct rte_flow *next = LIST_NEXT(flow, next);
1016
1017                 if (!flow->internal)
1018                         mlx4_flow_destroy(dev, flow, error);
1019                 flow = next;
1020         }
1021         return 0;
1022 }
1023
1024 /**
1025  * Helper function to determine the next configured VLAN filter.
1026  *
1027  * @param priv
1028  *   Pointer to private structure.
1029  * @param vlan
1030  *   VLAN ID to use as a starting point.
1031  *
1032  * @return
1033  *   Next configured VLAN ID or a high value (>= 4096) if there is none.
1034  */
1035 static uint16_t
1036 mlx4_flow_internal_next_vlan(struct priv *priv, uint16_t vlan)
1037 {
1038         while (vlan < 4096) {
1039                 if (priv->dev->data->vlan_filter_conf.ids[vlan / 64] &
1040                     (UINT64_C(1) << (vlan % 64)))
1041                         return vlan;
1042                 ++vlan;
1043         }
1044         return vlan;
1045 }
1046
1047 /**
1048  * Generate internal flow rules.
1049  *
1050  * Various flow rules are created depending on the mode the device is in:
1051  *
1052  * 1. Promiscuous: port MAC + catch-all (VLAN filtering is ignored).
1053  * 2. All multicast: port MAC/VLAN + catch-all multicast.
1054  * 3. Otherwise: port MAC/VLAN + broadcast MAC/VLAN.
1055  *
1056  * About MAC flow rules:
1057  *
1058  * - MAC flow rules are generated from @p dev->data->mac_addrs
1059  *   (@p priv->mac array).
1060  * - An additional flow rule for Ethernet broadcasts is also generated.
1061  * - All these are per-VLAN if @p dev->data->dev_conf.rxmode.hw_vlan_filter
1062  *   is enabled and VLAN filters are configured.
1063  *
1064  * @param priv
1065  *   Pointer to private structure.
1066  * @param[out] error
1067  *   Perform verbose error reporting if not NULL.
1068  *
1069  * @return
1070  *   0 on success, a negative errno value otherwise and rte_errno is set.
1071  */
1072 static int
1073 mlx4_flow_internal(struct priv *priv, struct rte_flow_error *error)
1074 {
1075         struct rte_flow_attr attr = {
1076                 .priority = MLX4_FLOW_PRIORITY_LAST,
1077                 .ingress = 1,
1078         };
1079         struct rte_flow_item_eth eth_spec;
1080         const struct rte_flow_item_eth eth_mask = {
1081                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1082         };
1083         const struct rte_flow_item_eth eth_allmulti = {
1084                 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
1085         };
1086         struct rte_flow_item_vlan vlan_spec;
1087         const struct rte_flow_item_vlan vlan_mask = {
1088                 .tci = RTE_BE16(0x0fff),
1089         };
1090         struct rte_flow_item pattern[] = {
1091                 {
1092                         .type = MLX4_FLOW_ITEM_TYPE_INTERNAL,
1093                 },
1094                 {
1095                         .type = RTE_FLOW_ITEM_TYPE_ETH,
1096                         .spec = &eth_spec,
1097                         .mask = &eth_mask,
1098                 },
1099                 {
1100                         /* Replaced with VLAN if filtering is enabled. */
1101                         .type = RTE_FLOW_ITEM_TYPE_END,
1102                 },
1103                 {
1104                         .type = RTE_FLOW_ITEM_TYPE_END,
1105                 },
1106         };
1107         struct rte_flow_action actions[] = {
1108                 {
1109                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
1110                         .conf = &(struct rte_flow_action_queue){
1111                                 .index = 0,
1112                         },
1113                 },
1114                 {
1115                         .type = RTE_FLOW_ACTION_TYPE_END,
1116                 },
1117         };
1118         struct ether_addr *rule_mac = &eth_spec.dst;
1119         rte_be16_t *rule_vlan =
1120                 priv->dev->data->dev_conf.rxmode.hw_vlan_filter &&
1121                 !priv->dev->data->promiscuous ?
1122                 &vlan_spec.tci :
1123                 NULL;
1124         int broadcast =
1125                 !priv->dev->data->promiscuous &&
1126                 !priv->dev->data->all_multicast;
1127         uint16_t vlan = 0;
1128         struct rte_flow *flow;
1129         unsigned int i;
1130         int err = 0;
1131
1132         /*
1133          * Set up VLAN item if filtering is enabled and at least one VLAN
1134          * filter is configured.
1135          */
1136         if (rule_vlan) {
1137                 vlan = mlx4_flow_internal_next_vlan(priv, 0);
1138                 if (vlan < 4096) {
1139                         pattern[2] = (struct rte_flow_item){
1140                                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
1141                                 .spec = &vlan_spec,
1142                                 .mask = &vlan_mask,
1143                         };
1144 next_vlan:
1145                         *rule_vlan = rte_cpu_to_be_16(vlan);
1146                 } else {
1147                         rule_vlan = NULL;
1148                 }
1149         }
1150         for (i = 0; i != RTE_DIM(priv->mac) + broadcast; ++i) {
1151                 const struct ether_addr *mac;
1152
1153                 /* Broadcasts are handled by an extra iteration. */
1154                 if (i < RTE_DIM(priv->mac))
1155                         mac = &priv->mac[i];
1156                 else
1157                         mac = &eth_mask.dst;
1158                 if (is_zero_ether_addr(mac))
1159                         continue;
1160                 /* Check if MAC flow rule is already present. */
1161                 for (flow = LIST_FIRST(&priv->flows);
1162                      flow && flow->internal;
1163                      flow = LIST_NEXT(flow, next)) {
1164                         const struct ibv_flow_spec_eth *eth =
1165                                 (const void *)((uintptr_t)flow->ibv_attr +
1166                                                sizeof(*flow->ibv_attr));
1167                         unsigned int j;
1168
1169                         if (!flow->mac)
1170                                 continue;
1171                         assert(flow->ibv_attr->type == IBV_FLOW_ATTR_NORMAL);
1172                         assert(flow->ibv_attr->num_of_specs == 1);
1173                         assert(eth->type == IBV_FLOW_SPEC_ETH);
1174                         if (rule_vlan &&
1175                             (eth->val.vlan_tag != *rule_vlan ||
1176                              eth->mask.vlan_tag != RTE_BE16(0x0fff)))
1177                                 continue;
1178                         if (!rule_vlan && eth->mask.vlan_tag)
1179                                 continue;
1180                         for (j = 0; j != sizeof(mac->addr_bytes); ++j)
1181                                 if (eth->val.dst_mac[j] != mac->addr_bytes[j] ||
1182                                     eth->mask.dst_mac[j] != UINT8_C(0xff) ||
1183                                     eth->val.src_mac[j] != UINT8_C(0x00) ||
1184                                     eth->mask.src_mac[j] != UINT8_C(0x00))
1185                                         break;
1186                         if (j == sizeof(mac->addr_bytes))
1187                                 break;
1188                 }
1189                 if (!flow || !flow->internal) {
1190                         /* Not found, create a new flow rule. */
1191                         memcpy(rule_mac, mac, sizeof(*mac));
1192                         flow = mlx4_flow_create(priv->dev, &attr, pattern,
1193                                                 actions, error);
1194                         if (!flow) {
1195                                 err = -rte_errno;
1196                                 goto error;
1197                         }
1198                 }
1199                 flow->select = 1;
1200                 flow->mac = 1;
1201         }
1202         if (rule_vlan) {
1203                 vlan = mlx4_flow_internal_next_vlan(priv, vlan + 1);
1204                 if (vlan < 4096)
1205                         goto next_vlan;
1206         }
1207         /* Take care of promiscuous and all multicast flow rules. */
1208         if (!broadcast) {
1209                 for (flow = LIST_FIRST(&priv->flows);
1210                      flow && flow->internal;
1211                      flow = LIST_NEXT(flow, next)) {
1212                         if (priv->dev->data->promiscuous) {
1213                                 if (flow->promisc)
1214                                         break;
1215                         } else {
1216                                 assert(priv->dev->data->all_multicast);
1217                                 if (flow->allmulti)
1218                                         break;
1219                         }
1220                 }
1221                 if (!flow || !flow->internal) {
1222                         /* Not found, create a new flow rule. */
1223                         if (priv->dev->data->promiscuous) {
1224                                 pattern[1].spec = NULL;
1225                                 pattern[1].mask = NULL;
1226                         } else {
1227                                 assert(priv->dev->data->all_multicast);
1228                                 pattern[1].spec = &eth_allmulti;
1229                                 pattern[1].mask = &eth_allmulti;
1230                         }
1231                         pattern[2] = pattern[3];
1232                         flow = mlx4_flow_create(priv->dev, &attr, pattern,
1233                                                 actions, error);
1234                         if (!flow) {
1235                                 err = -rte_errno;
1236                                 goto error;
1237                         }
1238                 }
1239                 assert(flow->promisc || flow->allmulti);
1240                 flow->select = 1;
1241         }
1242 error:
1243         /* Clear selection and clean up stale internal flow rules. */
1244         flow = LIST_FIRST(&priv->flows);
1245         while (flow && flow->internal) {
1246                 struct rte_flow *next = LIST_NEXT(flow, next);
1247
1248                 if (!flow->select)
1249                         claim_zero(mlx4_flow_destroy(priv->dev, flow, error));
1250                 else
1251                         flow->select = 0;
1252                 flow = next;
1253         }
1254         return err;
1255 }
1256
1257 /**
1258  * Synchronize flow rules.
1259  *
1260  * This function synchronizes flow rules with the state of the device by
1261  * taking into account isolated mode and whether target queues are
1262  * configured.
1263  *
1264  * @param priv
1265  *   Pointer to private structure.
1266  * @param[out] error
1267  *   Perform verbose error reporting if not NULL.
1268  *
1269  * @return
1270  *   0 on success, a negative errno value otherwise and rte_errno is set.
1271  */
1272 int
1273 mlx4_flow_sync(struct priv *priv, struct rte_flow_error *error)
1274 {
1275         struct rte_flow *flow;
1276         int ret;
1277
1278         /* Internal flow rules are guaranteed to come first in the list. */
1279         if (priv->isolated) {
1280                 /*
1281                  * Get rid of them in isolated mode, stop at the first
1282                  * non-internal rule found.
1283                  */
1284                 for (flow = LIST_FIRST(&priv->flows);
1285                      flow && flow->internal;
1286                      flow = LIST_FIRST(&priv->flows))
1287                         claim_zero(mlx4_flow_destroy(priv->dev, flow, error));
1288         } else {
1289                 /* Refresh internal rules. */
1290                 ret = mlx4_flow_internal(priv, error);
1291                 if (ret)
1292                         return ret;
1293         }
1294         /* Toggle the remaining flow rules . */
1295         for (flow = LIST_FIRST(&priv->flows);
1296              flow;
1297              flow = LIST_NEXT(flow, next)) {
1298                 ret = mlx4_flow_toggle(priv, flow, priv->started, error);
1299                 if (ret)
1300                         return ret;
1301         }
1302         if (!priv->started)
1303                 assert(!priv->drop);
1304         return 0;
1305 }
1306
1307 /**
1308  * Clean up all flow rules.
1309  *
1310  * Unlike mlx4_flow_flush(), this function takes care of all remaining flow
1311  * rules regardless of whether they are internal or user-configured.
1312  *
1313  * @param priv
1314  *   Pointer to private structure.
1315  */
1316 void
1317 mlx4_flow_clean(struct priv *priv)
1318 {
1319         struct rte_flow *flow;
1320
1321         while ((flow = LIST_FIRST(&priv->flows)))
1322                 mlx4_flow_destroy(priv->dev, flow, NULL);
1323 }
1324
1325 static const struct rte_flow_ops mlx4_flow_ops = {
1326         .validate = mlx4_flow_validate,
1327         .create = mlx4_flow_create,
1328         .destroy = mlx4_flow_destroy,
1329         .flush = mlx4_flow_flush,
1330         .isolate = mlx4_flow_isolate,
1331 };
1332
1333 /**
1334  * Manage filter operations.
1335  *
1336  * @param dev
1337  *   Pointer to Ethernet device structure.
1338  * @param filter_type
1339  *   Filter type.
1340  * @param filter_op
1341  *   Operation to perform.
1342  * @param arg
1343  *   Pointer to operation-specific structure.
1344  *
1345  * @return
1346  *   0 on success, negative errno value otherwise and rte_errno is set.
1347  */
1348 int
1349 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1350                  enum rte_filter_type filter_type,
1351                  enum rte_filter_op filter_op,
1352                  void *arg)
1353 {
1354         switch (filter_type) {
1355         case RTE_ETH_FILTER_GENERIC:
1356                 if (filter_op != RTE_ETH_FILTER_GET)
1357                         break;
1358                 *(const void **)arg = &mlx4_flow_ops;
1359                 return 0;
1360         default:
1361                 ERROR("%p: filter type (%d) not supported",
1362                       (void *)dev, filter_type);
1363                 break;
1364         }
1365         rte_errno = ENOTSUP;
1366         return -rte_errno;
1367 }