drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <assert.h>
  35
  36 #include <rte_flow.h>
  37 #include <rte_flow_driver.h>
  38 #include <rte_malloc.h>
  39
  40 /* PMD headers. */
  41 #include "mlx4.h"
  42 #include "mlx4_flow.h"
  43
  44 /** Static initializer for items. */
  45 #define ITEMS(...) \
  46         (const enum rte_flow_item_type []){ \
  47                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  48         }
  49
  50 /** Structure to generate a simple graph of layers supported by the NIC. */
  51 struct mlx4_flow_items {
  52         /** List of possible actions for these items. */
  53         const enum rte_flow_action_type *const actions;
  54         /** Bit-masks corresponding to the possibilities for the item. */
  55         const void *mask;
  56         /**
  57          * Default bit-masks to use when item->mask is not provided. When
  58          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  59          * used instead.
  60          */
  61         const void *default_mask;
  62         /** Bit-masks size in bytes. */
  63         const unsigned int mask_sz;
  64         /**
  65          * Check support for a given item.
  66          *
  67          * @param item[in]
  68          *   Item specification.
  69          * @param mask[in]
  70          *   Bit-masks covering supported fields to compare with spec,
  71          *   last and mask in
  72          *   \item.
  73          * @param size
  74          *   Bit-Mask size in bytes.
  75          *
  76          * @return
  77          *   0 on success, negative value otherwise.
  78          */
  79         int (*validate)(const struct rte_flow_item *item,
  80                         const uint8_t *mask, unsigned int size);
  81         /**
  82          * Conversion function from rte_flow to NIC specific flow.
  83          *
  84          * @param item
  85          *   rte_flow item to convert.
  86          * @param default_mask
  87          *   Default bit-masks to use when item->mask is not provided.
  88          * @param data
  89          *   Internal structure to store the conversion.
  90          *
  91          * @return
  92          *   0 on success, negative value otherwise.
  93          */
  94         int (*convert)(const struct rte_flow_item *item,
  95                        const void *default_mask,
  96                        void *data);
  97         /** Size in bytes of the destination structure. */
  98         const unsigned int dst_sz;
  99         /** List of possible following items.  */
 100         const enum rte_flow_item_type *const items;
 101 };
 102
 103 struct rte_flow_drop {
 104         struct ibv_qp *qp; /**< Verbs queue pair. */
 105         struct ibv_cq *cq; /**< Verbs completion queue. */
 106 };
 107
 108 /** Valid action for this PMD. */
 109 static const enum rte_flow_action_type valid_actions[] = {
 110         RTE_FLOW_ACTION_TYPE_DROP,
 111         RTE_FLOW_ACTION_TYPE_QUEUE,
 112         RTE_FLOW_ACTION_TYPE_RSS,
 113         RTE_FLOW_ACTION_TYPE_END,
 114 };
 115
 116 /**
 117  * Convert Ethernet item to Verbs specification.
 118  *
 119  * @param item[in]
 120  *   Item specification.
 121  * @param default_mask[in]
 122  *   Default bit-masks to use when item->mask is not provided.
 123  * @param data[in, out]
 124  *   User structure.
 125  */
 126 static int
 127 mlx4_flow_create_eth(const struct rte_flow_item *item,
 128                      const void *default_mask,
 129                      void *data)
 130 {
 131         const struct rte_flow_item_eth *spec = item->spec;
 132         const struct rte_flow_item_eth *mask = item->mask;
 133         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 134         struct ibv_flow_spec_eth *eth;
 135         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 136         unsigned int i;
 137
 138         ++flow->ibv_attr->num_of_specs;
 139         flow->ibv_attr->priority = 2;
 140         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 141         *eth = (struct ibv_flow_spec_eth) {
 142                 .type = IBV_FLOW_SPEC_ETH,
 143                 .size = eth_size,
 144         };
 145         if (!spec) {
 146                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 147                 return 0;
 148         }
 149         if (!mask)
 150                 mask = default_mask;
 151         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 152         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 153         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 154         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 155         /* Remove unwanted bits from values. */
 156         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 157                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 158                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 159         }
 160         return 0;
 161 }
 162
 163 /**
 164  * Convert VLAN item to Verbs specification.
 165  *
 166  * @param item[in]
 167  *   Item specification.
 168  * @param default_mask[in]
 169  *   Default bit-masks to use when item->mask is not provided.
 170  * @param data[in, out]
 171  *   User structure.
 172  */
 173 static int
 174 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 175                       const void *default_mask,
 176                       void *data)
 177 {
 178         const struct rte_flow_item_vlan *spec = item->spec;
 179         const struct rte_flow_item_vlan *mask = item->mask;
 180         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 181         struct ibv_flow_spec_eth *eth;
 182         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 183
 184         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 185         if (!spec)
 186                 return 0;
 187         if (!mask)
 188                 mask = default_mask;
 189         eth->val.vlan_tag = spec->tci;
 190         eth->mask.vlan_tag = mask->tci;
 191         eth->val.vlan_tag &= eth->mask.vlan_tag;
 192         return 0;
 193 }
 194
 195 /**
 196  * Convert IPv4 item to Verbs specification.
 197  *
 198  * @param item[in]
 199  *   Item specification.
 200  * @param default_mask[in]
 201  *   Default bit-masks to use when item->mask is not provided.
 202  * @param data[in, out]
 203  *   User structure.
 204  */
 205 static int
 206 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 207                       const void *default_mask,
 208                       void *data)
 209 {
 210         const struct rte_flow_item_ipv4 *spec = item->spec;
 211         const struct rte_flow_item_ipv4 *mask = item->mask;
 212         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 213         struct ibv_flow_spec_ipv4 *ipv4;
 214         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 215
 216         ++flow->ibv_attr->num_of_specs;
 217         flow->ibv_attr->priority = 1;
 218         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 219         *ipv4 = (struct ibv_flow_spec_ipv4) {
 220                 .type = IBV_FLOW_SPEC_IPV4,
 221                 .size = ipv4_size,
 222         };
 223         if (!spec)
 224                 return 0;
 225         ipv4->val = (struct ibv_flow_ipv4_filter) {
 226                 .src_ip = spec->hdr.src_addr,
 227                 .dst_ip = spec->hdr.dst_addr,
 228         };
 229         if (!mask)
 230                 mask = default_mask;
 231         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 232                 .src_ip = mask->hdr.src_addr,
 233                 .dst_ip = mask->hdr.dst_addr,
 234         };
 235         /* Remove unwanted bits from values. */
 236         ipv4->val.src_ip &= ipv4->mask.src_ip;
 237         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 238         return 0;
 239 }
 240
 241 /**
 242  * Convert UDP item to Verbs specification.
 243  *
 244  * @param item[in]
 245  *   Item specification.
 246  * @param default_mask[in]
 247  *   Default bit-masks to use when item->mask is not provided.
 248  * @param data[in, out]
 249  *   User structure.
 250  */
 251 static int
 252 mlx4_flow_create_udp(const struct rte_flow_item *item,
 253                      const void *default_mask,
 254                      void *data)
 255 {
 256         const struct rte_flow_item_udp *spec = item->spec;
 257         const struct rte_flow_item_udp *mask = item->mask;
 258         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 259         struct ibv_flow_spec_tcp_udp *udp;
 260         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 261
 262         ++flow->ibv_attr->num_of_specs;
 263         flow->ibv_attr->priority = 0;
 264         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 265         *udp = (struct ibv_flow_spec_tcp_udp) {
 266                 .type = IBV_FLOW_SPEC_UDP,
 267                 .size = udp_size,
 268         };
 269         if (!spec)
 270                 return 0;
 271         udp->val.dst_port = spec->hdr.dst_port;
 272         udp->val.src_port = spec->hdr.src_port;
 273         if (!mask)
 274                 mask = default_mask;
 275         udp->mask.dst_port = mask->hdr.dst_port;
 276         udp->mask.src_port = mask->hdr.src_port;
 277         /* Remove unwanted bits from values. */
 278         udp->val.src_port &= udp->mask.src_port;
 279         udp->val.dst_port &= udp->mask.dst_port;
 280         return 0;
 281 }
 282
 283 /**
 284  * Convert TCP item to Verbs specification.
 285  *
 286  * @param item[in]
 287  *   Item specification.
 288  * @param default_mask[in]
 289  *   Default bit-masks to use when item->mask is not provided.
 290  * @param data[in, out]
 291  *   User structure.
 292  */
 293 static int
 294 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 295                      const void *default_mask,
 296                      void *data)
 297 {
 298         const struct rte_flow_item_tcp *spec = item->spec;
 299         const struct rte_flow_item_tcp *mask = item->mask;
 300         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 301         struct ibv_flow_spec_tcp_udp *tcp;
 302         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 303
 304         ++flow->ibv_attr->num_of_specs;
 305         flow->ibv_attr->priority = 0;
 306         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 307         *tcp = (struct ibv_flow_spec_tcp_udp) {
 308                 .type = IBV_FLOW_SPEC_TCP,
 309                 .size = tcp_size,
 310         };
 311         if (!spec)
 312                 return 0;
 313         tcp->val.dst_port = spec->hdr.dst_port;
 314         tcp->val.src_port = spec->hdr.src_port;
 315         if (!mask)
 316                 mask = default_mask;
 317         tcp->mask.dst_port = mask->hdr.dst_port;
 318         tcp->mask.src_port = mask->hdr.src_port;
 319         /* Remove unwanted bits from values. */
 320         tcp->val.src_port &= tcp->mask.src_port;
 321         tcp->val.dst_port &= tcp->mask.dst_port;
 322         return 0;
 323 }
 324
 325 /**
 326  * Check support for a given item.
 327  *
 328  * @param item[in]
 329  *   Item specification.
 330  * @param mask[in]
 331  *   Bit-masks covering supported fields to compare with spec, last and mask in
 332  *   \item.
 333  * @param size
 334  *   Bit-Mask size in bytes.
 335  *
 336  * @return
 337  *   0 on success, negative value otherwise.
 338  */
 339 static int
 340 mlx4_flow_item_validate(const struct rte_flow_item *item,
 341                         const uint8_t *mask, unsigned int size)
 342 {
 343         int ret = 0;
 344
 345         if (!item->spec && (item->mask || item->last))
 346                 return -1;
 347         if (item->spec && !item->mask) {
 348                 unsigned int i;
 349                 const uint8_t *spec = item->spec;
 350
 351                 for (i = 0; i < size; ++i)
 352                         if ((spec[i] | mask[i]) != mask[i])
 353                                 return -1;
 354         }
 355         if (item->last && !item->mask) {
 356                 unsigned int i;
 357                 const uint8_t *spec = item->last;
 358
 359                 for (i = 0; i < size; ++i)
 360                         if ((spec[i] | mask[i]) != mask[i])
 361                                 return -1;
 362         }
 363         if (item->spec && item->last) {
 364                 uint8_t spec[size];
 365                 uint8_t last[size];
 366                 const uint8_t *apply = mask;
 367                 unsigned int i;
 368
 369                 if (item->mask)
 370                         apply = item->mask;
 371                 for (i = 0; i < size; ++i) {
 372                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 373                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 374                 }
 375                 ret = memcmp(spec, last, size);
 376         }
 377         return ret;
 378 }
 379
 380 static int
 381 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 382                        const uint8_t *mask, unsigned int size)
 383 {
 384         if (item->mask) {
 385                 const struct rte_flow_item_eth *mask = item->mask;
 386
 387                 if (mask->dst.addr_bytes[0] != 0xff ||
 388                                 mask->dst.addr_bytes[1] != 0xff ||
 389                                 mask->dst.addr_bytes[2] != 0xff ||
 390                                 mask->dst.addr_bytes[3] != 0xff ||
 391                                 mask->dst.addr_bytes[4] != 0xff ||
 392                                 mask->dst.addr_bytes[5] != 0xff)
 393                         return -1;
 394         }
 395         return mlx4_flow_item_validate(item, mask, size);
 396 }
 397
 398 static int
 399 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 400                         const uint8_t *mask, unsigned int size)
 401 {
 402         if (item->mask) {
 403                 const struct rte_flow_item_vlan *mask = item->mask;
 404
 405                 if (mask->tci != 0 &&
 406                     ntohs(mask->tci) != 0x0fff)
 407                         return -1;
 408         }
 409         return mlx4_flow_item_validate(item, mask, size);
 410 }
 411
 412 static int
 413 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 414                         const uint8_t *mask, unsigned int size)
 415 {
 416         if (item->mask) {
 417                 const struct rte_flow_item_ipv4 *mask = item->mask;
 418
 419                 if (mask->hdr.src_addr != 0 &&
 420                     mask->hdr.src_addr != 0xffffffff)
 421                         return -1;
 422                 if (mask->hdr.dst_addr != 0 &&
 423                     mask->hdr.dst_addr != 0xffffffff)
 424                         return -1;
 425         }
 426         return mlx4_flow_item_validate(item, mask, size);
 427 }
 428
 429 static int
 430 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 431                        const uint8_t *mask, unsigned int size)
 432 {
 433         if (item->mask) {
 434                 const struct rte_flow_item_udp *mask = item->mask;
 435
 436                 if (mask->hdr.src_port != 0 &&
 437                     mask->hdr.src_port != 0xffff)
 438                         return -1;
 439                 if (mask->hdr.dst_port != 0 &&
 440                     mask->hdr.dst_port != 0xffff)
 441                         return -1;
 442         }
 443         return mlx4_flow_item_validate(item, mask, size);
 444 }
 445
 446 static int
 447 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 448                        const uint8_t *mask, unsigned int size)
 449 {
 450         if (item->mask) {
 451                 const struct rte_flow_item_tcp *mask = item->mask;
 452
 453                 if (mask->hdr.src_port != 0 &&
 454                     mask->hdr.src_port != 0xffff)
 455                         return -1;
 456                 if (mask->hdr.dst_port != 0 &&
 457                     mask->hdr.dst_port != 0xffff)
 458                         return -1;
 459         }
 460         return mlx4_flow_item_validate(item, mask, size);
 461 }
 462
 463 /** Graph of supported items and associated actions. */
 464 static const struct mlx4_flow_items mlx4_flow_items[] = {
 465         [RTE_FLOW_ITEM_TYPE_END] = {
 466                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 467         },
 468         [RTE_FLOW_ITEM_TYPE_ETH] = {
 469                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 470                                RTE_FLOW_ITEM_TYPE_IPV4),
 471                 .actions = valid_actions,
 472                 .mask = &(const struct rte_flow_item_eth){
 473                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 474                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 475                 },
 476                 .default_mask = &rte_flow_item_eth_mask,
 477                 .mask_sz = sizeof(struct rte_flow_item_eth),
 478                 .validate = mlx4_flow_validate_eth,
 479                 .convert = mlx4_flow_create_eth,
 480                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 481         },
 482         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 483                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 484                 .actions = valid_actions,
 485                 .mask = &(const struct rte_flow_item_vlan){
 486                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 487 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 488                         .tci = 0x0fff,
 489 #else
 490                         .tci = 0xff0f,
 491 #endif
 492                 },
 493                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 494                 .validate = mlx4_flow_validate_vlan,
 495                 .convert = mlx4_flow_create_vlan,
 496                 .dst_sz = 0,
 497         },
 498         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 499                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 500                                RTE_FLOW_ITEM_TYPE_TCP),
 501                 .actions = valid_actions,
 502                 .mask = &(const struct rte_flow_item_ipv4){
 503                         .hdr = {
 504                                 .src_addr = -1,
 505                                 .dst_addr = -1,
 506                         },
 507                 },
 508                 .default_mask = &rte_flow_item_ipv4_mask,
 509                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 510                 .validate = mlx4_flow_validate_ipv4,
 511                 .convert = mlx4_flow_create_ipv4,
 512                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 513         },
 514         [RTE_FLOW_ITEM_TYPE_UDP] = {
 515                 .actions = valid_actions,
 516                 .mask = &(const struct rte_flow_item_udp){
 517                         .hdr = {
 518                                 .src_port = -1,
 519                                 .dst_port = -1,
 520                         },
 521                 },
 522                 .default_mask = &rte_flow_item_udp_mask,
 523                 .mask_sz = sizeof(struct rte_flow_item_udp),
 524                 .validate = mlx4_flow_validate_udp,
 525                 .convert = mlx4_flow_create_udp,
 526                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 527         },
 528         [RTE_FLOW_ITEM_TYPE_TCP] = {
 529                 .actions = valid_actions,
 530                 .mask = &(const struct rte_flow_item_tcp){
 531                         .hdr = {
 532                                 .src_port = -1,
 533                                 .dst_port = -1,
 534                         },
 535                 },
 536                 .default_mask = &rte_flow_item_tcp_mask,
 537                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 538                 .validate = mlx4_flow_validate_tcp,
 539                 .convert = mlx4_flow_create_tcp,
 540                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 541         },
 542 };
 543
 544 /**
 545  * Validate a flow supported by the NIC.
 546  *
 547  * @param priv
 548  *   Pointer to private structure.
 549  * @param[in] attr
 550  *   Flow rule attributes.
 551  * @param[in] items
 552  *   Pattern specification (list terminated by the END pattern item).
 553  * @param[in] actions
 554  *   Associated actions (list terminated by the END action).
 555  * @param[out] error
 556  *   Perform verbose error reporting if not NULL.
 557  * @param[in, out] flow
 558  *   Flow structure to update.
 559  *
 560  * @return
 561  *   0 on success, a negative errno value otherwise and rte_errno is set.
 562  */
 563 static int
 564 priv_flow_validate(struct priv *priv,
 565                    const struct rte_flow_attr *attr,
 566                    const struct rte_flow_item items[],
 567                    const struct rte_flow_action actions[],
 568                    struct rte_flow_error *error,
 569                    struct mlx4_flow *flow)
 570 {
 571         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 572         struct mlx4_flow_action action = {
 573                 .queue = 0,
 574                 .drop = 0,
 575         };
 576
 577         (void)priv;
 578         if (attr->group) {
 579                 rte_flow_error_set(error, ENOTSUP,
 580                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 581                                    NULL,
 582                                    "groups are not supported");
 583                 return -rte_errno;
 584         }
 585         if (attr->priority) {
 586                 rte_flow_error_set(error, ENOTSUP,
 587                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 588                                    NULL,
 589                                    "priorities are not supported");
 590                 return -rte_errno;
 591         }
 592         if (attr->egress) {
 593                 rte_flow_error_set(error, ENOTSUP,
 594                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 595                                    NULL,
 596                                    "egress is not supported");
 597                 return -rte_errno;
 598         }
 599         if (!attr->ingress) {
 600                 rte_flow_error_set(error, ENOTSUP,
 601                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 602                                    NULL,
 603                                    "only ingress is supported");
 604                 return -rte_errno;
 605         }
 606         /* Go over items list. */
 607         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 608                 const struct mlx4_flow_items *token = NULL;
 609                 unsigned int i;
 610                 int err;
 611
 612                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 613                         continue;
 614                 /*
 615                  * The nic can support patterns with NULL eth spec only
 616                  * if eth is a single item in a rule.
 617                  */
 618                 if (!items->spec &&
 619                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 620                         const struct rte_flow_item *next = items + 1;
 621
 622                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 623                                 rte_flow_error_set(error, ENOTSUP,
 624                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 625                                                    items,
 626                                                    "the rule requires"
 627                                                    " an Ethernet spec");
 628                                 return -rte_errno;
 629                         }
 630                 }
 631                 for (i = 0;
 632                      cur_item->items &&
 633                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 634                      ++i) {
 635                         if (cur_item->items[i] == items->type) {
 636                                 token = &mlx4_flow_items[items->type];
 637                                 break;
 638                         }
 639                 }
 640                 if (!token)
 641                         goto exit_item_not_supported;
 642                 cur_item = token;
 643                 err = cur_item->validate(items,
 644                                         (const uint8_t *)cur_item->mask,
 645                                          cur_item->mask_sz);
 646                 if (err)
 647                         goto exit_item_not_supported;
 648                 if (flow->ibv_attr && cur_item->convert) {
 649                         err = cur_item->convert(items,
 650                                                 (cur_item->default_mask ?
 651                                                  cur_item->default_mask :
 652                                                  cur_item->mask),
 653                                                  flow);
 654                         if (err)
 655                                 goto exit_item_not_supported;
 656                 }
 657                 flow->offset += cur_item->dst_sz;
 658         }
 659         /* Go over actions list */
 660         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 661                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 662                         continue;
 663                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 664                         action.drop = 1;
 665                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 666                         const struct rte_flow_action_queue *queue =
 667                                 (const struct rte_flow_action_queue *)
 668                                 actions->conf;
 669
 670                         if (!queue || (queue->index > (priv->rxqs_n - 1)))
 671                                 goto exit_action_not_supported;
 672                         action.queue = 1;
 673                         action.queues_n = 1;
 674                         action.queues[0] = queue->index;
 675                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
 676                         int i;
 677                         int ierr;
 678                         const struct rte_flow_action_rss *rss =
 679                                 (const struct rte_flow_action_rss *)
 680                                 actions->conf;
 681
 682                         if (!priv->hw_rss) {
 683                                 rte_flow_error_set(error, ENOTSUP,
 684                                            RTE_FLOW_ERROR_TYPE_ACTION,
 685                                            actions,
 686                                            "RSS cannot be used with "
 687                                            "the current configuration");
 688                                 return -rte_errno;
 689                         }
 690                         if (!priv->isolated) {
 691                                 rte_flow_error_set(error, ENOTSUP,
 692                                            RTE_FLOW_ERROR_TYPE_ACTION,
 693                                            actions,
 694                                            "RSS cannot be used without "
 695                                            "isolated mode");
 696                                 return -rte_errno;
 697                         }
 698                         if (!rte_is_power_of_2(rss->num)) {
 699                                 rte_flow_error_set(error, ENOTSUP,
 700                                            RTE_FLOW_ERROR_TYPE_ACTION,
 701                                            actions,
 702                                            "the number of queues "
 703                                            "should be power of two");
 704                                 return -rte_errno;
 705                         }
 706                         if (priv->max_rss_tbl_sz < rss->num) {
 707                                 rte_flow_error_set(error, ENOTSUP,
 708                                            RTE_FLOW_ERROR_TYPE_ACTION,
 709                                            actions,
 710                                            "the number of queues "
 711                                            "is too large");
 712                                 return -rte_errno;
 713                         }
 714                         /* checking indexes array */
 715                         ierr = 0;
 716                         for (i = 0; i < rss->num; ++i) {
 717                                 int j;
 718                                 if (rss->queue[i] >= priv->rxqs_n)
 719                                         ierr = 1;
 720                                 /*
 721                                  * Prevent the user from specifying
 722                                  * the same queue twice in the RSS array.
 723                                  */
 724                                 for (j = i + 1; j < rss->num && !ierr; ++j)
 725                                         if (rss->queue[j] == rss->queue[i])
 726                                                 ierr = 1;
 727                                 if (ierr) {
 728                                         rte_flow_error_set(
 729                                                 error,
 730                                                 ENOTSUP,
 731                                                 RTE_FLOW_ERROR_TYPE_HANDLE,
 732                                                 NULL,
 733                                                 "RSS action only supports "
 734                                                 "unique queue indices "
 735                                                 "in a list");
 736                                         return -rte_errno;
 737                                 }
 738                         }
 739                         action.queue = 1;
 740                         action.queues_n = rss->num;
 741                         for (i = 0; i < rss->num; ++i)
 742                                 action.queues[i] = rss->queue[i];
 743                 } else {
 744                         goto exit_action_not_supported;
 745                 }
 746         }
 747         if (!action.queue && !action.drop) {
 748                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 749                                    NULL, "no valid action");
 750                 return -rte_errno;
 751         }
 752         return 0;
 753 exit_item_not_supported:
 754         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 755                            items, "item not supported");
 756         return -rte_errno;
 757 exit_action_not_supported:
 758         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 759                            actions, "action not supported");
 760         return -rte_errno;
 761 }
 762
 763 /**
 764  * Validate a flow supported by the NIC.
 765  *
 766  * @see rte_flow_validate()
 767  * @see rte_flow_ops
 768  */
 769 int
 770 mlx4_flow_validate(struct rte_eth_dev *dev,
 771                    const struct rte_flow_attr *attr,
 772                    const struct rte_flow_item items[],
 773                    const struct rte_flow_action actions[],
 774                    struct rte_flow_error *error)
 775 {
 776         struct priv *priv = dev->data->dev_private;
 777         int ret;
 778         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 779
 780         priv_lock(priv);
 781         ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
 782         priv_unlock(priv);
 783         return ret;
 784 }
 785
 786 /**
 787  * Destroy a drop queue.
 788  *
 789  * @param priv
 790  *   Pointer to private structure.
 791  */
 792 static void
 793 mlx4_flow_destroy_drop_queue(struct priv *priv)
 794 {
 795         if (priv->flow_drop_queue) {
 796                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 797
 798                 priv->flow_drop_queue = NULL;
 799                 claim_zero(ibv_destroy_qp(fdq->qp));
 800                 claim_zero(ibv_destroy_cq(fdq->cq));
 801                 rte_free(fdq);
 802         }
 803 }
 804
 805 /**
 806  * Create a single drop queue for all drop flows.
 807  *
 808  * @param priv
 809  *   Pointer to private structure.
 810  *
 811  * @return
 812  *   0 on success, negative value otherwise.
 813  */
 814 static int
 815 mlx4_flow_create_drop_queue(struct priv *priv)
 816 {
 817         struct ibv_qp *qp;
 818         struct ibv_cq *cq;
 819         struct rte_flow_drop *fdq;
 820
 821         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 822         if (!fdq) {
 823                 ERROR("Cannot allocate memory for drop struct");
 824                 goto err;
 825         }
 826         cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
 827                               &(struct ibv_exp_cq_init_attr){
 828                                         .comp_mask = 0,
 829                               });
 830         if (!cq) {
 831                 ERROR("Cannot create drop CQ");
 832                 goto err_create_cq;
 833         }
 834         qp = ibv_exp_create_qp(priv->ctx,
 835                               &(struct ibv_exp_qp_init_attr){
 836                                         .send_cq = cq,
 837                                         .recv_cq = cq,
 838                                         .cap = {
 839                                                 .max_recv_wr = 1,
 840                                                 .max_recv_sge = 1,
 841                                         },
 842                                         .qp_type = IBV_QPT_RAW_PACKET,
 843                                         .comp_mask =
 844                                                 IBV_EXP_QP_INIT_ATTR_PD |
 845                                                 IBV_EXP_QP_INIT_ATTR_PORT,
 846                                         .pd = priv->pd,
 847                                         .port_num = priv->port,
 848                               });
 849         if (!qp) {
 850                 ERROR("Cannot create drop QP");
 851                 goto err_create_qp;
 852         }
 853         *fdq = (struct rte_flow_drop){
 854                 .qp = qp,
 855                 .cq = cq,
 856         };
 857         priv->flow_drop_queue = fdq;
 858         return 0;
 859 err_create_qp:
 860         claim_zero(ibv_destroy_cq(cq));
 861 err_create_cq:
 862         rte_free(fdq);
 863 err:
 864         return -1;
 865 }
 866
 867 /**
 868  * Get RSS parent rxq structure for given queues.
 869  *
 870  * Creates a new or returns an existed one.
 871  *
 872  * @param priv
 873  *   Pointer to private structure.
 874  * @param queues
 875  *   queues indices array, NULL in default RSS case.
 876  * @param children_n
 877  *   the size of queues array.
 878  *
 879  * @return
 880  *   Pointer to a parent rxq structure, NULL on failure.
 881  */
 882 static struct rxq *
 883 priv_parent_get(struct priv *priv,
 884                 uint16_t queues[],
 885                 uint16_t children_n,
 886                 struct rte_flow_error *error)
 887 {
 888         unsigned int i;
 889         struct rxq *parent;
 890
 891         for (parent = LIST_FIRST(&priv->parents);
 892              parent;
 893              parent = LIST_NEXT(parent, next)) {
 894                 unsigned int same = 0;
 895                 unsigned int overlap = 0;
 896
 897                 /*
 898                  * Find out whether an appropriate parent queue already exists
 899                  * and can be reused, otherwise make sure there are no overlaps.
 900                  */
 901                 for (i = 0; i < children_n; ++i) {
 902                         unsigned int j;
 903
 904                         for (j = 0; j < parent->rss.queues_n; ++j) {
 905                                 if (parent->rss.queues[j] != queues[i])
 906                                         continue;
 907                                 ++overlap;
 908                                 if (i == j)
 909                                         ++same;
 910                         }
 911                 }
 912                 if (same == children_n &&
 913                         children_n == parent->rss.queues_n)
 914                         return parent;
 915                 else if (overlap)
 916                         goto error;
 917         }
 918         /* Exclude the cases when some QPs were created without RSS */
 919         for (i = 0; i < children_n; ++i) {
 920                 struct rxq *rxq = (*priv->rxqs)[queues[i]];
 921                 if (rxq->qp)
 922                         goto error;
 923         }
 924         parent = priv_parent_create(priv, queues, children_n);
 925         if (!parent) {
 926                 rte_flow_error_set(error,
 927                                    ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 928                                    NULL, "flow rule creation failure");
 929                 return NULL;
 930         }
 931         return parent;
 932
 933 error:
 934         rte_flow_error_set(error,
 935                            EEXIST,
 936                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 937                            NULL,
 938                            "sharing a queue between several"
 939                            " RSS groups is not supported");
 940         return NULL;
 941 }
 942
 943 /**
 944  * Complete flow rule creation.
 945  *
 946  * @param priv
 947  *   Pointer to private structure.
 948  * @param ibv_attr
 949  *   Verbs flow attributes.
 950  * @param action
 951  *   Target action structure.
 952  * @param[out] error
 953  *   Perform verbose error reporting if not NULL.
 954  *
 955  * @return
 956  *   A flow if the rule could be created.
 957  */
 958 static struct rte_flow *
 959 priv_flow_create_action_queue(struct priv *priv,
 960                               struct ibv_flow_attr *ibv_attr,
 961                               struct mlx4_flow_action *action,
 962                               struct rte_flow_error *error)
 963 {
 964         struct ibv_qp *qp;
 965         struct rte_flow *rte_flow;
 966         struct rxq *rxq_parent = NULL;
 967
 968         assert(priv->pd);
 969         assert(priv->ctx);
 970         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 971         if (!rte_flow) {
 972                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 973                                    NULL, "cannot allocate flow memory");
 974                 return NULL;
 975         }
 976         if (action->drop) {
 977                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 978         } else {
 979                 int ret;
 980                 unsigned int i;
 981                 struct rxq *rxq = NULL;
 982
 983                 if (action->queues_n > 1) {
 984                         rxq_parent = priv_parent_get(priv, action->queues,
 985                                                      action->queues_n, error);
 986                         if (!rxq_parent)
 987                                 goto error;
 988                 }
 989                 for (i = 0; i < action->queues_n; ++i) {
 990                         rxq = (*priv->rxqs)[action->queues[i]];
 991                         /*
 992                          * In case of isolated mode we postpone
 993                          * ibv receive queue creation till the first
 994                          * rte_flow rule will be applied on that queue.
 995                          */
 996                         if (!rxq->qp) {
 997                                 assert(priv->isolated);
 998                                 ret = rxq_create_qp(rxq, rxq->elts_n,
 999                                                     0, 0, rxq_parent);
1000                                 if (ret) {
1001                                         rte_flow_error_set(
1002                                                 error,
1003                                                 ENOMEM,
1004                                                 RTE_FLOW_ERROR_TYPE_HANDLE,
1005                                                 NULL,
1006                                                 "flow rule creation failure");
1007                                         goto error;
1008                                 }
1009                         }
1010                 }
1011                 qp = action->queues_n > 1 ? rxq_parent->qp : rxq->qp;
1012                 rte_flow->qp = qp;
1013         }
1014         rte_flow->ibv_attr = ibv_attr;
1015         if (!priv->started)
1016                 return rte_flow;
1017         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
1018         if (!rte_flow->ibv_flow) {
1019                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1020                                    NULL, "flow rule creation failure");
1021                 goto error;
1022         }
1023         return rte_flow;
1024
1025 error:
1026         if (rxq_parent)
1027                 rxq_parent_cleanup(rxq_parent);
1028         rte_free(rte_flow);
1029         return NULL;
1030 }
1031
1032 /**
1033  * Convert a flow.
1034  *
1035  * @param priv
1036  *   Pointer to private structure.
1037  * @param[in] attr
1038  *   Flow rule attributes.
1039  * @param[in] items
1040  *   Pattern specification (list terminated by the END pattern item).
1041  * @param[in] actions
1042  *   Associated actions (list terminated by the END action).
1043  * @param[out] error
1044  *   Perform verbose error reporting if not NULL.
1045  *
1046  * @return
1047  *   A flow on success, NULL otherwise.
1048  */
1049 static struct rte_flow *
1050 priv_flow_create(struct priv *priv,
1051                  const struct rte_flow_attr *attr,
1052                  const struct rte_flow_item items[],
1053                  const struct rte_flow_action actions[],
1054                  struct rte_flow_error *error)
1055 {
1056         struct rte_flow *rte_flow;
1057         struct mlx4_flow_action action;
1058         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
1059         int err;
1060
1061         err = priv_flow_validate(priv, attr, items, actions, error, &flow);
1062         if (err)
1063                 return NULL;
1064         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
1065         if (!flow.ibv_attr) {
1066                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1067                                    NULL, "cannot allocate ibv_attr memory");
1068                 return NULL;
1069         }
1070         flow.offset = sizeof(struct ibv_flow_attr);
1071         *flow.ibv_attr = (struct ibv_flow_attr){
1072                 .comp_mask = 0,
1073                 .type = IBV_FLOW_ATTR_NORMAL,
1074                 .size = sizeof(struct ibv_flow_attr),
1075                 .priority = attr->priority,
1076                 .num_of_specs = 0,
1077                 .port = priv->port,
1078                 .flags = 0,
1079         };
1080         claim_zero(priv_flow_validate(priv, attr, items, actions,
1081                                       error, &flow));
1082         action = (struct mlx4_flow_action){
1083                 .queue = 0,
1084                 .drop = 0,
1085         };
1086         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1087                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1088                         continue;
1089                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1090                         action.queue = 1;
1091                         action.queues_n = 1;
1092                         action.queues[0] =
1093                                 ((const struct rte_flow_action_queue *)
1094                                  actions->conf)->index;
1095                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1096                         action.drop = 1;
1097                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
1098                         unsigned int i;
1099                         const struct rte_flow_action_rss *rss =
1100                                 (const struct rte_flow_action_rss *)
1101                                  actions->conf;
1102
1103                         action.queue = 1;
1104                         action.queues_n = rss->num;
1105                         for (i = 0; i < rss->num; ++i)
1106                                 action.queues[i] = rss->queue[i];
1107                 } else {
1108                         rte_flow_error_set(error, ENOTSUP,
1109                                            RTE_FLOW_ERROR_TYPE_ACTION,
1110                                            actions, "unsupported action");
1111                         goto exit;
1112                 }
1113         }
1114         rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
1115                                                  &action, error);
1116         if (rte_flow)
1117                 return rte_flow;
1118 exit:
1119         rte_free(flow.ibv_attr);
1120         return NULL;
1121 }
1122
1123 /**
1124  * Create a flow.
1125  *
1126  * @see rte_flow_create()
1127  * @see rte_flow_ops
1128  */
1129 struct rte_flow *
1130 mlx4_flow_create(struct rte_eth_dev *dev,
1131                  const struct rte_flow_attr *attr,
1132                  const struct rte_flow_item items[],
1133                  const struct rte_flow_action actions[],
1134                  struct rte_flow_error *error)
1135 {
1136         struct priv *priv = dev->data->dev_private;
1137         struct rte_flow *flow;
1138
1139         priv_lock(priv);
1140         flow = priv_flow_create(priv, attr, items, actions, error);
1141         if (flow) {
1142                 LIST_INSERT_HEAD(&priv->flows, flow, next);
1143                 DEBUG("Flow created %p", (void *)flow);
1144         }
1145         priv_unlock(priv);
1146         return flow;
1147 }
1148
1149 /**
1150  * @see rte_flow_isolate()
1151  *
1152  * Must be done before calling dev_configure().
1153  *
1154  * @param dev
1155  *   Pointer to the ethernet device structure.
1156  * @param enable
1157  *   Nonzero to enter isolated mode, attempt to leave it otherwise.
1158  * @param[out] error
1159  *   Perform verbose error reporting if not NULL. PMDs initialize this
1160  *   structure in case of error only.
1161  *
1162  * @return
1163  *   0 on success, a negative value on error.
1164  */
1165 int
1166 mlx4_flow_isolate(struct rte_eth_dev *dev,
1167                   int enable,
1168                   struct rte_flow_error *error)
1169 {
1170         struct priv *priv = dev->data->dev_private;
1171
1172         priv_lock(priv);
1173         if (priv->rxqs) {
1174                 rte_flow_error_set(error, ENOTSUP,
1175                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1176                                    NULL, "isolated mode must be set"
1177                                    " before configuring the device");
1178                 priv_unlock(priv);
1179                 return -rte_errno;
1180         }
1181         priv->isolated = !!enable;
1182         priv_unlock(priv);
1183         return 0;
1184 }
1185
1186 /**
1187  * Destroy a flow.
1188  *
1189  * @param priv
1190  *   Pointer to private structure.
1191  * @param[in] flow
1192  *   Flow to destroy.
1193  */
1194 static void
1195 priv_flow_destroy(struct priv *priv, struct rte_flow *flow)
1196 {
1197         (void)priv;
1198         LIST_REMOVE(flow, next);
1199         if (flow->ibv_flow)
1200                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1201         rte_free(flow->ibv_attr);
1202         DEBUG("Flow destroyed %p", (void *)flow);
1203         rte_free(flow);
1204 }
1205
1206 /**
1207  * Destroy a flow.
1208  *
1209  * @see rte_flow_destroy()
1210  * @see rte_flow_ops
1211  */
1212 int
1213 mlx4_flow_destroy(struct rte_eth_dev *dev,
1214                   struct rte_flow *flow,
1215                   struct rte_flow_error *error)
1216 {
1217         struct priv *priv = dev->data->dev_private;
1218
1219         (void)error;
1220         priv_lock(priv);
1221         priv_flow_destroy(priv, flow);
1222         priv_unlock(priv);
1223         return 0;
1224 }
1225
1226 /**
1227  * Destroy all flows.
1228  *
1229  * @param priv
1230  *   Pointer to private structure.
1231  */
1232 static void
1233 priv_flow_flush(struct priv *priv)
1234 {
1235         while (!LIST_EMPTY(&priv->flows)) {
1236                 struct rte_flow *flow;
1237
1238                 flow = LIST_FIRST(&priv->flows);
1239                 priv_flow_destroy(priv, flow);
1240         }
1241 }
1242
1243 /**
1244  * Destroy all flows.
1245  *
1246  * @see rte_flow_flush()
1247  * @see rte_flow_ops
1248  */
1249 int
1250 mlx4_flow_flush(struct rte_eth_dev *dev,
1251                 struct rte_flow_error *error)
1252 {
1253         struct priv *priv = dev->data->dev_private;
1254
1255         (void)error;
1256         priv_lock(priv);
1257         priv_flow_flush(priv);
1258         priv_unlock(priv);
1259         return 0;
1260 }
1261
1262 /**
1263  * Remove all flows.
1264  *
1265  * Called by dev_stop() to remove all flows.
1266  *
1267  * @param priv
1268  *   Pointer to private structure.
1269  */
1270 void
1271 mlx4_priv_flow_stop(struct priv *priv)
1272 {
1273         struct rte_flow *flow;
1274
1275         for (flow = LIST_FIRST(&priv->flows);
1276              flow;
1277              flow = LIST_NEXT(flow, next)) {
1278                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1279                 flow->ibv_flow = NULL;
1280                 DEBUG("Flow %p removed", (void *)flow);
1281         }
1282         mlx4_flow_destroy_drop_queue(priv);
1283 }
1284
1285 /**
1286  * Add all flows.
1287  *
1288  * @param priv
1289  *   Pointer to private structure.
1290  *
1291  * @return
1292  *   0 on success, a errno value otherwise and rte_errno is set.
1293  */
1294 int
1295 mlx4_priv_flow_start(struct priv *priv)
1296 {
1297         int ret;
1298         struct ibv_qp *qp;
1299         struct rte_flow *flow;
1300
1301         ret = mlx4_flow_create_drop_queue(priv);
1302         if (ret)
1303                 return -1;
1304         for (flow = LIST_FIRST(&priv->flows);
1305              flow;
1306              flow = LIST_NEXT(flow, next)) {
1307                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1308                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1309                 if (!flow->ibv_flow) {
1310                         DEBUG("Flow %p cannot be applied", (void *)flow);
1311                         rte_errno = EINVAL;
1312                         return rte_errno;
1313                 }
1314                 DEBUG("Flow %p applied", (void *)flow);
1315         }
1316         return 0;
1317 }