drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <assert.h>
  35
  36 #include <rte_flow.h>
  37 #include <rte_flow_driver.h>
  38 #include <rte_malloc.h>
  39
  40 /* PMD headers. */
  41 #include "mlx4.h"
  42 #include "mlx4_flow.h"
  43 #include "mlx4_utils.h"
  44
  45 /** Static initializer for items. */
  46 #define ITEMS(...) \
  47         (const enum rte_flow_item_type []){ \
  48                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  49         }
  50
  51 /** Structure to generate a simple graph of layers supported by the NIC. */
  52 struct mlx4_flow_items {
  53         /** List of possible actions for these items. */
  54         const enum rte_flow_action_type *const actions;
  55         /** Bit-masks corresponding to the possibilities for the item. */
  56         const void *mask;
  57         /**
  58          * Default bit-masks to use when item->mask is not provided. When
  59          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  60          * used instead.
  61          */
  62         const void *default_mask;
  63         /** Bit-masks size in bytes. */
  64         const unsigned int mask_sz;
  65         /**
  66          * Check support for a given item.
  67          *
  68          * @param item[in]
  69          *   Item specification.
  70          * @param mask[in]
  71          *   Bit-masks covering supported fields to compare with spec,
  72          *   last and mask in
  73          *   \item.
  74          * @param size
  75          *   Bit-Mask size in bytes.
  76          *
  77          * @return
  78          *   0 on success, negative value otherwise.
  79          */
  80         int (*validate)(const struct rte_flow_item *item,
  81                         const uint8_t *mask, unsigned int size);
  82         /**
  83          * Conversion function from rte_flow to NIC specific flow.
  84          *
  85          * @param item
  86          *   rte_flow item to convert.
  87          * @param default_mask
  88          *   Default bit-masks to use when item->mask is not provided.
  89          * @param data
  90          *   Internal structure to store the conversion.
  91          *
  92          * @return
  93          *   0 on success, negative value otherwise.
  94          */
  95         int (*convert)(const struct rte_flow_item *item,
  96                        const void *default_mask,
  97                        void *data);
  98         /** Size in bytes of the destination structure. */
  99         const unsigned int dst_sz;
 100         /** List of possible following items.  */
 101         const enum rte_flow_item_type *const items;
 102 };
 103
 104 struct rte_flow_drop {
 105         struct ibv_qp *qp; /**< Verbs queue pair. */
 106         struct ibv_cq *cq; /**< Verbs completion queue. */
 107 };
 108
 109 /** Valid action for this PMD. */
 110 static const enum rte_flow_action_type valid_actions[] = {
 111         RTE_FLOW_ACTION_TYPE_DROP,
 112         RTE_FLOW_ACTION_TYPE_QUEUE,
 113         RTE_FLOW_ACTION_TYPE_END,
 114 };
 115
 116 /**
 117  * Convert Ethernet item to Verbs specification.
 118  *
 119  * @param item[in]
 120  *   Item specification.
 121  * @param default_mask[in]
 122  *   Default bit-masks to use when item->mask is not provided.
 123  * @param data[in, out]
 124  *   User structure.
 125  */
 126 static int
 127 mlx4_flow_create_eth(const struct rte_flow_item *item,
 128                      const void *default_mask,
 129                      void *data)
 130 {
 131         const struct rte_flow_item_eth *spec = item->spec;
 132         const struct rte_flow_item_eth *mask = item->mask;
 133         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 134         struct ibv_flow_spec_eth *eth;
 135         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 136         unsigned int i;
 137
 138         ++flow->ibv_attr->num_of_specs;
 139         flow->ibv_attr->priority = 2;
 140         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 141         *eth = (struct ibv_flow_spec_eth) {
 142                 .type = IBV_FLOW_SPEC_ETH,
 143                 .size = eth_size,
 144         };
 145         if (!spec) {
 146                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 147                 return 0;
 148         }
 149         if (!mask)
 150                 mask = default_mask;
 151         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 152         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 153         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 154         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 155         /* Remove unwanted bits from values. */
 156         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 157                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 158                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 159         }
 160         return 0;
 161 }
 162
 163 /**
 164  * Convert VLAN item to Verbs specification.
 165  *
 166  * @param item[in]
 167  *   Item specification.
 168  * @param default_mask[in]
 169  *   Default bit-masks to use when item->mask is not provided.
 170  * @param data[in, out]
 171  *   User structure.
 172  */
 173 static int
 174 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 175                       const void *default_mask,
 176                       void *data)
 177 {
 178         const struct rte_flow_item_vlan *spec = item->spec;
 179         const struct rte_flow_item_vlan *mask = item->mask;
 180         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 181         struct ibv_flow_spec_eth *eth;
 182         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 183
 184         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 185         if (!spec)
 186                 return 0;
 187         if (!mask)
 188                 mask = default_mask;
 189         eth->val.vlan_tag = spec->tci;
 190         eth->mask.vlan_tag = mask->tci;
 191         eth->val.vlan_tag &= eth->mask.vlan_tag;
 192         return 0;
 193 }
 194
 195 /**
 196  * Convert IPv4 item to Verbs specification.
 197  *
 198  * @param item[in]
 199  *   Item specification.
 200  * @param default_mask[in]
 201  *   Default bit-masks to use when item->mask is not provided.
 202  * @param data[in, out]
 203  *   User structure.
 204  */
 205 static int
 206 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 207                       const void *default_mask,
 208                       void *data)
 209 {
 210         const struct rte_flow_item_ipv4 *spec = item->spec;
 211         const struct rte_flow_item_ipv4 *mask = item->mask;
 212         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 213         struct ibv_flow_spec_ipv4 *ipv4;
 214         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 215
 216         ++flow->ibv_attr->num_of_specs;
 217         flow->ibv_attr->priority = 1;
 218         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 219         *ipv4 = (struct ibv_flow_spec_ipv4) {
 220                 .type = IBV_FLOW_SPEC_IPV4,
 221                 .size = ipv4_size,
 222         };
 223         if (!spec)
 224                 return 0;
 225         ipv4->val = (struct ibv_flow_ipv4_filter) {
 226                 .src_ip = spec->hdr.src_addr,
 227                 .dst_ip = spec->hdr.dst_addr,
 228         };
 229         if (!mask)
 230                 mask = default_mask;
 231         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 232                 .src_ip = mask->hdr.src_addr,
 233                 .dst_ip = mask->hdr.dst_addr,
 234         };
 235         /* Remove unwanted bits from values. */
 236         ipv4->val.src_ip &= ipv4->mask.src_ip;
 237         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 238         return 0;
 239 }
 240
 241 /**
 242  * Convert UDP item to Verbs specification.
 243  *
 244  * @param item[in]
 245  *   Item specification.
 246  * @param default_mask[in]
 247  *   Default bit-masks to use when item->mask is not provided.
 248  * @param data[in, out]
 249  *   User structure.
 250  */
 251 static int
 252 mlx4_flow_create_udp(const struct rte_flow_item *item,
 253                      const void *default_mask,
 254                      void *data)
 255 {
 256         const struct rte_flow_item_udp *spec = item->spec;
 257         const struct rte_flow_item_udp *mask = item->mask;
 258         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 259         struct ibv_flow_spec_tcp_udp *udp;
 260         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 261
 262         ++flow->ibv_attr->num_of_specs;
 263         flow->ibv_attr->priority = 0;
 264         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 265         *udp = (struct ibv_flow_spec_tcp_udp) {
 266                 .type = IBV_FLOW_SPEC_UDP,
 267                 .size = udp_size,
 268         };
 269         if (!spec)
 270                 return 0;
 271         udp->val.dst_port = spec->hdr.dst_port;
 272         udp->val.src_port = spec->hdr.src_port;
 273         if (!mask)
 274                 mask = default_mask;
 275         udp->mask.dst_port = mask->hdr.dst_port;
 276         udp->mask.src_port = mask->hdr.src_port;
 277         /* Remove unwanted bits from values. */
 278         udp->val.src_port &= udp->mask.src_port;
 279         udp->val.dst_port &= udp->mask.dst_port;
 280         return 0;
 281 }
 282
 283 /**
 284  * Convert TCP item to Verbs specification.
 285  *
 286  * @param item[in]
 287  *   Item specification.
 288  * @param default_mask[in]
 289  *   Default bit-masks to use when item->mask is not provided.
 290  * @param data[in, out]
 291  *   User structure.
 292  */
 293 static int
 294 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 295                      const void *default_mask,
 296                      void *data)
 297 {
 298         const struct rte_flow_item_tcp *spec = item->spec;
 299         const struct rte_flow_item_tcp *mask = item->mask;
 300         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 301         struct ibv_flow_spec_tcp_udp *tcp;
 302         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 303
 304         ++flow->ibv_attr->num_of_specs;
 305         flow->ibv_attr->priority = 0;
 306         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 307         *tcp = (struct ibv_flow_spec_tcp_udp) {
 308                 .type = IBV_FLOW_SPEC_TCP,
 309                 .size = tcp_size,
 310         };
 311         if (!spec)
 312                 return 0;
 313         tcp->val.dst_port = spec->hdr.dst_port;
 314         tcp->val.src_port = spec->hdr.src_port;
 315         if (!mask)
 316                 mask = default_mask;
 317         tcp->mask.dst_port = mask->hdr.dst_port;
 318         tcp->mask.src_port = mask->hdr.src_port;
 319         /* Remove unwanted bits from values. */
 320         tcp->val.src_port &= tcp->mask.src_port;
 321         tcp->val.dst_port &= tcp->mask.dst_port;
 322         return 0;
 323 }
 324
 325 /**
 326  * Check support for a given item.
 327  *
 328  * @param item[in]
 329  *   Item specification.
 330  * @param mask[in]
 331  *   Bit-masks covering supported fields to compare with spec, last and mask in
 332  *   \item.
 333  * @param size
 334  *   Bit-Mask size in bytes.
 335  *
 336  * @return
 337  *   0 on success, negative value otherwise.
 338  */
 339 static int
 340 mlx4_flow_item_validate(const struct rte_flow_item *item,
 341                         const uint8_t *mask, unsigned int size)
 342 {
 343         int ret = 0;
 344
 345         if (!item->spec && (item->mask || item->last))
 346                 return -1;
 347         if (item->spec && !item->mask) {
 348                 unsigned int i;
 349                 const uint8_t *spec = item->spec;
 350
 351                 for (i = 0; i < size; ++i)
 352                         if ((spec[i] | mask[i]) != mask[i])
 353                                 return -1;
 354         }
 355         if (item->last && !item->mask) {
 356                 unsigned int i;
 357                 const uint8_t *spec = item->last;
 358
 359                 for (i = 0; i < size; ++i)
 360                         if ((spec[i] | mask[i]) != mask[i])
 361                                 return -1;
 362         }
 363         if (item->spec && item->last) {
 364                 uint8_t spec[size];
 365                 uint8_t last[size];
 366                 const uint8_t *apply = mask;
 367                 unsigned int i;
 368
 369                 if (item->mask)
 370                         apply = item->mask;
 371                 for (i = 0; i < size; ++i) {
 372                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 373                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 374                 }
 375                 ret = memcmp(spec, last, size);
 376         }
 377         return ret;
 378 }
 379
 380 static int
 381 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 382                        const uint8_t *mask, unsigned int size)
 383 {
 384         if (item->mask) {
 385                 const struct rte_flow_item_eth *mask = item->mask;
 386
 387                 if (mask->dst.addr_bytes[0] != 0xff ||
 388                                 mask->dst.addr_bytes[1] != 0xff ||
 389                                 mask->dst.addr_bytes[2] != 0xff ||
 390                                 mask->dst.addr_bytes[3] != 0xff ||
 391                                 mask->dst.addr_bytes[4] != 0xff ||
 392                                 mask->dst.addr_bytes[5] != 0xff)
 393                         return -1;
 394         }
 395         return mlx4_flow_item_validate(item, mask, size);
 396 }
 397
 398 static int
 399 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 400                         const uint8_t *mask, unsigned int size)
 401 {
 402         if (item->mask) {
 403                 const struct rte_flow_item_vlan *mask = item->mask;
 404
 405                 if (mask->tci != 0 &&
 406                     ntohs(mask->tci) != 0x0fff)
 407                         return -1;
 408         }
 409         return mlx4_flow_item_validate(item, mask, size);
 410 }
 411
 412 static int
 413 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 414                         const uint8_t *mask, unsigned int size)
 415 {
 416         if (item->mask) {
 417                 const struct rte_flow_item_ipv4 *mask = item->mask;
 418
 419                 if (mask->hdr.src_addr != 0 &&
 420                     mask->hdr.src_addr != 0xffffffff)
 421                         return -1;
 422                 if (mask->hdr.dst_addr != 0 &&
 423                     mask->hdr.dst_addr != 0xffffffff)
 424                         return -1;
 425         }
 426         return mlx4_flow_item_validate(item, mask, size);
 427 }
 428
 429 static int
 430 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 431                        const uint8_t *mask, unsigned int size)
 432 {
 433         if (item->mask) {
 434                 const struct rte_flow_item_udp *mask = item->mask;
 435
 436                 if (mask->hdr.src_port != 0 &&
 437                     mask->hdr.src_port != 0xffff)
 438                         return -1;
 439                 if (mask->hdr.dst_port != 0 &&
 440                     mask->hdr.dst_port != 0xffff)
 441                         return -1;
 442         }
 443         return mlx4_flow_item_validate(item, mask, size);
 444 }
 445
 446 static int
 447 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 448                        const uint8_t *mask, unsigned int size)
 449 {
 450         if (item->mask) {
 451                 const struct rte_flow_item_tcp *mask = item->mask;
 452
 453                 if (mask->hdr.src_port != 0 &&
 454                     mask->hdr.src_port != 0xffff)
 455                         return -1;
 456                 if (mask->hdr.dst_port != 0 &&
 457                     mask->hdr.dst_port != 0xffff)
 458                         return -1;
 459         }
 460         return mlx4_flow_item_validate(item, mask, size);
 461 }
 462
 463 /** Graph of supported items and associated actions. */
 464 static const struct mlx4_flow_items mlx4_flow_items[] = {
 465         [RTE_FLOW_ITEM_TYPE_END] = {
 466                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 467         },
 468         [RTE_FLOW_ITEM_TYPE_ETH] = {
 469                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 470                                RTE_FLOW_ITEM_TYPE_IPV4),
 471                 .actions = valid_actions,
 472                 .mask = &(const struct rte_flow_item_eth){
 473                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 474                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 475                 },
 476                 .default_mask = &rte_flow_item_eth_mask,
 477                 .mask_sz = sizeof(struct rte_flow_item_eth),
 478                 .validate = mlx4_flow_validate_eth,
 479                 .convert = mlx4_flow_create_eth,
 480                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 481         },
 482         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 483                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 484                 .actions = valid_actions,
 485                 .mask = &(const struct rte_flow_item_vlan){
 486                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 487 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 488                         .tci = 0x0fff,
 489 #else
 490                         .tci = 0xff0f,
 491 #endif
 492                 },
 493                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 494                 .validate = mlx4_flow_validate_vlan,
 495                 .convert = mlx4_flow_create_vlan,
 496                 .dst_sz = 0,
 497         },
 498         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 499                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 500                                RTE_FLOW_ITEM_TYPE_TCP),
 501                 .actions = valid_actions,
 502                 .mask = &(const struct rte_flow_item_ipv4){
 503                         .hdr = {
 504                                 .src_addr = -1,
 505                                 .dst_addr = -1,
 506                         },
 507                 },
 508                 .default_mask = &rte_flow_item_ipv4_mask,
 509                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 510                 .validate = mlx4_flow_validate_ipv4,
 511                 .convert = mlx4_flow_create_ipv4,
 512                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 513         },
 514         [RTE_FLOW_ITEM_TYPE_UDP] = {
 515                 .actions = valid_actions,
 516                 .mask = &(const struct rte_flow_item_udp){
 517                         .hdr = {
 518                                 .src_port = -1,
 519                                 .dst_port = -1,
 520                         },
 521                 },
 522                 .default_mask = &rte_flow_item_udp_mask,
 523                 .mask_sz = sizeof(struct rte_flow_item_udp),
 524                 .validate = mlx4_flow_validate_udp,
 525                 .convert = mlx4_flow_create_udp,
 526                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 527         },
 528         [RTE_FLOW_ITEM_TYPE_TCP] = {
 529                 .actions = valid_actions,
 530                 .mask = &(const struct rte_flow_item_tcp){
 531                         .hdr = {
 532                                 .src_port = -1,
 533                                 .dst_port = -1,
 534                         },
 535                 },
 536                 .default_mask = &rte_flow_item_tcp_mask,
 537                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 538                 .validate = mlx4_flow_validate_tcp,
 539                 .convert = mlx4_flow_create_tcp,
 540                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 541         },
 542 };
 543
 544 /**
 545  * Validate a flow supported by the NIC.
 546  *
 547  * @param priv
 548  *   Pointer to private structure.
 549  * @param[in] attr
 550  *   Flow rule attributes.
 551  * @param[in] items
 552  *   Pattern specification (list terminated by the END pattern item).
 553  * @param[in] actions
 554  *   Associated actions (list terminated by the END action).
 555  * @param[out] error
 556  *   Perform verbose error reporting if not NULL.
 557  * @param[in, out] flow
 558  *   Flow structure to update.
 559  *
 560  * @return
 561  *   0 on success, a negative errno value otherwise and rte_errno is set.
 562  */
 563 static int
 564 priv_flow_validate(struct priv *priv,
 565                    const struct rte_flow_attr *attr,
 566                    const struct rte_flow_item items[],
 567                    const struct rte_flow_action actions[],
 568                    struct rte_flow_error *error,
 569                    struct mlx4_flow *flow)
 570 {
 571         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 572         struct mlx4_flow_action action = {
 573                 .queue = 0,
 574                 .drop = 0,
 575         };
 576
 577         (void)priv;
 578         if (attr->group) {
 579                 rte_flow_error_set(error, ENOTSUP,
 580                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 581                                    NULL,
 582                                    "groups are not supported");
 583                 return -rte_errno;
 584         }
 585         if (attr->priority) {
 586                 rte_flow_error_set(error, ENOTSUP,
 587                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 588                                    NULL,
 589                                    "priorities are not supported");
 590                 return -rte_errno;
 591         }
 592         if (attr->egress) {
 593                 rte_flow_error_set(error, ENOTSUP,
 594                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 595                                    NULL,
 596                                    "egress is not supported");
 597                 return -rte_errno;
 598         }
 599         if (!attr->ingress) {
 600                 rte_flow_error_set(error, ENOTSUP,
 601                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 602                                    NULL,
 603                                    "only ingress is supported");
 604                 return -rte_errno;
 605         }
 606         /* Go over items list. */
 607         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 608                 const struct mlx4_flow_items *token = NULL;
 609                 unsigned int i;
 610                 int err;
 611
 612                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 613                         continue;
 614                 /*
 615                  * The nic can support patterns with NULL eth spec only
 616                  * if eth is a single item in a rule.
 617                  */
 618                 if (!items->spec &&
 619                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 620                         const struct rte_flow_item *next = items + 1;
 621
 622                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 623                                 rte_flow_error_set(error, ENOTSUP,
 624                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 625                                                    items,
 626                                                    "the rule requires"
 627                                                    " an Ethernet spec");
 628                                 return -rte_errno;
 629                         }
 630                 }
 631                 for (i = 0;
 632                      cur_item->items &&
 633                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 634                      ++i) {
 635                         if (cur_item->items[i] == items->type) {
 636                                 token = &mlx4_flow_items[items->type];
 637                                 break;
 638                         }
 639                 }
 640                 if (!token)
 641                         goto exit_item_not_supported;
 642                 cur_item = token;
 643                 err = cur_item->validate(items,
 644                                         (const uint8_t *)cur_item->mask,
 645                                          cur_item->mask_sz);
 646                 if (err)
 647                         goto exit_item_not_supported;
 648                 if (flow->ibv_attr && cur_item->convert) {
 649                         err = cur_item->convert(items,
 650                                                 (cur_item->default_mask ?
 651                                                  cur_item->default_mask :
 652                                                  cur_item->mask),
 653                                                  flow);
 654                         if (err)
 655                                 goto exit_item_not_supported;
 656                 }
 657                 flow->offset += cur_item->dst_sz;
 658         }
 659         /* Go over actions list */
 660         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 661                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 662                         continue;
 663                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 664                         action.drop = 1;
 665                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 666                         const struct rte_flow_action_queue *queue =
 667                                 (const struct rte_flow_action_queue *)
 668                                 actions->conf;
 669
 670                         if (!queue || (queue->index > (priv->rxqs_n - 1)))
 671                                 goto exit_action_not_supported;
 672                         action.queue = 1;
 673                 } else {
 674                         goto exit_action_not_supported;
 675                 }
 676         }
 677         if (!action.queue && !action.drop) {
 678                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 679                                    NULL, "no valid action");
 680                 return -rte_errno;
 681         }
 682         return 0;
 683 exit_item_not_supported:
 684         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 685                            items, "item not supported");
 686         return -rte_errno;
 687 exit_action_not_supported:
 688         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 689                            actions, "action not supported");
 690         return -rte_errno;
 691 }
 692
 693 /**
 694  * Validate a flow supported by the NIC.
 695  *
 696  * @see rte_flow_validate()
 697  * @see rte_flow_ops
 698  */
 699 int
 700 mlx4_flow_validate(struct rte_eth_dev *dev,
 701                    const struct rte_flow_attr *attr,
 702                    const struct rte_flow_item items[],
 703                    const struct rte_flow_action actions[],
 704                    struct rte_flow_error *error)
 705 {
 706         struct priv *priv = dev->data->dev_private;
 707         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 708
 709         return priv_flow_validate(priv, attr, items, actions, error, &flow);
 710 }
 711
 712 /**
 713  * Destroy a drop queue.
 714  *
 715  * @param priv
 716  *   Pointer to private structure.
 717  */
 718 static void
 719 mlx4_flow_destroy_drop_queue(struct priv *priv)
 720 {
 721         if (priv->flow_drop_queue) {
 722                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 723
 724                 priv->flow_drop_queue = NULL;
 725                 claim_zero(ibv_destroy_qp(fdq->qp));
 726                 claim_zero(ibv_destroy_cq(fdq->cq));
 727                 rte_free(fdq);
 728         }
 729 }
 730
 731 /**
 732  * Create a single drop queue for all drop flows.
 733  *
 734  * @param priv
 735  *   Pointer to private structure.
 736  *
 737  * @return
 738  *   0 on success, negative value otherwise.
 739  */
 740 static int
 741 mlx4_flow_create_drop_queue(struct priv *priv)
 742 {
 743         struct ibv_qp *qp;
 744         struct ibv_cq *cq;
 745         struct rte_flow_drop *fdq;
 746
 747         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 748         if (!fdq) {
 749                 ERROR("Cannot allocate memory for drop struct");
 750                 goto err;
 751         }
 752         cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 753         if (!cq) {
 754                 ERROR("Cannot create drop CQ");
 755                 goto err_create_cq;
 756         }
 757         qp = ibv_create_qp(priv->pd,
 758                            &(struct ibv_qp_init_attr){
 759                                 .send_cq = cq,
 760                                 .recv_cq = cq,
 761                                 .cap = {
 762                                         .max_recv_wr = 1,
 763                                         .max_recv_sge = 1,
 764                                 },
 765                                 .qp_type = IBV_QPT_RAW_PACKET,
 766                            });
 767         if (!qp) {
 768                 ERROR("Cannot create drop QP");
 769                 goto err_create_qp;
 770         }
 771         *fdq = (struct rte_flow_drop){
 772                 .qp = qp,
 773                 .cq = cq,
 774         };
 775         priv->flow_drop_queue = fdq;
 776         return 0;
 777 err_create_qp:
 778         claim_zero(ibv_destroy_cq(cq));
 779 err_create_cq:
 780         rte_free(fdq);
 781 err:
 782         return -1;
 783 }
 784
 785 /**
 786  * Complete flow rule creation.
 787  *
 788  * @param priv
 789  *   Pointer to private structure.
 790  * @param ibv_attr
 791  *   Verbs flow attributes.
 792  * @param action
 793  *   Target action structure.
 794  * @param[out] error
 795  *   Perform verbose error reporting if not NULL.
 796  *
 797  * @return
 798  *   A flow if the rule could be created.
 799  */
 800 static struct rte_flow *
 801 priv_flow_create_action_queue(struct priv *priv,
 802                               struct ibv_flow_attr *ibv_attr,
 803                               struct mlx4_flow_action *action,
 804                               struct rte_flow_error *error)
 805 {
 806         struct ibv_qp *qp;
 807         struct rte_flow *rte_flow;
 808
 809         assert(priv->pd);
 810         assert(priv->ctx);
 811         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 812         if (!rte_flow) {
 813                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 814                                    NULL, "cannot allocate flow memory");
 815                 return NULL;
 816         }
 817         if (action->drop) {
 818                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 819         } else {
 820                 struct rxq *rxq = (*priv->rxqs)[action->queue_id];
 821
 822                 qp = rxq->qp;
 823                 rte_flow->qp = qp;
 824         }
 825         rte_flow->ibv_attr = ibv_attr;
 826         if (!priv->started)
 827                 return rte_flow;
 828         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
 829         if (!rte_flow->ibv_flow) {
 830                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 831                                    NULL, "flow rule creation failure");
 832                 goto error;
 833         }
 834         return rte_flow;
 835 error:
 836         rte_free(rte_flow);
 837         return NULL;
 838 }
 839
 840 /**
 841  * Create a flow.
 842  *
 843  * @see rte_flow_create()
 844  * @see rte_flow_ops
 845  */
 846 struct rte_flow *
 847 mlx4_flow_create(struct rte_eth_dev *dev,
 848                  const struct rte_flow_attr *attr,
 849                  const struct rte_flow_item items[],
 850                  const struct rte_flow_action actions[],
 851                  struct rte_flow_error *error)
 852 {
 853         struct priv *priv = dev->data->dev_private;
 854         struct rte_flow *rte_flow;
 855         struct mlx4_flow_action action;
 856         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
 857         int err;
 858
 859         err = priv_flow_validate(priv, attr, items, actions, error, &flow);
 860         if (err)
 861                 return NULL;
 862         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
 863         if (!flow.ibv_attr) {
 864                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 865                                    NULL, "cannot allocate ibv_attr memory");
 866                 return NULL;
 867         }
 868         flow.offset = sizeof(struct ibv_flow_attr);
 869         *flow.ibv_attr = (struct ibv_flow_attr){
 870                 .comp_mask = 0,
 871                 .type = IBV_FLOW_ATTR_NORMAL,
 872                 .size = sizeof(struct ibv_flow_attr),
 873                 .priority = attr->priority,
 874                 .num_of_specs = 0,
 875                 .port = priv->port,
 876                 .flags = 0,
 877         };
 878         claim_zero(priv_flow_validate(priv, attr, items, actions,
 879                                       error, &flow));
 880         action = (struct mlx4_flow_action){
 881                 .queue = 0,
 882                 .drop = 0,
 883         };
 884         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 885                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 886                         continue;
 887                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 888                         action.queue = 1;
 889                         action.queue_id =
 890                                 ((const struct rte_flow_action_queue *)
 891                                  actions->conf)->index;
 892                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 893                         action.drop = 1;
 894                 } else {
 895                         rte_flow_error_set(error, ENOTSUP,
 896                                            RTE_FLOW_ERROR_TYPE_ACTION,
 897                                            actions, "unsupported action");
 898                         goto exit;
 899                 }
 900         }
 901         rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
 902                                                  &action, error);
 903         if (rte_flow) {
 904                 LIST_INSERT_HEAD(&priv->flows, rte_flow, next);
 905                 DEBUG("Flow created %p", (void *)rte_flow);
 906                 return rte_flow;
 907         }
 908 exit:
 909         rte_free(flow.ibv_attr);
 910         return NULL;
 911 }
 912
 913 /**
 914  * @see rte_flow_isolate()
 915  *
 916  * Must be done before calling dev_configure().
 917  *
 918  * @param dev
 919  *   Pointer to the ethernet device structure.
 920  * @param enable
 921  *   Nonzero to enter isolated mode, attempt to leave it otherwise.
 922  * @param[out] error
 923  *   Perform verbose error reporting if not NULL. PMDs initialize this
 924  *   structure in case of error only.
 925  *
 926  * @return
 927  *   0 on success, a negative value on error.
 928  */
 929 int
 930 mlx4_flow_isolate(struct rte_eth_dev *dev,
 931                   int enable,
 932                   struct rte_flow_error *error)
 933 {
 934         struct priv *priv = dev->data->dev_private;
 935
 936         if (priv->rxqs) {
 937                 rte_flow_error_set(error, ENOTSUP,
 938                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 939                                    NULL, "isolated mode must be set"
 940                                    " before configuring the device");
 941                 return -rte_errno;
 942         }
 943         priv->isolated = !!enable;
 944         return 0;
 945 }
 946
 947 /**
 948  * Destroy a flow.
 949  *
 950  * @see rte_flow_destroy()
 951  * @see rte_flow_ops
 952  */
 953 int
 954 mlx4_flow_destroy(struct rte_eth_dev *dev,
 955                   struct rte_flow *flow,
 956                   struct rte_flow_error *error)
 957 {
 958         (void)dev;
 959         (void)error;
 960         LIST_REMOVE(flow, next);
 961         if (flow->ibv_flow)
 962                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 963         rte_free(flow->ibv_attr);
 964         DEBUG("Flow destroyed %p", (void *)flow);
 965         rte_free(flow);
 966         return 0;
 967 }
 968
 969 /**
 970  * Destroy all flows.
 971  *
 972  * @see rte_flow_flush()
 973  * @see rte_flow_ops
 974  */
 975 int
 976 mlx4_flow_flush(struct rte_eth_dev *dev,
 977                 struct rte_flow_error *error)
 978 {
 979         struct priv *priv = dev->data->dev_private;
 980
 981         while (!LIST_EMPTY(&priv->flows)) {
 982                 struct rte_flow *flow;
 983
 984                 flow = LIST_FIRST(&priv->flows);
 985                 mlx4_flow_destroy(dev, flow, error);
 986         }
 987         return 0;
 988 }
 989
 990 /**
 991  * Remove all flows.
 992  *
 993  * Called by dev_stop() to remove all flows.
 994  *
 995  * @param priv
 996  *   Pointer to private structure.
 997  */
 998 void
 999 mlx4_priv_flow_stop(struct priv *priv)
1000 {
1001         struct rte_flow *flow;
1002
1003         for (flow = LIST_FIRST(&priv->flows);
1004              flow;
1005              flow = LIST_NEXT(flow, next)) {
1006                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1007                 flow->ibv_flow = NULL;
1008                 DEBUG("Flow %p removed", (void *)flow);
1009         }
1010         mlx4_flow_destroy_drop_queue(priv);
1011 }
1012
1013 /**
1014  * Add all flows.
1015  *
1016  * @param priv
1017  *   Pointer to private structure.
1018  *
1019  * @return
1020  *   0 on success, a errno value otherwise and rte_errno is set.
1021  */
1022 int
1023 mlx4_priv_flow_start(struct priv *priv)
1024 {
1025         int ret;
1026         struct ibv_qp *qp;
1027         struct rte_flow *flow;
1028
1029         ret = mlx4_flow_create_drop_queue(priv);
1030         if (ret)
1031                 return -1;
1032         for (flow = LIST_FIRST(&priv->flows);
1033              flow;
1034              flow = LIST_NEXT(flow, next)) {
1035                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1036                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1037                 if (!flow->ibv_flow) {
1038                         DEBUG("Flow %p cannot be applied", (void *)flow);
1039                         rte_errno = EINVAL;
1040                         return rte_errno;
1041                 }
1042                 DEBUG("Flow %p applied", (void *)flow);
1043         }
1044         return 0;
1045 }