drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <assert.h>
  35
  36 #include <rte_flow.h>
  37 #include <rte_flow_driver.h>
  38 #include <rte_malloc.h>
  39
  40 /* PMD headers. */
  41 #include "mlx4.h"
  42 #include "mlx4_flow.h"
  43
  44 /** Static initializer for items. */
  45 #define ITEMS(...) \
  46         (const enum rte_flow_item_type []){ \
  47                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  48         }
  49
  50 /** Structure to generate a simple graph of layers supported by the NIC. */
  51 struct mlx4_flow_items {
  52         /** List of possible actions for these items. */
  53         const enum rte_flow_action_type *const actions;
  54         /** Bit-masks corresponding to the possibilities for the item. */
  55         const void *mask;
  56         /**
  57          * Default bit-masks to use when item->mask is not provided. When
  58          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  59          * used instead.
  60          */
  61         const void *default_mask;
  62         /** Bit-masks size in bytes. */
  63         const unsigned int mask_sz;
  64         /**
  65          * Check support for a given item.
  66          *
  67          * @param item[in]
  68          *   Item specification.
  69          * @param mask[in]
  70          *   Bit-masks covering supported fields to compare with spec,
  71          *   last and mask in
  72          *   \item.
  73          * @param size
  74          *   Bit-Mask size in bytes.
  75          *
  76          * @return
  77          *   0 on success, negative value otherwise.
  78          */
  79         int (*validate)(const struct rte_flow_item *item,
  80                         const uint8_t *mask, unsigned int size);
  81         /**
  82          * Conversion function from rte_flow to NIC specific flow.
  83          *
  84          * @param item
  85          *   rte_flow item to convert.
  86          * @param default_mask
  87          *   Default bit-masks to use when item->mask is not provided.
  88          * @param data
  89          *   Internal structure to store the conversion.
  90          *
  91          * @return
  92          *   0 on success, negative value otherwise.
  93          */
  94         int (*convert)(const struct rte_flow_item *item,
  95                        const void *default_mask,
  96                        void *data);
  97         /** Size in bytes of the destination structure. */
  98         const unsigned int dst_sz;
  99         /** List of possible following items.  */
 100         const enum rte_flow_item_type *const items;
 101 };
 102
 103 struct rte_flow_drop {
 104         struct ibv_qp *qp; /**< Verbs queue pair. */
 105         struct ibv_cq *cq; /**< Verbs completion queue. */
 106 };
 107
 108 /** Valid action for this PMD. */
 109 static const enum rte_flow_action_type valid_actions[] = {
 110         RTE_FLOW_ACTION_TYPE_DROP,
 111         RTE_FLOW_ACTION_TYPE_QUEUE,
 112         RTE_FLOW_ACTION_TYPE_END,
 113 };
 114
 115 /**
 116  * Convert Ethernet item to Verbs specification.
 117  *
 118  * @param item[in]
 119  *   Item specification.
 120  * @param default_mask[in]
 121  *   Default bit-masks to use when item->mask is not provided.
 122  * @param data[in, out]
 123  *   User structure.
 124  */
 125 static int
 126 mlx4_flow_create_eth(const struct rte_flow_item *item,
 127                      const void *default_mask,
 128                      void *data)
 129 {
 130         const struct rte_flow_item_eth *spec = item->spec;
 131         const struct rte_flow_item_eth *mask = item->mask;
 132         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 133         struct ibv_flow_spec_eth *eth;
 134         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 135         unsigned int i;
 136
 137         ++flow->ibv_attr->num_of_specs;
 138         flow->ibv_attr->priority = 2;
 139         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 140         *eth = (struct ibv_flow_spec_eth) {
 141                 .type = IBV_FLOW_SPEC_ETH,
 142                 .size = eth_size,
 143         };
 144         if (!spec) {
 145                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 146                 return 0;
 147         }
 148         if (!mask)
 149                 mask = default_mask;
 150         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 151         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 152         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 153         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 154         /* Remove unwanted bits from values. */
 155         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 156                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 157                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 158         }
 159         return 0;
 160 }
 161
 162 /**
 163  * Convert VLAN item to Verbs specification.
 164  *
 165  * @param item[in]
 166  *   Item specification.
 167  * @param default_mask[in]
 168  *   Default bit-masks to use when item->mask is not provided.
 169  * @param data[in, out]
 170  *   User structure.
 171  */
 172 static int
 173 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 174                       const void *default_mask,
 175                       void *data)
 176 {
 177         const struct rte_flow_item_vlan *spec = item->spec;
 178         const struct rte_flow_item_vlan *mask = item->mask;
 179         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 180         struct ibv_flow_spec_eth *eth;
 181         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 182
 183         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 184         if (!spec)
 185                 return 0;
 186         if (!mask)
 187                 mask = default_mask;
 188         eth->val.vlan_tag = spec->tci;
 189         eth->mask.vlan_tag = mask->tci;
 190         eth->val.vlan_tag &= eth->mask.vlan_tag;
 191         return 0;
 192 }
 193
 194 /**
 195  * Convert IPv4 item to Verbs specification.
 196  *
 197  * @param item[in]
 198  *   Item specification.
 199  * @param default_mask[in]
 200  *   Default bit-masks to use when item->mask is not provided.
 201  * @param data[in, out]
 202  *   User structure.
 203  */
 204 static int
 205 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 206                       const void *default_mask,
 207                       void *data)
 208 {
 209         const struct rte_flow_item_ipv4 *spec = item->spec;
 210         const struct rte_flow_item_ipv4 *mask = item->mask;
 211         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 212         struct ibv_flow_spec_ipv4 *ipv4;
 213         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 214
 215         ++flow->ibv_attr->num_of_specs;
 216         flow->ibv_attr->priority = 1;
 217         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 218         *ipv4 = (struct ibv_flow_spec_ipv4) {
 219                 .type = IBV_FLOW_SPEC_IPV4,
 220                 .size = ipv4_size,
 221         };
 222         if (!spec)
 223                 return 0;
 224         ipv4->val = (struct ibv_flow_ipv4_filter) {
 225                 .src_ip = spec->hdr.src_addr,
 226                 .dst_ip = spec->hdr.dst_addr,
 227         };
 228         if (!mask)
 229                 mask = default_mask;
 230         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 231                 .src_ip = mask->hdr.src_addr,
 232                 .dst_ip = mask->hdr.dst_addr,
 233         };
 234         /* Remove unwanted bits from values. */
 235         ipv4->val.src_ip &= ipv4->mask.src_ip;
 236         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 237         return 0;
 238 }
 239
 240 /**
 241  * Convert UDP item to Verbs specification.
 242  *
 243  * @param item[in]
 244  *   Item specification.
 245  * @param default_mask[in]
 246  *   Default bit-masks to use when item->mask is not provided.
 247  * @param data[in, out]
 248  *   User structure.
 249  */
 250 static int
 251 mlx4_flow_create_udp(const struct rte_flow_item *item,
 252                      const void *default_mask,
 253                      void *data)
 254 {
 255         const struct rte_flow_item_udp *spec = item->spec;
 256         const struct rte_flow_item_udp *mask = item->mask;
 257         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 258         struct ibv_flow_spec_tcp_udp *udp;
 259         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 260
 261         ++flow->ibv_attr->num_of_specs;
 262         flow->ibv_attr->priority = 0;
 263         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 264         *udp = (struct ibv_flow_spec_tcp_udp) {
 265                 .type = IBV_FLOW_SPEC_UDP,
 266                 .size = udp_size,
 267         };
 268         if (!spec)
 269                 return 0;
 270         udp->val.dst_port = spec->hdr.dst_port;
 271         udp->val.src_port = spec->hdr.src_port;
 272         if (!mask)
 273                 mask = default_mask;
 274         udp->mask.dst_port = mask->hdr.dst_port;
 275         udp->mask.src_port = mask->hdr.src_port;
 276         /* Remove unwanted bits from values. */
 277         udp->val.src_port &= udp->mask.src_port;
 278         udp->val.dst_port &= udp->mask.dst_port;
 279         return 0;
 280 }
 281
 282 /**
 283  * Convert TCP item to Verbs specification.
 284  *
 285  * @param item[in]
 286  *   Item specification.
 287  * @param default_mask[in]
 288  *   Default bit-masks to use when item->mask is not provided.
 289  * @param data[in, out]
 290  *   User structure.
 291  */
 292 static int
 293 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 294                      const void *default_mask,
 295                      void *data)
 296 {
 297         const struct rte_flow_item_tcp *spec = item->spec;
 298         const struct rte_flow_item_tcp *mask = item->mask;
 299         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 300         struct ibv_flow_spec_tcp_udp *tcp;
 301         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 302
 303         ++flow->ibv_attr->num_of_specs;
 304         flow->ibv_attr->priority = 0;
 305         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 306         *tcp = (struct ibv_flow_spec_tcp_udp) {
 307                 .type = IBV_FLOW_SPEC_TCP,
 308                 .size = tcp_size,
 309         };
 310         if (!spec)
 311                 return 0;
 312         tcp->val.dst_port = spec->hdr.dst_port;
 313         tcp->val.src_port = spec->hdr.src_port;
 314         if (!mask)
 315                 mask = default_mask;
 316         tcp->mask.dst_port = mask->hdr.dst_port;
 317         tcp->mask.src_port = mask->hdr.src_port;
 318         /* Remove unwanted bits from values. */
 319         tcp->val.src_port &= tcp->mask.src_port;
 320         tcp->val.dst_port &= tcp->mask.dst_port;
 321         return 0;
 322 }
 323
 324 /**
 325  * Check support for a given item.
 326  *
 327  * @param item[in]
 328  *   Item specification.
 329  * @param mask[in]
 330  *   Bit-masks covering supported fields to compare with spec, last and mask in
 331  *   \item.
 332  * @param size
 333  *   Bit-Mask size in bytes.
 334  *
 335  * @return
 336  *   0 on success, negative value otherwise.
 337  */
 338 static int
 339 mlx4_flow_item_validate(const struct rte_flow_item *item,
 340                         const uint8_t *mask, unsigned int size)
 341 {
 342         int ret = 0;
 343
 344         if (!item->spec && (item->mask || item->last))
 345                 return -1;
 346         if (item->spec && !item->mask) {
 347                 unsigned int i;
 348                 const uint8_t *spec = item->spec;
 349
 350                 for (i = 0; i < size; ++i)
 351                         if ((spec[i] | mask[i]) != mask[i])
 352                                 return -1;
 353         }
 354         if (item->last && !item->mask) {
 355                 unsigned int i;
 356                 const uint8_t *spec = item->last;
 357
 358                 for (i = 0; i < size; ++i)
 359                         if ((spec[i] | mask[i]) != mask[i])
 360                                 return -1;
 361         }
 362         if (item->spec && item->last) {
 363                 uint8_t spec[size];
 364                 uint8_t last[size];
 365                 const uint8_t *apply = mask;
 366                 unsigned int i;
 367
 368                 if (item->mask)
 369                         apply = item->mask;
 370                 for (i = 0; i < size; ++i) {
 371                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 372                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 373                 }
 374                 ret = memcmp(spec, last, size);
 375         }
 376         return ret;
 377 }
 378
 379 static int
 380 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 381                        const uint8_t *mask, unsigned int size)
 382 {
 383         if (item->mask) {
 384                 const struct rte_flow_item_eth *mask = item->mask;
 385
 386                 if (mask->dst.addr_bytes[0] != 0xff ||
 387                                 mask->dst.addr_bytes[1] != 0xff ||
 388                                 mask->dst.addr_bytes[2] != 0xff ||
 389                                 mask->dst.addr_bytes[3] != 0xff ||
 390                                 mask->dst.addr_bytes[4] != 0xff ||
 391                                 mask->dst.addr_bytes[5] != 0xff)
 392                         return -1;
 393         }
 394         return mlx4_flow_item_validate(item, mask, size);
 395 }
 396
 397 static int
 398 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 399                         const uint8_t *mask, unsigned int size)
 400 {
 401         if (item->mask) {
 402                 const struct rte_flow_item_vlan *mask = item->mask;
 403
 404                 if (mask->tci != 0 &&
 405                     ntohs(mask->tci) != 0x0fff)
 406                         return -1;
 407         }
 408         return mlx4_flow_item_validate(item, mask, size);
 409 }
 410
 411 static int
 412 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 413                         const uint8_t *mask, unsigned int size)
 414 {
 415         if (item->mask) {
 416                 const struct rte_flow_item_ipv4 *mask = item->mask;
 417
 418                 if (mask->hdr.src_addr != 0 &&
 419                     mask->hdr.src_addr != 0xffffffff)
 420                         return -1;
 421                 if (mask->hdr.dst_addr != 0 &&
 422                     mask->hdr.dst_addr != 0xffffffff)
 423                         return -1;
 424         }
 425         return mlx4_flow_item_validate(item, mask, size);
 426 }
 427
 428 static int
 429 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 430                        const uint8_t *mask, unsigned int size)
 431 {
 432         if (item->mask) {
 433                 const struct rte_flow_item_udp *mask = item->mask;
 434
 435                 if (mask->hdr.src_port != 0 &&
 436                     mask->hdr.src_port != 0xffff)
 437                         return -1;
 438                 if (mask->hdr.dst_port != 0 &&
 439                     mask->hdr.dst_port != 0xffff)
 440                         return -1;
 441         }
 442         return mlx4_flow_item_validate(item, mask, size);
 443 }
 444
 445 static int
 446 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 447                        const uint8_t *mask, unsigned int size)
 448 {
 449         if (item->mask) {
 450                 const struct rte_flow_item_tcp *mask = item->mask;
 451
 452                 if (mask->hdr.src_port != 0 &&
 453                     mask->hdr.src_port != 0xffff)
 454                         return -1;
 455                 if (mask->hdr.dst_port != 0 &&
 456                     mask->hdr.dst_port != 0xffff)
 457                         return -1;
 458         }
 459         return mlx4_flow_item_validate(item, mask, size);
 460 }
 461
 462 /** Graph of supported items and associated actions. */
 463 static const struct mlx4_flow_items mlx4_flow_items[] = {
 464         [RTE_FLOW_ITEM_TYPE_END] = {
 465                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 466         },
 467         [RTE_FLOW_ITEM_TYPE_ETH] = {
 468                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 469                                RTE_FLOW_ITEM_TYPE_IPV4),
 470                 .actions = valid_actions,
 471                 .mask = &(const struct rte_flow_item_eth){
 472                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 473                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 474                 },
 475                 .default_mask = &rte_flow_item_eth_mask,
 476                 .mask_sz = sizeof(struct rte_flow_item_eth),
 477                 .validate = mlx4_flow_validate_eth,
 478                 .convert = mlx4_flow_create_eth,
 479                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 480         },
 481         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 482                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 483                 .actions = valid_actions,
 484                 .mask = &(const struct rte_flow_item_vlan){
 485                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 486 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 487                         .tci = 0x0fff,
 488 #else
 489                         .tci = 0xff0f,
 490 #endif
 491                 },
 492                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 493                 .validate = mlx4_flow_validate_vlan,
 494                 .convert = mlx4_flow_create_vlan,
 495                 .dst_sz = 0,
 496         },
 497         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 498                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 499                                RTE_FLOW_ITEM_TYPE_TCP),
 500                 .actions = valid_actions,
 501                 .mask = &(const struct rte_flow_item_ipv4){
 502                         .hdr = {
 503                                 .src_addr = -1,
 504                                 .dst_addr = -1,
 505                         },
 506                 },
 507                 .default_mask = &rte_flow_item_ipv4_mask,
 508                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 509                 .validate = mlx4_flow_validate_ipv4,
 510                 .convert = mlx4_flow_create_ipv4,
 511                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 512         },
 513         [RTE_FLOW_ITEM_TYPE_UDP] = {
 514                 .actions = valid_actions,
 515                 .mask = &(const struct rte_flow_item_udp){
 516                         .hdr = {
 517                                 .src_port = -1,
 518                                 .dst_port = -1,
 519                         },
 520                 },
 521                 .default_mask = &rte_flow_item_udp_mask,
 522                 .mask_sz = sizeof(struct rte_flow_item_udp),
 523                 .validate = mlx4_flow_validate_udp,
 524                 .convert = mlx4_flow_create_udp,
 525                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 526         },
 527         [RTE_FLOW_ITEM_TYPE_TCP] = {
 528                 .actions = valid_actions,
 529                 .mask = &(const struct rte_flow_item_tcp){
 530                         .hdr = {
 531                                 .src_port = -1,
 532                                 .dst_port = -1,
 533                         },
 534                 },
 535                 .default_mask = &rte_flow_item_tcp_mask,
 536                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 537                 .validate = mlx4_flow_validate_tcp,
 538                 .convert = mlx4_flow_create_tcp,
 539                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 540         },
 541 };
 542
 543 /**
 544  * Validate a flow supported by the NIC.
 545  *
 546  * @param priv
 547  *   Pointer to private structure.
 548  * @param[in] attr
 549  *   Flow rule attributes.
 550  * @param[in] items
 551  *   Pattern specification (list terminated by the END pattern item).
 552  * @param[in] actions
 553  *   Associated actions (list terminated by the END action).
 554  * @param[out] error
 555  *   Perform verbose error reporting if not NULL.
 556  * @param[in, out] flow
 557  *   Flow structure to update.
 558  *
 559  * @return
 560  *   0 on success, a negative errno value otherwise and rte_errno is set.
 561  */
 562 static int
 563 priv_flow_validate(struct priv *priv,
 564                    const struct rte_flow_attr *attr,
 565                    const struct rte_flow_item items[],
 566                    const struct rte_flow_action actions[],
 567                    struct rte_flow_error *error,
 568                    struct mlx4_flow *flow)
 569 {
 570         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 571         struct mlx4_flow_action action = {
 572                 .queue = 0,
 573                 .drop = 0,
 574         };
 575
 576         (void)priv;
 577         if (attr->group) {
 578                 rte_flow_error_set(error, ENOTSUP,
 579                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 580                                    NULL,
 581                                    "groups are not supported");
 582                 return -rte_errno;
 583         }
 584         if (attr->priority) {
 585                 rte_flow_error_set(error, ENOTSUP,
 586                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 587                                    NULL,
 588                                    "priorities are not supported");
 589                 return -rte_errno;
 590         }
 591         if (attr->egress) {
 592                 rte_flow_error_set(error, ENOTSUP,
 593                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 594                                    NULL,
 595                                    "egress is not supported");
 596                 return -rte_errno;
 597         }
 598         if (!attr->ingress) {
 599                 rte_flow_error_set(error, ENOTSUP,
 600                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 601                                    NULL,
 602                                    "only ingress is supported");
 603                 return -rte_errno;
 604         }
 605         /* Go over items list. */
 606         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 607                 const struct mlx4_flow_items *token = NULL;
 608                 unsigned int i;
 609                 int err;
 610
 611                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 612                         continue;
 613                 /*
 614                  * The nic can support patterns with NULL eth spec only
 615                  * if eth is a single item in a rule.
 616                  */
 617                 if (!items->spec &&
 618                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 619                         const struct rte_flow_item *next = items + 1;
 620
 621                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 622                                 rte_flow_error_set(error, ENOTSUP,
 623                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 624                                                    items,
 625                                                    "the rule requires"
 626                                                    " an Ethernet spec");
 627                                 return -rte_errno;
 628                         }
 629                 }
 630                 for (i = 0;
 631                      cur_item->items &&
 632                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 633                      ++i) {
 634                         if (cur_item->items[i] == items->type) {
 635                                 token = &mlx4_flow_items[items->type];
 636                                 break;
 637                         }
 638                 }
 639                 if (!token)
 640                         goto exit_item_not_supported;
 641                 cur_item = token;
 642                 err = cur_item->validate(items,
 643                                         (const uint8_t *)cur_item->mask,
 644                                          cur_item->mask_sz);
 645                 if (err)
 646                         goto exit_item_not_supported;
 647                 if (flow->ibv_attr && cur_item->convert) {
 648                         err = cur_item->convert(items,
 649                                                 (cur_item->default_mask ?
 650                                                  cur_item->default_mask :
 651                                                  cur_item->mask),
 652                                                  flow);
 653                         if (err)
 654                                 goto exit_item_not_supported;
 655                 }
 656                 flow->offset += cur_item->dst_sz;
 657         }
 658         /* Go over actions list */
 659         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 660                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 661                         continue;
 662                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 663                         action.drop = 1;
 664                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 665                         const struct rte_flow_action_queue *queue =
 666                                 (const struct rte_flow_action_queue *)
 667                                 actions->conf;
 668
 669                         if (!queue || (queue->index > (priv->rxqs_n - 1)))
 670                                 goto exit_action_not_supported;
 671                         action.queue = 1;
 672                 } else {
 673                         goto exit_action_not_supported;
 674                 }
 675         }
 676         if (!action.queue && !action.drop) {
 677                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 678                                    NULL, "no valid action");
 679                 return -rte_errno;
 680         }
 681         return 0;
 682 exit_item_not_supported:
 683         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 684                            items, "item not supported");
 685         return -rte_errno;
 686 exit_action_not_supported:
 687         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 688                            actions, "action not supported");
 689         return -rte_errno;
 690 }
 691
 692 /**
 693  * Validate a flow supported by the NIC.
 694  *
 695  * @see rte_flow_validate()
 696  * @see rte_flow_ops
 697  */
 698 int
 699 mlx4_flow_validate(struct rte_eth_dev *dev,
 700                    const struct rte_flow_attr *attr,
 701                    const struct rte_flow_item items[],
 702                    const struct rte_flow_action actions[],
 703                    struct rte_flow_error *error)
 704 {
 705         struct priv *priv = dev->data->dev_private;
 706         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 707
 708         return priv_flow_validate(priv, attr, items, actions, error, &flow);
 709 }
 710
 711 /**
 712  * Destroy a drop queue.
 713  *
 714  * @param priv
 715  *   Pointer to private structure.
 716  */
 717 static void
 718 mlx4_flow_destroy_drop_queue(struct priv *priv)
 719 {
 720         if (priv->flow_drop_queue) {
 721                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 722
 723                 priv->flow_drop_queue = NULL;
 724                 claim_zero(ibv_destroy_qp(fdq->qp));
 725                 claim_zero(ibv_destroy_cq(fdq->cq));
 726                 rte_free(fdq);
 727         }
 728 }
 729
 730 /**
 731  * Create a single drop queue for all drop flows.
 732  *
 733  * @param priv
 734  *   Pointer to private structure.
 735  *
 736  * @return
 737  *   0 on success, negative value otherwise.
 738  */
 739 static int
 740 mlx4_flow_create_drop_queue(struct priv *priv)
 741 {
 742         struct ibv_qp *qp;
 743         struct ibv_cq *cq;
 744         struct rte_flow_drop *fdq;
 745
 746         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 747         if (!fdq) {
 748                 ERROR("Cannot allocate memory for drop struct");
 749                 goto err;
 750         }
 751         cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 752         if (!cq) {
 753                 ERROR("Cannot create drop CQ");
 754                 goto err_create_cq;
 755         }
 756         qp = ibv_create_qp(priv->pd,
 757                            &(struct ibv_qp_init_attr){
 758                                 .send_cq = cq,
 759                                 .recv_cq = cq,
 760                                 .cap = {
 761                                         .max_recv_wr = 1,
 762                                         .max_recv_sge = 1,
 763                                 },
 764                                 .qp_type = IBV_QPT_RAW_PACKET,
 765                            });
 766         if (!qp) {
 767                 ERROR("Cannot create drop QP");
 768                 goto err_create_qp;
 769         }
 770         *fdq = (struct rte_flow_drop){
 771                 .qp = qp,
 772                 .cq = cq,
 773         };
 774         priv->flow_drop_queue = fdq;
 775         return 0;
 776 err_create_qp:
 777         claim_zero(ibv_destroy_cq(cq));
 778 err_create_cq:
 779         rte_free(fdq);
 780 err:
 781         return -1;
 782 }
 783
 784 /**
 785  * Complete flow rule creation.
 786  *
 787  * @param priv
 788  *   Pointer to private structure.
 789  * @param ibv_attr
 790  *   Verbs flow attributes.
 791  * @param action
 792  *   Target action structure.
 793  * @param[out] error
 794  *   Perform verbose error reporting if not NULL.
 795  *
 796  * @return
 797  *   A flow if the rule could be created.
 798  */
 799 static struct rte_flow *
 800 priv_flow_create_action_queue(struct priv *priv,
 801                               struct ibv_flow_attr *ibv_attr,
 802                               struct mlx4_flow_action *action,
 803                               struct rte_flow_error *error)
 804 {
 805         struct ibv_qp *qp;
 806         struct rte_flow *rte_flow;
 807
 808         assert(priv->pd);
 809         assert(priv->ctx);
 810         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 811         if (!rte_flow) {
 812                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 813                                    NULL, "cannot allocate flow memory");
 814                 return NULL;
 815         }
 816         if (action->drop) {
 817                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 818         } else {
 819                 struct rxq *rxq = (*priv->rxqs)[action->queue_id];
 820
 821                 qp = rxq->qp;
 822                 rte_flow->qp = qp;
 823         }
 824         rte_flow->ibv_attr = ibv_attr;
 825         if (!priv->started)
 826                 return rte_flow;
 827         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
 828         if (!rte_flow->ibv_flow) {
 829                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 830                                    NULL, "flow rule creation failure");
 831                 goto error;
 832         }
 833         return rte_flow;
 834 error:
 835         rte_free(rte_flow);
 836         return NULL;
 837 }
 838
 839 /**
 840  * Create a flow.
 841  *
 842  * @see rte_flow_create()
 843  * @see rte_flow_ops
 844  */
 845 struct rte_flow *
 846 mlx4_flow_create(struct rte_eth_dev *dev,
 847                  const struct rte_flow_attr *attr,
 848                  const struct rte_flow_item items[],
 849                  const struct rte_flow_action actions[],
 850                  struct rte_flow_error *error)
 851 {
 852         struct priv *priv = dev->data->dev_private;
 853         struct rte_flow *rte_flow;
 854         struct mlx4_flow_action action;
 855         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
 856         int err;
 857
 858         err = priv_flow_validate(priv, attr, items, actions, error, &flow);
 859         if (err)
 860                 return NULL;
 861         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
 862         if (!flow.ibv_attr) {
 863                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 864                                    NULL, "cannot allocate ibv_attr memory");
 865                 return NULL;
 866         }
 867         flow.offset = sizeof(struct ibv_flow_attr);
 868         *flow.ibv_attr = (struct ibv_flow_attr){
 869                 .comp_mask = 0,
 870                 .type = IBV_FLOW_ATTR_NORMAL,
 871                 .size = sizeof(struct ibv_flow_attr),
 872                 .priority = attr->priority,
 873                 .num_of_specs = 0,
 874                 .port = priv->port,
 875                 .flags = 0,
 876         };
 877         claim_zero(priv_flow_validate(priv, attr, items, actions,
 878                                       error, &flow));
 879         action = (struct mlx4_flow_action){
 880                 .queue = 0,
 881                 .drop = 0,
 882         };
 883         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 884                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 885                         continue;
 886                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 887                         action.queue = 1;
 888                         action.queue_id =
 889                                 ((const struct rte_flow_action_queue *)
 890                                  actions->conf)->index;
 891                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 892                         action.drop = 1;
 893                 } else {
 894                         rte_flow_error_set(error, ENOTSUP,
 895                                            RTE_FLOW_ERROR_TYPE_ACTION,
 896                                            actions, "unsupported action");
 897                         goto exit;
 898                 }
 899         }
 900         rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
 901                                                  &action, error);
 902         if (rte_flow) {
 903                 LIST_INSERT_HEAD(&priv->flows, rte_flow, next);
 904                 DEBUG("Flow created %p", (void *)rte_flow);
 905                 return rte_flow;
 906         }
 907 exit:
 908         rte_free(flow.ibv_attr);
 909         return NULL;
 910 }
 911
 912 /**
 913  * @see rte_flow_isolate()
 914  *
 915  * Must be done before calling dev_configure().
 916  *
 917  * @param dev
 918  *   Pointer to the ethernet device structure.
 919  * @param enable
 920  *   Nonzero to enter isolated mode, attempt to leave it otherwise.
 921  * @param[out] error
 922  *   Perform verbose error reporting if not NULL. PMDs initialize this
 923  *   structure in case of error only.
 924  *
 925  * @return
 926  *   0 on success, a negative value on error.
 927  */
 928 int
 929 mlx4_flow_isolate(struct rte_eth_dev *dev,
 930                   int enable,
 931                   struct rte_flow_error *error)
 932 {
 933         struct priv *priv = dev->data->dev_private;
 934
 935         if (priv->rxqs) {
 936                 rte_flow_error_set(error, ENOTSUP,
 937                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 938                                    NULL, "isolated mode must be set"
 939                                    " before configuring the device");
 940                 return -rte_errno;
 941         }
 942         priv->isolated = !!enable;
 943         return 0;
 944 }
 945
 946 /**
 947  * Destroy a flow.
 948  *
 949  * @see rte_flow_destroy()
 950  * @see rte_flow_ops
 951  */
 952 int
 953 mlx4_flow_destroy(struct rte_eth_dev *dev,
 954                   struct rte_flow *flow,
 955                   struct rte_flow_error *error)
 956 {
 957         (void)dev;
 958         (void)error;
 959         LIST_REMOVE(flow, next);
 960         if (flow->ibv_flow)
 961                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 962         rte_free(flow->ibv_attr);
 963         DEBUG("Flow destroyed %p", (void *)flow);
 964         rte_free(flow);
 965         return 0;
 966 }
 967
 968 /**
 969  * Destroy all flows.
 970  *
 971  * @see rte_flow_flush()
 972  * @see rte_flow_ops
 973  */
 974 int
 975 mlx4_flow_flush(struct rte_eth_dev *dev,
 976                 struct rte_flow_error *error)
 977 {
 978         struct priv *priv = dev->data->dev_private;
 979
 980         while (!LIST_EMPTY(&priv->flows)) {
 981                 struct rte_flow *flow;
 982
 983                 flow = LIST_FIRST(&priv->flows);
 984                 mlx4_flow_destroy(dev, flow, error);
 985         }
 986         return 0;
 987 }
 988
 989 /**
 990  * Remove all flows.
 991  *
 992  * Called by dev_stop() to remove all flows.
 993  *
 994  * @param priv
 995  *   Pointer to private structure.
 996  */
 997 void
 998 mlx4_priv_flow_stop(struct priv *priv)
 999 {
1000         struct rte_flow *flow;
1001
1002         for (flow = LIST_FIRST(&priv->flows);
1003              flow;
1004              flow = LIST_NEXT(flow, next)) {
1005                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1006                 flow->ibv_flow = NULL;
1007                 DEBUG("Flow %p removed", (void *)flow);
1008         }
1009         mlx4_flow_destroy_drop_queue(priv);
1010 }
1011
1012 /**
1013  * Add all flows.
1014  *
1015  * @param priv
1016  *   Pointer to private structure.
1017  *
1018  * @return
1019  *   0 on success, a errno value otherwise and rte_errno is set.
1020  */
1021 int
1022 mlx4_priv_flow_start(struct priv *priv)
1023 {
1024         int ret;
1025         struct ibv_qp *qp;
1026         struct rte_flow *flow;
1027
1028         ret = mlx4_flow_create_drop_queue(priv);
1029         if (ret)
1030                 return -1;
1031         for (flow = LIST_FIRST(&priv->flows);
1032              flow;
1033              flow = LIST_NEXT(flow, next)) {
1034                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1035                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1036                 if (!flow->ibv_flow) {
1037                         DEBUG("Flow %p cannot be applied", (void *)flow);
1038                         rte_errno = EINVAL;
1039                         return rte_errno;
1040                 }
1041                 DEBUG("Flow %p applied", (void *)flow);
1042         }
1043         return 0;
1044 }