drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <assert.h>
  35
  36 #include <rte_flow.h>
  37 #include <rte_flow_driver.h>
  38 #include <rte_malloc.h>
  39
  40 /* PMD headers. */
  41 #include "mlx4.h"
  42 #include "mlx4_flow.h"
  43
  44 /** Static initializer for items. */
  45 #define ITEMS(...) \
  46         (const enum rte_flow_item_type []){ \
  47                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  48         }
  49
  50 /** Structure to generate a simple graph of layers supported by the NIC. */
  51 struct mlx4_flow_items {
  52         /** List of possible actions for these items. */
  53         const enum rte_flow_action_type *const actions;
  54         /** Bit-masks corresponding to the possibilities for the item. */
  55         const void *mask;
  56         /**
  57          * Default bit-masks to use when item->mask is not provided. When
  58          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  59          * used instead.
  60          */
  61         const void *default_mask;
  62         /** Bit-masks size in bytes. */
  63         const unsigned int mask_sz;
  64         /**
  65          * Check support for a given item.
  66          *
  67          * @param item[in]
  68          *   Item specification.
  69          * @param mask[in]
  70          *   Bit-masks covering supported fields to compare with spec,
  71          *   last and mask in
  72          *   \item.
  73          * @param size
  74          *   Bit-Mask size in bytes.
  75          *
  76          * @return
  77          *   0 on success, negative value otherwise.
  78          */
  79         int (*validate)(const struct rte_flow_item *item,
  80                         const uint8_t *mask, unsigned int size);
  81         /**
  82          * Conversion function from rte_flow to NIC specific flow.
  83          *
  84          * @param item
  85          *   rte_flow item to convert.
  86          * @param default_mask
  87          *   Default bit-masks to use when item->mask is not provided.
  88          * @param data
  89          *   Internal structure to store the conversion.
  90          *
  91          * @return
  92          *   0 on success, negative value otherwise.
  93          */
  94         int (*convert)(const struct rte_flow_item *item,
  95                        const void *default_mask,
  96                        void *data);
  97         /** Size in bytes of the destination structure. */
  98         const unsigned int dst_sz;
  99         /** List of possible following items.  */
 100         const enum rte_flow_item_type *const items;
 101 };
 102
 103 struct rte_flow_drop {
 104         struct ibv_qp *qp; /**< Verbs queue pair. */
 105         struct ibv_cq *cq; /**< Verbs completion queue. */
 106 };
 107
 108 /** Valid action for this PMD. */
 109 static const enum rte_flow_action_type valid_actions[] = {
 110         RTE_FLOW_ACTION_TYPE_DROP,
 111         RTE_FLOW_ACTION_TYPE_QUEUE,
 112         RTE_FLOW_ACTION_TYPE_END,
 113 };
 114
 115 /**
 116  * Convert Ethernet item to Verbs specification.
 117  *
 118  * @param item[in]
 119  *   Item specification.
 120  * @param default_mask[in]
 121  *   Default bit-masks to use when item->mask is not provided.
 122  * @param data[in, out]
 123  *   User structure.
 124  */
 125 static int
 126 mlx4_flow_create_eth(const struct rte_flow_item *item,
 127                      const void *default_mask,
 128                      void *data)
 129 {
 130         const struct rte_flow_item_eth *spec = item->spec;
 131         const struct rte_flow_item_eth *mask = item->mask;
 132         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 133         struct ibv_flow_spec_eth *eth;
 134         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 135         unsigned int i;
 136
 137         ++flow->ibv_attr->num_of_specs;
 138         flow->ibv_attr->priority = 2;
 139         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 140         *eth = (struct ibv_flow_spec_eth) {
 141                 .type = IBV_FLOW_SPEC_ETH,
 142                 .size = eth_size,
 143         };
 144         if (!spec) {
 145                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 146                 return 0;
 147         }
 148         if (!mask)
 149                 mask = default_mask;
 150         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 151         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 152         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 153         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 154         /* Remove unwanted bits from values. */
 155         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 156                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 157                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 158         }
 159         return 0;
 160 }
 161
 162 /**
 163  * Convert VLAN item to Verbs specification.
 164  *
 165  * @param item[in]
 166  *   Item specification.
 167  * @param default_mask[in]
 168  *   Default bit-masks to use when item->mask is not provided.
 169  * @param data[in, out]
 170  *   User structure.
 171  */
 172 static int
 173 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 174                       const void *default_mask,
 175                       void *data)
 176 {
 177         const struct rte_flow_item_vlan *spec = item->spec;
 178         const struct rte_flow_item_vlan *mask = item->mask;
 179         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 180         struct ibv_flow_spec_eth *eth;
 181         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 182
 183         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 184         if (!spec)
 185                 return 0;
 186         if (!mask)
 187                 mask = default_mask;
 188         eth->val.vlan_tag = spec->tci;
 189         eth->mask.vlan_tag = mask->tci;
 190         eth->val.vlan_tag &= eth->mask.vlan_tag;
 191         return 0;
 192 }
 193
 194 /**
 195  * Convert IPv4 item to Verbs specification.
 196  *
 197  * @param item[in]
 198  *   Item specification.
 199  * @param default_mask[in]
 200  *   Default bit-masks to use when item->mask is not provided.
 201  * @param data[in, out]
 202  *   User structure.
 203  */
 204 static int
 205 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 206                       const void *default_mask,
 207                       void *data)
 208 {
 209         const struct rte_flow_item_ipv4 *spec = item->spec;
 210         const struct rte_flow_item_ipv4 *mask = item->mask;
 211         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 212         struct ibv_flow_spec_ipv4 *ipv4;
 213         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 214
 215         ++flow->ibv_attr->num_of_specs;
 216         flow->ibv_attr->priority = 1;
 217         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 218         *ipv4 = (struct ibv_flow_spec_ipv4) {
 219                 .type = IBV_FLOW_SPEC_IPV4,
 220                 .size = ipv4_size,
 221         };
 222         if (!spec)
 223                 return 0;
 224         ipv4->val = (struct ibv_flow_ipv4_filter) {
 225                 .src_ip = spec->hdr.src_addr,
 226                 .dst_ip = spec->hdr.dst_addr,
 227         };
 228         if (!mask)
 229                 mask = default_mask;
 230         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 231                 .src_ip = mask->hdr.src_addr,
 232                 .dst_ip = mask->hdr.dst_addr,
 233         };
 234         /* Remove unwanted bits from values. */
 235         ipv4->val.src_ip &= ipv4->mask.src_ip;
 236         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 237         return 0;
 238 }
 239
 240 /**
 241  * Convert UDP item to Verbs specification.
 242  *
 243  * @param item[in]
 244  *   Item specification.
 245  * @param default_mask[in]
 246  *   Default bit-masks to use when item->mask is not provided.
 247  * @param data[in, out]
 248  *   User structure.
 249  */
 250 static int
 251 mlx4_flow_create_udp(const struct rte_flow_item *item,
 252                      const void *default_mask,
 253                      void *data)
 254 {
 255         const struct rte_flow_item_udp *spec = item->spec;
 256         const struct rte_flow_item_udp *mask = item->mask;
 257         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 258         struct ibv_flow_spec_tcp_udp *udp;
 259         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 260
 261         ++flow->ibv_attr->num_of_specs;
 262         flow->ibv_attr->priority = 0;
 263         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 264         *udp = (struct ibv_flow_spec_tcp_udp) {
 265                 .type = IBV_FLOW_SPEC_UDP,
 266                 .size = udp_size,
 267         };
 268         if (!spec)
 269                 return 0;
 270         udp->val.dst_port = spec->hdr.dst_port;
 271         udp->val.src_port = spec->hdr.src_port;
 272         if (!mask)
 273                 mask = default_mask;
 274         udp->mask.dst_port = mask->hdr.dst_port;
 275         udp->mask.src_port = mask->hdr.src_port;
 276         /* Remove unwanted bits from values. */
 277         udp->val.src_port &= udp->mask.src_port;
 278         udp->val.dst_port &= udp->mask.dst_port;
 279         return 0;
 280 }
 281
 282 /**
 283  * Convert TCP item to Verbs specification.
 284  *
 285  * @param item[in]
 286  *   Item specification.
 287  * @param default_mask[in]
 288  *   Default bit-masks to use when item->mask is not provided.
 289  * @param data[in, out]
 290  *   User structure.
 291  */
 292 static int
 293 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 294                      const void *default_mask,
 295                      void *data)
 296 {
 297         const struct rte_flow_item_tcp *spec = item->spec;
 298         const struct rte_flow_item_tcp *mask = item->mask;
 299         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 300         struct ibv_flow_spec_tcp_udp *tcp;
 301         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 302
 303         ++flow->ibv_attr->num_of_specs;
 304         flow->ibv_attr->priority = 0;
 305         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 306         *tcp = (struct ibv_flow_spec_tcp_udp) {
 307                 .type = IBV_FLOW_SPEC_TCP,
 308                 .size = tcp_size,
 309         };
 310         if (!spec)
 311                 return 0;
 312         tcp->val.dst_port = spec->hdr.dst_port;
 313         tcp->val.src_port = spec->hdr.src_port;
 314         if (!mask)
 315                 mask = default_mask;
 316         tcp->mask.dst_port = mask->hdr.dst_port;
 317         tcp->mask.src_port = mask->hdr.src_port;
 318         /* Remove unwanted bits from values. */
 319         tcp->val.src_port &= tcp->mask.src_port;
 320         tcp->val.dst_port &= tcp->mask.dst_port;
 321         return 0;
 322 }
 323
 324 /**
 325  * Check support for a given item.
 326  *
 327  * @param item[in]
 328  *   Item specification.
 329  * @param mask[in]
 330  *   Bit-masks covering supported fields to compare with spec, last and mask in
 331  *   \item.
 332  * @param size
 333  *   Bit-Mask size in bytes.
 334  *
 335  * @return
 336  *   0 on success, negative value otherwise.
 337  */
 338 static int
 339 mlx4_flow_item_validate(const struct rte_flow_item *item,
 340                         const uint8_t *mask, unsigned int size)
 341 {
 342         int ret = 0;
 343
 344         if (!item->spec && (item->mask || item->last))
 345                 return -1;
 346         if (item->spec && !item->mask) {
 347                 unsigned int i;
 348                 const uint8_t *spec = item->spec;
 349
 350                 for (i = 0; i < size; ++i)
 351                         if ((spec[i] | mask[i]) != mask[i])
 352                                 return -1;
 353         }
 354         if (item->last && !item->mask) {
 355                 unsigned int i;
 356                 const uint8_t *spec = item->last;
 357
 358                 for (i = 0; i < size; ++i)
 359                         if ((spec[i] | mask[i]) != mask[i])
 360                                 return -1;
 361         }
 362         if (item->spec && item->last) {
 363                 uint8_t spec[size];
 364                 uint8_t last[size];
 365                 const uint8_t *apply = mask;
 366                 unsigned int i;
 367
 368                 if (item->mask)
 369                         apply = item->mask;
 370                 for (i = 0; i < size; ++i) {
 371                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 372                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 373                 }
 374                 ret = memcmp(spec, last, size);
 375         }
 376         return ret;
 377 }
 378
 379 static int
 380 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 381                        const uint8_t *mask, unsigned int size)
 382 {
 383         if (item->mask) {
 384                 const struct rte_flow_item_eth *mask = item->mask;
 385
 386                 if (mask->dst.addr_bytes[0] != 0xff ||
 387                                 mask->dst.addr_bytes[1] != 0xff ||
 388                                 mask->dst.addr_bytes[2] != 0xff ||
 389                                 mask->dst.addr_bytes[3] != 0xff ||
 390                                 mask->dst.addr_bytes[4] != 0xff ||
 391                                 mask->dst.addr_bytes[5] != 0xff)
 392                         return -1;
 393         }
 394         return mlx4_flow_item_validate(item, mask, size);
 395 }
 396
 397 static int
 398 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 399                         const uint8_t *mask, unsigned int size)
 400 {
 401         if (item->mask) {
 402                 const struct rte_flow_item_vlan *mask = item->mask;
 403
 404                 if (mask->tci != 0 &&
 405                     ntohs(mask->tci) != 0x0fff)
 406                         return -1;
 407         }
 408         return mlx4_flow_item_validate(item, mask, size);
 409 }
 410
 411 static int
 412 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 413                         const uint8_t *mask, unsigned int size)
 414 {
 415         if (item->mask) {
 416                 const struct rte_flow_item_ipv4 *mask = item->mask;
 417
 418                 if (mask->hdr.src_addr != 0 &&
 419                     mask->hdr.src_addr != 0xffffffff)
 420                         return -1;
 421                 if (mask->hdr.dst_addr != 0 &&
 422                     mask->hdr.dst_addr != 0xffffffff)
 423                         return -1;
 424         }
 425         return mlx4_flow_item_validate(item, mask, size);
 426 }
 427
 428 static int
 429 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 430                        const uint8_t *mask, unsigned int size)
 431 {
 432         if (item->mask) {
 433                 const struct rte_flow_item_udp *mask = item->mask;
 434
 435                 if (mask->hdr.src_port != 0 &&
 436                     mask->hdr.src_port != 0xffff)
 437                         return -1;
 438                 if (mask->hdr.dst_port != 0 &&
 439                     mask->hdr.dst_port != 0xffff)
 440                         return -1;
 441         }
 442         return mlx4_flow_item_validate(item, mask, size);
 443 }
 444
 445 static int
 446 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 447                        const uint8_t *mask, unsigned int size)
 448 {
 449         if (item->mask) {
 450                 const struct rte_flow_item_tcp *mask = item->mask;
 451
 452                 if (mask->hdr.src_port != 0 &&
 453                     mask->hdr.src_port != 0xffff)
 454                         return -1;
 455                 if (mask->hdr.dst_port != 0 &&
 456                     mask->hdr.dst_port != 0xffff)
 457                         return -1;
 458         }
 459         return mlx4_flow_item_validate(item, mask, size);
 460 }
 461
 462 /** Graph of supported items and associated actions. */
 463 static const struct mlx4_flow_items mlx4_flow_items[] = {
 464         [RTE_FLOW_ITEM_TYPE_END] = {
 465                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 466         },
 467         [RTE_FLOW_ITEM_TYPE_ETH] = {
 468                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 469                                RTE_FLOW_ITEM_TYPE_IPV4),
 470                 .actions = valid_actions,
 471                 .mask = &(const struct rte_flow_item_eth){
 472                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 473                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 474                 },
 475                 .default_mask = &rte_flow_item_eth_mask,
 476                 .mask_sz = sizeof(struct rte_flow_item_eth),
 477                 .validate = mlx4_flow_validate_eth,
 478                 .convert = mlx4_flow_create_eth,
 479                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 480         },
 481         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 482                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 483                 .actions = valid_actions,
 484                 .mask = &(const struct rte_flow_item_vlan){
 485                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 486 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 487                         .tci = 0x0fff,
 488 #else
 489                         .tci = 0xff0f,
 490 #endif
 491                 },
 492                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 493                 .validate = mlx4_flow_validate_vlan,
 494                 .convert = mlx4_flow_create_vlan,
 495                 .dst_sz = 0,
 496         },
 497         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 498                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 499                                RTE_FLOW_ITEM_TYPE_TCP),
 500                 .actions = valid_actions,
 501                 .mask = &(const struct rte_flow_item_ipv4){
 502                         .hdr = {
 503                                 .src_addr = -1,
 504                                 .dst_addr = -1,
 505                         },
 506                 },
 507                 .default_mask = &rte_flow_item_ipv4_mask,
 508                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 509                 .validate = mlx4_flow_validate_ipv4,
 510                 .convert = mlx4_flow_create_ipv4,
 511                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 512         },
 513         [RTE_FLOW_ITEM_TYPE_UDP] = {
 514                 .actions = valid_actions,
 515                 .mask = &(const struct rte_flow_item_udp){
 516                         .hdr = {
 517                                 .src_port = -1,
 518                                 .dst_port = -1,
 519                         },
 520                 },
 521                 .default_mask = &rte_flow_item_udp_mask,
 522                 .mask_sz = sizeof(struct rte_flow_item_udp),
 523                 .validate = mlx4_flow_validate_udp,
 524                 .convert = mlx4_flow_create_udp,
 525                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 526         },
 527         [RTE_FLOW_ITEM_TYPE_TCP] = {
 528                 .actions = valid_actions,
 529                 .mask = &(const struct rte_flow_item_tcp){
 530                         .hdr = {
 531                                 .src_port = -1,
 532                                 .dst_port = -1,
 533                         },
 534                 },
 535                 .default_mask = &rte_flow_item_tcp_mask,
 536                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 537                 .validate = mlx4_flow_validate_tcp,
 538                 .convert = mlx4_flow_create_tcp,
 539                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 540         },
 541 };
 542
 543 /**
 544  * Validate a flow supported by the NIC.
 545  *
 546  * @param priv
 547  *   Pointer to private structure.
 548  * @param[in] attr
 549  *   Flow rule attributes.
 550  * @param[in] items
 551  *   Pattern specification (list terminated by the END pattern item).
 552  * @param[in] actions
 553  *   Associated actions (list terminated by the END action).
 554  * @param[out] error
 555  *   Perform verbose error reporting if not NULL.
 556  * @param[in, out] flow
 557  *   Flow structure to update.
 558  *
 559  * @return
 560  *   0 on success, a negative errno value otherwise and rte_errno is set.
 561  */
 562 static int
 563 priv_flow_validate(struct priv *priv,
 564                    const struct rte_flow_attr *attr,
 565                    const struct rte_flow_item items[],
 566                    const struct rte_flow_action actions[],
 567                    struct rte_flow_error *error,
 568                    struct mlx4_flow *flow)
 569 {
 570         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 571         struct mlx4_flow_action action = {
 572                 .queue = 0,
 573                 .drop = 0,
 574         };
 575
 576         (void)priv;
 577         if (attr->group) {
 578                 rte_flow_error_set(error, ENOTSUP,
 579                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 580                                    NULL,
 581                                    "groups are not supported");
 582                 return -rte_errno;
 583         }
 584         if (attr->priority) {
 585                 rte_flow_error_set(error, ENOTSUP,
 586                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 587                                    NULL,
 588                                    "priorities are not supported");
 589                 return -rte_errno;
 590         }
 591         if (attr->egress) {
 592                 rte_flow_error_set(error, ENOTSUP,
 593                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 594                                    NULL,
 595                                    "egress is not supported");
 596                 return -rte_errno;
 597         }
 598         if (!attr->ingress) {
 599                 rte_flow_error_set(error, ENOTSUP,
 600                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 601                                    NULL,
 602                                    "only ingress is supported");
 603                 return -rte_errno;
 604         }
 605         /* Go over items list. */
 606         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 607                 const struct mlx4_flow_items *token = NULL;
 608                 unsigned int i;
 609                 int err;
 610
 611                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 612                         continue;
 613                 /*
 614                  * The nic can support patterns with NULL eth spec only
 615                  * if eth is a single item in a rule.
 616                  */
 617                 if (!items->spec &&
 618                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 619                         const struct rte_flow_item *next = items + 1;
 620
 621                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 622                                 rte_flow_error_set(error, ENOTSUP,
 623                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 624                                                    items,
 625                                                    "the rule requires"
 626                                                    " an Ethernet spec");
 627                                 return -rte_errno;
 628                         }
 629                 }
 630                 for (i = 0;
 631                      cur_item->items &&
 632                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 633                      ++i) {
 634                         if (cur_item->items[i] == items->type) {
 635                                 token = &mlx4_flow_items[items->type];
 636                                 break;
 637                         }
 638                 }
 639                 if (!token)
 640                         goto exit_item_not_supported;
 641                 cur_item = token;
 642                 err = cur_item->validate(items,
 643                                         (const uint8_t *)cur_item->mask,
 644                                          cur_item->mask_sz);
 645                 if (err)
 646                         goto exit_item_not_supported;
 647                 if (flow->ibv_attr && cur_item->convert) {
 648                         err = cur_item->convert(items,
 649                                                 (cur_item->default_mask ?
 650                                                  cur_item->default_mask :
 651                                                  cur_item->mask),
 652                                                  flow);
 653                         if (err)
 654                                 goto exit_item_not_supported;
 655                 }
 656                 flow->offset += cur_item->dst_sz;
 657         }
 658         /* Go over actions list */
 659         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 660                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 661                         continue;
 662                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 663                         action.drop = 1;
 664                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 665                         const struct rte_flow_action_queue *queue =
 666                                 (const struct rte_flow_action_queue *)
 667                                 actions->conf;
 668
 669                         if (!queue || (queue->index > (priv->rxqs_n - 1)))
 670                                 goto exit_action_not_supported;
 671                         action.queue = 1;
 672                 } else {
 673                         goto exit_action_not_supported;
 674                 }
 675         }
 676         if (!action.queue && !action.drop) {
 677                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 678                                    NULL, "no valid action");
 679                 return -rte_errno;
 680         }
 681         return 0;
 682 exit_item_not_supported:
 683         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 684                            items, "item not supported");
 685         return -rte_errno;
 686 exit_action_not_supported:
 687         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 688                            actions, "action not supported");
 689         return -rte_errno;
 690 }
 691
 692 /**
 693  * Validate a flow supported by the NIC.
 694  *
 695  * @see rte_flow_validate()
 696  * @see rte_flow_ops
 697  */
 698 int
 699 mlx4_flow_validate(struct rte_eth_dev *dev,
 700                    const struct rte_flow_attr *attr,
 701                    const struct rte_flow_item items[],
 702                    const struct rte_flow_action actions[],
 703                    struct rte_flow_error *error)
 704 {
 705         struct priv *priv = dev->data->dev_private;
 706         int ret;
 707         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 708
 709         priv_lock(priv);
 710         ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
 711         priv_unlock(priv);
 712         return ret;
 713 }
 714
 715 /**
 716  * Destroy a drop queue.
 717  *
 718  * @param priv
 719  *   Pointer to private structure.
 720  */
 721 static void
 722 mlx4_flow_destroy_drop_queue(struct priv *priv)
 723 {
 724         if (priv->flow_drop_queue) {
 725                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 726
 727                 priv->flow_drop_queue = NULL;
 728                 claim_zero(ibv_destroy_qp(fdq->qp));
 729                 claim_zero(ibv_destroy_cq(fdq->cq));
 730                 rte_free(fdq);
 731         }
 732 }
 733
 734 /**
 735  * Create a single drop queue for all drop flows.
 736  *
 737  * @param priv
 738  *   Pointer to private structure.
 739  *
 740  * @return
 741  *   0 on success, negative value otherwise.
 742  */
 743 static int
 744 mlx4_flow_create_drop_queue(struct priv *priv)
 745 {
 746         struct ibv_qp *qp;
 747         struct ibv_cq *cq;
 748         struct rte_flow_drop *fdq;
 749
 750         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 751         if (!fdq) {
 752                 ERROR("Cannot allocate memory for drop struct");
 753                 goto err;
 754         }
 755         cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 756         if (!cq) {
 757                 ERROR("Cannot create drop CQ");
 758                 goto err_create_cq;
 759         }
 760         qp = ibv_create_qp(priv->pd,
 761                            &(struct ibv_qp_init_attr){
 762                                 .send_cq = cq,
 763                                 .recv_cq = cq,
 764                                 .cap = {
 765                                         .max_recv_wr = 1,
 766                                         .max_recv_sge = 1,
 767                                 },
 768                                 .qp_type = IBV_QPT_RAW_PACKET,
 769                            });
 770         if (!qp) {
 771                 ERROR("Cannot create drop QP");
 772                 goto err_create_qp;
 773         }
 774         *fdq = (struct rte_flow_drop){
 775                 .qp = qp,
 776                 .cq = cq,
 777         };
 778         priv->flow_drop_queue = fdq;
 779         return 0;
 780 err_create_qp:
 781         claim_zero(ibv_destroy_cq(cq));
 782 err_create_cq:
 783         rte_free(fdq);
 784 err:
 785         return -1;
 786 }
 787
 788 /**
 789  * Complete flow rule creation.
 790  *
 791  * @param priv
 792  *   Pointer to private structure.
 793  * @param ibv_attr
 794  *   Verbs flow attributes.
 795  * @param action
 796  *   Target action structure.
 797  * @param[out] error
 798  *   Perform verbose error reporting if not NULL.
 799  *
 800  * @return
 801  *   A flow if the rule could be created.
 802  */
 803 static struct rte_flow *
 804 priv_flow_create_action_queue(struct priv *priv,
 805                               struct ibv_flow_attr *ibv_attr,
 806                               struct mlx4_flow_action *action,
 807                               struct rte_flow_error *error)
 808 {
 809         struct ibv_qp *qp;
 810         struct rte_flow *rte_flow;
 811
 812         assert(priv->pd);
 813         assert(priv->ctx);
 814         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 815         if (!rte_flow) {
 816                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 817                                    NULL, "cannot allocate flow memory");
 818                 return NULL;
 819         }
 820         if (action->drop) {
 821                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 822         } else {
 823                 struct rxq *rxq = (*priv->rxqs)[action->queue_id];
 824
 825                 qp = rxq->qp;
 826                 rte_flow->qp = qp;
 827         }
 828         rte_flow->ibv_attr = ibv_attr;
 829         if (!priv->started)
 830                 return rte_flow;
 831         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
 832         if (!rte_flow->ibv_flow) {
 833                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 834                                    NULL, "flow rule creation failure");
 835                 goto error;
 836         }
 837         return rte_flow;
 838
 839 error:
 840         rte_free(rte_flow);
 841         return NULL;
 842 }
 843
 844 /**
 845  * Convert a flow.
 846  *
 847  * @param priv
 848  *   Pointer to private structure.
 849  * @param[in] attr
 850  *   Flow rule attributes.
 851  * @param[in] items
 852  *   Pattern specification (list terminated by the END pattern item).
 853  * @param[in] actions
 854  *   Associated actions (list terminated by the END action).
 855  * @param[out] error
 856  *   Perform verbose error reporting if not NULL.
 857  *
 858  * @return
 859  *   A flow on success, NULL otherwise.
 860  */
 861 static struct rte_flow *
 862 priv_flow_create(struct priv *priv,
 863                  const struct rte_flow_attr *attr,
 864                  const struct rte_flow_item items[],
 865                  const struct rte_flow_action actions[],
 866                  struct rte_flow_error *error)
 867 {
 868         struct rte_flow *rte_flow;
 869         struct mlx4_flow_action action;
 870         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
 871         int err;
 872
 873         err = priv_flow_validate(priv, attr, items, actions, error, &flow);
 874         if (err)
 875                 return NULL;
 876         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
 877         if (!flow.ibv_attr) {
 878                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 879                                    NULL, "cannot allocate ibv_attr memory");
 880                 return NULL;
 881         }
 882         flow.offset = sizeof(struct ibv_flow_attr);
 883         *flow.ibv_attr = (struct ibv_flow_attr){
 884                 .comp_mask = 0,
 885                 .type = IBV_FLOW_ATTR_NORMAL,
 886                 .size = sizeof(struct ibv_flow_attr),
 887                 .priority = attr->priority,
 888                 .num_of_specs = 0,
 889                 .port = priv->port,
 890                 .flags = 0,
 891         };
 892         claim_zero(priv_flow_validate(priv, attr, items, actions,
 893                                       error, &flow));
 894         action = (struct mlx4_flow_action){
 895                 .queue = 0,
 896                 .drop = 0,
 897         };
 898         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 899                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 900                         continue;
 901                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 902                         action.queue = 1;
 903                         action.queue_id =
 904                                 ((const struct rte_flow_action_queue *)
 905                                  actions->conf)->index;
 906                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 907                         action.drop = 1;
 908                 } else {
 909                         rte_flow_error_set(error, ENOTSUP,
 910                                            RTE_FLOW_ERROR_TYPE_ACTION,
 911                                            actions, "unsupported action");
 912                         goto exit;
 913                 }
 914         }
 915         rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
 916                                                  &action, error);
 917         if (rte_flow)
 918                 return rte_flow;
 919 exit:
 920         rte_free(flow.ibv_attr);
 921         return NULL;
 922 }
 923
 924 /**
 925  * Create a flow.
 926  *
 927  * @see rte_flow_create()
 928  * @see rte_flow_ops
 929  */
 930 struct rte_flow *
 931 mlx4_flow_create(struct rte_eth_dev *dev,
 932                  const struct rte_flow_attr *attr,
 933                  const struct rte_flow_item items[],
 934                  const struct rte_flow_action actions[],
 935                  struct rte_flow_error *error)
 936 {
 937         struct priv *priv = dev->data->dev_private;
 938         struct rte_flow *flow;
 939
 940         priv_lock(priv);
 941         flow = priv_flow_create(priv, attr, items, actions, error);
 942         if (flow) {
 943                 LIST_INSERT_HEAD(&priv->flows, flow, next);
 944                 DEBUG("Flow created %p", (void *)flow);
 945         }
 946         priv_unlock(priv);
 947         return flow;
 948 }
 949
 950 /**
 951  * @see rte_flow_isolate()
 952  *
 953  * Must be done before calling dev_configure().
 954  *
 955  * @param dev
 956  *   Pointer to the ethernet device structure.
 957  * @param enable
 958  *   Nonzero to enter isolated mode, attempt to leave it otherwise.
 959  * @param[out] error
 960  *   Perform verbose error reporting if not NULL. PMDs initialize this
 961  *   structure in case of error only.
 962  *
 963  * @return
 964  *   0 on success, a negative value on error.
 965  */
 966 int
 967 mlx4_flow_isolate(struct rte_eth_dev *dev,
 968                   int enable,
 969                   struct rte_flow_error *error)
 970 {
 971         struct priv *priv = dev->data->dev_private;
 972
 973         priv_lock(priv);
 974         if (priv->rxqs) {
 975                 rte_flow_error_set(error, ENOTSUP,
 976                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 977                                    NULL, "isolated mode must be set"
 978                                    " before configuring the device");
 979                 priv_unlock(priv);
 980                 return -rte_errno;
 981         }
 982         priv->isolated = !!enable;
 983         priv_unlock(priv);
 984         return 0;
 985 }
 986
 987 /**
 988  * Destroy a flow.
 989  *
 990  * @param priv
 991  *   Pointer to private structure.
 992  * @param[in] flow
 993  *   Flow to destroy.
 994  */
 995 static void
 996 priv_flow_destroy(struct priv *priv, struct rte_flow *flow)
 997 {
 998         (void)priv;
 999         LIST_REMOVE(flow, next);
1000         if (flow->ibv_flow)
1001                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1002         rte_free(flow->ibv_attr);
1003         DEBUG("Flow destroyed %p", (void *)flow);
1004         rte_free(flow);
1005 }
1006
1007 /**
1008  * Destroy a flow.
1009  *
1010  * @see rte_flow_destroy()
1011  * @see rte_flow_ops
1012  */
1013 int
1014 mlx4_flow_destroy(struct rte_eth_dev *dev,
1015                   struct rte_flow *flow,
1016                   struct rte_flow_error *error)
1017 {
1018         struct priv *priv = dev->data->dev_private;
1019
1020         (void)error;
1021         priv_lock(priv);
1022         priv_flow_destroy(priv, flow);
1023         priv_unlock(priv);
1024         return 0;
1025 }
1026
1027 /**
1028  * Destroy all flows.
1029  *
1030  * @param priv
1031  *   Pointer to private structure.
1032  */
1033 static void
1034 priv_flow_flush(struct priv *priv)
1035 {
1036         while (!LIST_EMPTY(&priv->flows)) {
1037                 struct rte_flow *flow;
1038
1039                 flow = LIST_FIRST(&priv->flows);
1040                 priv_flow_destroy(priv, flow);
1041         }
1042 }
1043
1044 /**
1045  * Destroy all flows.
1046  *
1047  * @see rte_flow_flush()
1048  * @see rte_flow_ops
1049  */
1050 int
1051 mlx4_flow_flush(struct rte_eth_dev *dev,
1052                 struct rte_flow_error *error)
1053 {
1054         struct priv *priv = dev->data->dev_private;
1055
1056         (void)error;
1057         priv_lock(priv);
1058         priv_flow_flush(priv);
1059         priv_unlock(priv);
1060         return 0;
1061 }
1062
1063 /**
1064  * Remove all flows.
1065  *
1066  * Called by dev_stop() to remove all flows.
1067  *
1068  * @param priv
1069  *   Pointer to private structure.
1070  */
1071 void
1072 mlx4_priv_flow_stop(struct priv *priv)
1073 {
1074         struct rte_flow *flow;
1075
1076         for (flow = LIST_FIRST(&priv->flows);
1077              flow;
1078              flow = LIST_NEXT(flow, next)) {
1079                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1080                 flow->ibv_flow = NULL;
1081                 DEBUG("Flow %p removed", (void *)flow);
1082         }
1083         mlx4_flow_destroy_drop_queue(priv);
1084 }
1085
1086 /**
1087  * Add all flows.
1088  *
1089  * @param priv
1090  *   Pointer to private structure.
1091  *
1092  * @return
1093  *   0 on success, a errno value otherwise and rte_errno is set.
1094  */
1095 int
1096 mlx4_priv_flow_start(struct priv *priv)
1097 {
1098         int ret;
1099         struct ibv_qp *qp;
1100         struct rte_flow *flow;
1101
1102         ret = mlx4_flow_create_drop_queue(priv);
1103         if (ret)
1104                 return -1;
1105         for (flow = LIST_FIRST(&priv->flows);
1106              flow;
1107              flow = LIST_NEXT(flow, next)) {
1108                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1109                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1110                 if (!flow->ibv_flow) {
1111                         DEBUG("Flow %p cannot be applied", (void *)flow);
1112                         rte_errno = EINVAL;
1113                         return rte_errno;
1114                 }
1115                 DEBUG("Flow %p applied", (void *)flow);
1116         }
1117         return 0;
1118 }