drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Flow API operations for mlx4 driver.
  37  */
  38
  39 #include <arpa/inet.h>
  40 #include <assert.h>
  41 #include <errno.h>
  42 #include <stdalign.h>
  43 #include <stddef.h>
  44 #include <stdint.h>
  45 #include <string.h>
  46 #include <sys/queue.h>
  47
  48 /* Verbs headers do not support -pedantic. */
  49 #ifdef PEDANTIC
  50 #pragma GCC diagnostic ignored "-Wpedantic"
  51 #endif
  52 #include <infiniband/verbs.h>
  53 #ifdef PEDANTIC
  54 #pragma GCC diagnostic error "-Wpedantic"
  55 #endif
  56
  57 #include <rte_byteorder.h>
  58 #include <rte_errno.h>
  59 #include <rte_eth_ctrl.h>
  60 #include <rte_ethdev.h>
  61 #include <rte_flow.h>
  62 #include <rte_flow_driver.h>
  63 #include <rte_malloc.h>
  64
  65 /* PMD headers. */
  66 #include "mlx4.h"
  67 #include "mlx4_flow.h"
  68 #include "mlx4_rxtx.h"
  69 #include "mlx4_utils.h"
  70
  71 /** Static initializer for a list of subsequent item types. */
  72 #define NEXT_ITEM(...) \
  73         (const enum rte_flow_item_type []){ \
  74                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  75         }
  76
  77 /** Processor structure associated with a flow item. */
  78 struct mlx4_flow_proc_item {
  79         /** Bit-masks corresponding to the possibilities for the item. */
  80         const void *mask;
  81         /**
  82          * Default bit-masks to use when item->mask is not provided. When
  83          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  84          * used instead.
  85          */
  86         const void *default_mask;
  87         /** Bit-masks size in bytes. */
  88         const unsigned int mask_sz;
  89         /**
  90          * Check support for a given item.
  91          *
  92          * @param item[in]
  93          *   Item specification.
  94          * @param mask[in]
  95          *   Bit-masks covering supported fields to compare with spec,
  96          *   last and mask in
  97          *   \item.
  98          * @param size
  99          *   Bit-Mask size in bytes.
 100          *
 101          * @return
 102          *   0 on success, negative value otherwise.
 103          */
 104         int (*validate)(const struct rte_flow_item *item,
 105                         const uint8_t *mask, unsigned int size);
 106         /**
 107          * Conversion function from rte_flow to NIC specific flow.
 108          *
 109          * @param item
 110          *   rte_flow item to convert.
 111          * @param default_mask
 112          *   Default bit-masks to use when item->mask is not provided.
 113          * @param flow
 114          *   Flow rule handle to update.
 115          *
 116          * @return
 117          *   0 on success, negative value otherwise.
 118          */
 119         int (*convert)(const struct rte_flow_item *item,
 120                        const void *default_mask,
 121                        struct rte_flow *flow);
 122         /** Size in bytes of the destination structure. */
 123         const unsigned int dst_sz;
 124         /** List of possible subsequent items. */
 125         const enum rte_flow_item_type *const next_item;
 126 };
 127
 128 struct rte_flow_drop {
 129         struct ibv_qp *qp; /**< Verbs queue pair. */
 130         struct ibv_cq *cq; /**< Verbs completion queue. */
 131 };
 132
 133 /**
 134  * Convert Ethernet item to Verbs specification.
 135  *
 136  * @param item[in]
 137  *   Item specification.
 138  * @param default_mask[in]
 139  *   Default bit-masks to use when item->mask is not provided.
 140  * @param flow[in, out]
 141  *   Flow rule handle to update.
 142  */
 143 static int
 144 mlx4_flow_create_eth(const struct rte_flow_item *item,
 145                      const void *default_mask,
 146                      struct rte_flow *flow)
 147 {
 148         const struct rte_flow_item_eth *spec = item->spec;
 149         const struct rte_flow_item_eth *mask = item->mask;
 150         struct ibv_flow_spec_eth *eth;
 151         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 152         unsigned int i;
 153
 154         ++flow->ibv_attr->num_of_specs;
 155         flow->ibv_attr->priority = 2;
 156         eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 157         *eth = (struct ibv_flow_spec_eth) {
 158                 .type = IBV_FLOW_SPEC_ETH,
 159                 .size = eth_size,
 160         };
 161         if (!spec) {
 162                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 163                 return 0;
 164         }
 165         if (!mask)
 166                 mask = default_mask;
 167         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 168         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 169         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 170         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 171         /* Remove unwanted bits from values. */
 172         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 173                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 174                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 175         }
 176         return 0;
 177 }
 178
 179 /**
 180  * Convert VLAN item to Verbs specification.
 181  *
 182  * @param item[in]
 183  *   Item specification.
 184  * @param default_mask[in]
 185  *   Default bit-masks to use when item->mask is not provided.
 186  * @param flow[in, out]
 187  *   Flow rule handle to update.
 188  */
 189 static int
 190 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 191                       const void *default_mask,
 192                       struct rte_flow *flow)
 193 {
 194         const struct rte_flow_item_vlan *spec = item->spec;
 195         const struct rte_flow_item_vlan *mask = item->mask;
 196         struct ibv_flow_spec_eth *eth;
 197         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 198
 199         eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size -
 200                        eth_size);
 201         if (!spec)
 202                 return 0;
 203         if (!mask)
 204                 mask = default_mask;
 205         eth->val.vlan_tag = spec->tci;
 206         eth->mask.vlan_tag = mask->tci;
 207         eth->val.vlan_tag &= eth->mask.vlan_tag;
 208         return 0;
 209 }
 210
 211 /**
 212  * Convert IPv4 item to Verbs specification.
 213  *
 214  * @param item[in]
 215  *   Item specification.
 216  * @param default_mask[in]
 217  *   Default bit-masks to use when item->mask is not provided.
 218  * @param flow[in, out]
 219  *   Flow rule handle to update.
 220  */
 221 static int
 222 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 223                       const void *default_mask,
 224                       struct rte_flow *flow)
 225 {
 226         const struct rte_flow_item_ipv4 *spec = item->spec;
 227         const struct rte_flow_item_ipv4 *mask = item->mask;
 228         struct ibv_flow_spec_ipv4 *ipv4;
 229         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 230
 231         ++flow->ibv_attr->num_of_specs;
 232         flow->ibv_attr->priority = 1;
 233         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 234         *ipv4 = (struct ibv_flow_spec_ipv4) {
 235                 .type = IBV_FLOW_SPEC_IPV4,
 236                 .size = ipv4_size,
 237         };
 238         if (!spec)
 239                 return 0;
 240         ipv4->val = (struct ibv_flow_ipv4_filter) {
 241                 .src_ip = spec->hdr.src_addr,
 242                 .dst_ip = spec->hdr.dst_addr,
 243         };
 244         if (!mask)
 245                 mask = default_mask;
 246         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 247                 .src_ip = mask->hdr.src_addr,
 248                 .dst_ip = mask->hdr.dst_addr,
 249         };
 250         /* Remove unwanted bits from values. */
 251         ipv4->val.src_ip &= ipv4->mask.src_ip;
 252         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 253         return 0;
 254 }
 255
 256 /**
 257  * Convert UDP item to Verbs specification.
 258  *
 259  * @param item[in]
 260  *   Item specification.
 261  * @param default_mask[in]
 262  *   Default bit-masks to use when item->mask is not provided.
 263  * @param flow[in, out]
 264  *   Flow rule handle to update.
 265  */
 266 static int
 267 mlx4_flow_create_udp(const struct rte_flow_item *item,
 268                      const void *default_mask,
 269                      struct rte_flow *flow)
 270 {
 271         const struct rte_flow_item_udp *spec = item->spec;
 272         const struct rte_flow_item_udp *mask = item->mask;
 273         struct ibv_flow_spec_tcp_udp *udp;
 274         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 275
 276         ++flow->ibv_attr->num_of_specs;
 277         flow->ibv_attr->priority = 0;
 278         udp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 279         *udp = (struct ibv_flow_spec_tcp_udp) {
 280                 .type = IBV_FLOW_SPEC_UDP,
 281                 .size = udp_size,
 282         };
 283         if (!spec)
 284                 return 0;
 285         udp->val.dst_port = spec->hdr.dst_port;
 286         udp->val.src_port = spec->hdr.src_port;
 287         if (!mask)
 288                 mask = default_mask;
 289         udp->mask.dst_port = mask->hdr.dst_port;
 290         udp->mask.src_port = mask->hdr.src_port;
 291         /* Remove unwanted bits from values. */
 292         udp->val.src_port &= udp->mask.src_port;
 293         udp->val.dst_port &= udp->mask.dst_port;
 294         return 0;
 295 }
 296
 297 /**
 298  * Convert TCP item to Verbs specification.
 299  *
 300  * @param item[in]
 301  *   Item specification.
 302  * @param default_mask[in]
 303  *   Default bit-masks to use when item->mask is not provided.
 304  * @param flow[in, out]
 305  *   Flow rule handle to update.
 306  */
 307 static int
 308 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 309                      const void *default_mask,
 310                      struct rte_flow *flow)
 311 {
 312         const struct rte_flow_item_tcp *spec = item->spec;
 313         const struct rte_flow_item_tcp *mask = item->mask;
 314         struct ibv_flow_spec_tcp_udp *tcp;
 315         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 316
 317         ++flow->ibv_attr->num_of_specs;
 318         flow->ibv_attr->priority = 0;
 319         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
 320         *tcp = (struct ibv_flow_spec_tcp_udp) {
 321                 .type = IBV_FLOW_SPEC_TCP,
 322                 .size = tcp_size,
 323         };
 324         if (!spec)
 325                 return 0;
 326         tcp->val.dst_port = spec->hdr.dst_port;
 327         tcp->val.src_port = spec->hdr.src_port;
 328         if (!mask)
 329                 mask = default_mask;
 330         tcp->mask.dst_port = mask->hdr.dst_port;
 331         tcp->mask.src_port = mask->hdr.src_port;
 332         /* Remove unwanted bits from values. */
 333         tcp->val.src_port &= tcp->mask.src_port;
 334         tcp->val.dst_port &= tcp->mask.dst_port;
 335         return 0;
 336 }
 337
 338 /**
 339  * Check support for a given item.
 340  *
 341  * @param item[in]
 342  *   Item specification.
 343  * @param mask[in]
 344  *   Bit-masks covering supported fields to compare with spec, last and mask in
 345  *   \item.
 346  * @param size
 347  *   Bit-Mask size in bytes.
 348  *
 349  * @return
 350  *   0 on success, negative value otherwise.
 351  */
 352 static int
 353 mlx4_flow_item_validate(const struct rte_flow_item *item,
 354                         const uint8_t *mask, unsigned int size)
 355 {
 356         int ret = 0;
 357
 358         if (!item->spec && (item->mask || item->last))
 359                 return -1;
 360         if (item->spec && !item->mask) {
 361                 unsigned int i;
 362                 const uint8_t *spec = item->spec;
 363
 364                 for (i = 0; i < size; ++i)
 365                         if ((spec[i] | mask[i]) != mask[i])
 366                                 return -1;
 367         }
 368         if (item->last && !item->mask) {
 369                 unsigned int i;
 370                 const uint8_t *spec = item->last;
 371
 372                 for (i = 0; i < size; ++i)
 373                         if ((spec[i] | mask[i]) != mask[i])
 374                                 return -1;
 375         }
 376         if (item->spec && item->last) {
 377                 uint8_t spec[size];
 378                 uint8_t last[size];
 379                 const uint8_t *apply = mask;
 380                 unsigned int i;
 381
 382                 if (item->mask)
 383                         apply = item->mask;
 384                 for (i = 0; i < size; ++i) {
 385                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 386                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 387                 }
 388                 ret = memcmp(spec, last, size);
 389         }
 390         return ret;
 391 }
 392
 393 static int
 394 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 395                        const uint8_t *mask, unsigned int size)
 396 {
 397         if (item->mask) {
 398                 const struct rte_flow_item_eth *mask = item->mask;
 399
 400                 if (mask->dst.addr_bytes[0] != 0xff ||
 401                                 mask->dst.addr_bytes[1] != 0xff ||
 402                                 mask->dst.addr_bytes[2] != 0xff ||
 403                                 mask->dst.addr_bytes[3] != 0xff ||
 404                                 mask->dst.addr_bytes[4] != 0xff ||
 405                                 mask->dst.addr_bytes[5] != 0xff)
 406                         return -1;
 407         }
 408         return mlx4_flow_item_validate(item, mask, size);
 409 }
 410
 411 static int
 412 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 413                         const uint8_t *mask, unsigned int size)
 414 {
 415         if (item->mask) {
 416                 const struct rte_flow_item_vlan *mask = item->mask;
 417
 418                 if (mask->tci != 0 &&
 419                     ntohs(mask->tci) != 0x0fff)
 420                         return -1;
 421         }
 422         return mlx4_flow_item_validate(item, mask, size);
 423 }
 424
 425 static int
 426 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 427                         const uint8_t *mask, unsigned int size)
 428 {
 429         if (item->mask) {
 430                 const struct rte_flow_item_ipv4 *mask = item->mask;
 431
 432                 if (mask->hdr.src_addr != 0 &&
 433                     mask->hdr.src_addr != 0xffffffff)
 434                         return -1;
 435                 if (mask->hdr.dst_addr != 0 &&
 436                     mask->hdr.dst_addr != 0xffffffff)
 437                         return -1;
 438         }
 439         return mlx4_flow_item_validate(item, mask, size);
 440 }
 441
 442 static int
 443 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 444                        const uint8_t *mask, unsigned int size)
 445 {
 446         if (item->mask) {
 447                 const struct rte_flow_item_udp *mask = item->mask;
 448
 449                 if (mask->hdr.src_port != 0 &&
 450                     mask->hdr.src_port != 0xffff)
 451                         return -1;
 452                 if (mask->hdr.dst_port != 0 &&
 453                     mask->hdr.dst_port != 0xffff)
 454                         return -1;
 455         }
 456         return mlx4_flow_item_validate(item, mask, size);
 457 }
 458
 459 static int
 460 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 461                        const uint8_t *mask, unsigned int size)
 462 {
 463         if (item->mask) {
 464                 const struct rte_flow_item_tcp *mask = item->mask;
 465
 466                 if (mask->hdr.src_port != 0 &&
 467                     mask->hdr.src_port != 0xffff)
 468                         return -1;
 469                 if (mask->hdr.dst_port != 0 &&
 470                     mask->hdr.dst_port != 0xffff)
 471                         return -1;
 472         }
 473         return mlx4_flow_item_validate(item, mask, size);
 474 }
 475
 476 /** Graph of supported items and associated actions. */
 477 static const struct mlx4_flow_proc_item mlx4_flow_proc_item_list[] = {
 478         [RTE_FLOW_ITEM_TYPE_END] = {
 479                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_ETH),
 480         },
 481         [RTE_FLOW_ITEM_TYPE_ETH] = {
 482                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_VLAN,
 483                                        RTE_FLOW_ITEM_TYPE_IPV4),
 484                 .mask = &(const struct rte_flow_item_eth){
 485                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 486                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 487                 },
 488                 .default_mask = &rte_flow_item_eth_mask,
 489                 .mask_sz = sizeof(struct rte_flow_item_eth),
 490                 .validate = mlx4_flow_validate_eth,
 491                 .convert = mlx4_flow_create_eth,
 492                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 493         },
 494         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 495                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_IPV4),
 496                 .mask = &(const struct rte_flow_item_vlan){
 497                         /* Only TCI VID matching is supported. */
 498                         .tci = RTE_BE16(0x0fff),
 499                 },
 500                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 501                 .validate = mlx4_flow_validate_vlan,
 502                 .convert = mlx4_flow_create_vlan,
 503                 .dst_sz = 0,
 504         },
 505         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 506                 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_UDP,
 507                                        RTE_FLOW_ITEM_TYPE_TCP),
 508                 .mask = &(const struct rte_flow_item_ipv4){
 509                         .hdr = {
 510                                 .src_addr = RTE_BE32(0xffffffff),
 511                                 .dst_addr = RTE_BE32(0xffffffff),
 512                         },
 513                 },
 514                 .default_mask = &rte_flow_item_ipv4_mask,
 515                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 516                 .validate = mlx4_flow_validate_ipv4,
 517                 .convert = mlx4_flow_create_ipv4,
 518                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 519         },
 520         [RTE_FLOW_ITEM_TYPE_UDP] = {
 521                 .mask = &(const struct rte_flow_item_udp){
 522                         .hdr = {
 523                                 .src_port = RTE_BE16(0xffff),
 524                                 .dst_port = RTE_BE16(0xffff),
 525                         },
 526                 },
 527                 .default_mask = &rte_flow_item_udp_mask,
 528                 .mask_sz = sizeof(struct rte_flow_item_udp),
 529                 .validate = mlx4_flow_validate_udp,
 530                 .convert = mlx4_flow_create_udp,
 531                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 532         },
 533         [RTE_FLOW_ITEM_TYPE_TCP] = {
 534                 .mask = &(const struct rte_flow_item_tcp){
 535                         .hdr = {
 536                                 .src_port = RTE_BE16(0xffff),
 537                                 .dst_port = RTE_BE16(0xffff),
 538                         },
 539                 },
 540                 .default_mask = &rte_flow_item_tcp_mask,
 541                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 542                 .validate = mlx4_flow_validate_tcp,
 543                 .convert = mlx4_flow_create_tcp,
 544                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 545         },
 546 };
 547
 548 /**
 549  * Make sure a flow rule is supported and initialize associated structure.
 550  *
 551  * @param priv
 552  *   Pointer to private structure.
 553  * @param[in] attr
 554  *   Flow rule attributes.
 555  * @param[in] pattern
 556  *   Pattern specification (list terminated by the END pattern item).
 557  * @param[in] actions
 558  *   Associated actions (list terminated by the END action).
 559  * @param[out] error
 560  *   Perform verbose error reporting if not NULL.
 561  * @param[in, out] addr
 562  *   Buffer where the resulting flow rule handle pointer must be stored.
 563  *   If NULL, stop processing after validation stage.
 564  *
 565  * @return
 566  *   0 on success, a negative errno value otherwise and rte_errno is set.
 567  */
 568 static int
 569 mlx4_flow_prepare(struct priv *priv,
 570                   const struct rte_flow_attr *attr,
 571                   const struct rte_flow_item pattern[],
 572                   const struct rte_flow_action actions[],
 573                   struct rte_flow_error *error,
 574                   struct rte_flow **addr)
 575 {
 576         const struct rte_flow_item *item;
 577         const struct rte_flow_action *action;
 578         const struct mlx4_flow_proc_item *proc;
 579         struct rte_flow temp = { .ibv_attr_size = sizeof(*temp.ibv_attr) };
 580         struct rte_flow *flow = &temp;
 581         uint32_t priority_override = 0;
 582
 583         if (attr->group)
 584                 return rte_flow_error_set
 585                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 586                          NULL, "groups are not supported");
 587         if (priv->isolated)
 588                 priority_override = attr->priority;
 589         else if (attr->priority)
 590                 return rte_flow_error_set
 591                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 592                          NULL,
 593                          "priorities are not supported outside isolated mode");
 594         if (attr->priority > MLX4_FLOW_PRIORITY_LAST)
 595                 return rte_flow_error_set
 596                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 597                          NULL, "maximum priority level is "
 598                          MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST));
 599         if (attr->egress)
 600                 return rte_flow_error_set
 601                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 602                          NULL, "egress is not supported");
 603         if (!attr->ingress)
 604                 return rte_flow_error_set
 605                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 606                          NULL, "only ingress is supported");
 607 fill:
 608         proc = mlx4_flow_proc_item_list;
 609         /* Go over pattern. */
 610         for (item = pattern; item->type; ++item) {
 611                 const struct mlx4_flow_proc_item *next = NULL;
 612                 unsigned int i;
 613                 int err;
 614
 615                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 616                         continue;
 617                 /*
 618                  * The nic can support patterns with NULL eth spec only
 619                  * if eth is a single item in a rule.
 620                  */
 621                 if (!item->spec && item->type == RTE_FLOW_ITEM_TYPE_ETH) {
 622                         const struct rte_flow_item *next = item + 1;
 623
 624                         if (next->type)
 625                                 return rte_flow_error_set
 626                                         (error, ENOTSUP,
 627                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
 628                                          "the rule requires an Ethernet spec");
 629                 }
 630                 for (i = 0; proc->next_item && proc->next_item[i]; ++i) {
 631                         if (proc->next_item[i] == item->type) {
 632                                 next = &mlx4_flow_proc_item_list[item->type];
 633                                 break;
 634                         }
 635                 }
 636                 if (!next)
 637                         goto exit_item_not_supported;
 638                 proc = next;
 639                 /* Perform validation once, while handle is not allocated. */
 640                 if (flow == &temp) {
 641                         err = proc->validate(item, proc->mask, proc->mask_sz);
 642                         if (err)
 643                                 goto exit_item_not_supported;
 644                 } else if (proc->convert) {
 645                         err = proc->convert(item,
 646                                             (proc->default_mask ?
 647                                              proc->default_mask :
 648                                              proc->mask),
 649                                             flow);
 650                         if (err)
 651                                 goto exit_item_not_supported;
 652                 }
 653                 flow->ibv_attr_size += proc->dst_sz;
 654         }
 655         /* Use specified priority level when in isolated mode. */
 656         if (priv->isolated && flow != &temp)
 657                 flow->ibv_attr->priority = priority_override;
 658         /* Go over actions list. */
 659         for (action = actions; action->type; ++action) {
 660                 switch (action->type) {
 661                         const struct rte_flow_action_queue *queue;
 662
 663                 case RTE_FLOW_ACTION_TYPE_VOID:
 664                         continue;
 665                 case RTE_FLOW_ACTION_TYPE_DROP:
 666                         flow->drop = 1;
 667                         break;
 668                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 669                         queue = action->conf;
 670                         if (queue->index >= priv->dev->data->nb_rx_queues)
 671                                 goto exit_action_not_supported;
 672                         flow->queue = 1;
 673                         flow->queue_id = queue->index;
 674                         break;
 675                 default:
 676                         goto exit_action_not_supported;
 677                 }
 678         }
 679         if (!flow->queue && !flow->drop)
 680                 return rte_flow_error_set
 681                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 682                          NULL, "no valid action");
 683         /* Validation ends here. */
 684         if (!addr)
 685                 return 0;
 686         if (flow == &temp) {
 687                 /* Allocate proper handle based on collected data. */
 688                 const struct mlx4_malloc_vec vec[] = {
 689                         {
 690                                 .align = alignof(struct rte_flow),
 691                                 .size = sizeof(*flow),
 692                                 .addr = (void **)&flow,
 693                         },
 694                         {
 695                                 .align = alignof(struct ibv_flow_attr),
 696                                 .size = temp.ibv_attr_size,
 697                                 .addr = (void **)&temp.ibv_attr,
 698                         },
 699                 };
 700
 701                 if (!mlx4_zmallocv(__func__, vec, RTE_DIM(vec)))
 702                         return rte_flow_error_set
 703                                 (error, -rte_errno,
 704                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 705                                  "flow rule handle allocation failure");
 706                 /* Most fields will be updated by second pass. */
 707                 *flow = (struct rte_flow){
 708                         .ibv_attr = temp.ibv_attr,
 709                         .ibv_attr_size = sizeof(*flow->ibv_attr),
 710                 };
 711                 *flow->ibv_attr = (struct ibv_flow_attr){
 712                         .type = IBV_FLOW_ATTR_NORMAL,
 713                         .size = sizeof(*flow->ibv_attr),
 714                         .port = priv->port,
 715                 };
 716                 goto fill;
 717         }
 718         *addr = flow;
 719         return 0;
 720 exit_item_not_supported:
 721         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 722                                   item, "item not supported");
 723 exit_action_not_supported:
 724         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 725                                   action, "action not supported");
 726 }
 727
 728 /**
 729  * Validate a flow supported by the NIC.
 730  *
 731  * @see rte_flow_validate()
 732  * @see rte_flow_ops
 733  */
 734 static int
 735 mlx4_flow_validate(struct rte_eth_dev *dev,
 736                    const struct rte_flow_attr *attr,
 737                    const struct rte_flow_item pattern[],
 738                    const struct rte_flow_action actions[],
 739                    struct rte_flow_error *error)
 740 {
 741         struct priv *priv = dev->data->dev_private;
 742
 743         return mlx4_flow_prepare(priv, attr, pattern, actions, error, NULL);
 744 }
 745
 746 /**
 747  * Destroy a drop queue.
 748  *
 749  * @param priv
 750  *   Pointer to private structure.
 751  */
 752 static void
 753 mlx4_flow_destroy_drop_queue(struct priv *priv)
 754 {
 755         if (priv->flow_drop_queue) {
 756                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 757
 758                 priv->flow_drop_queue = NULL;
 759                 claim_zero(ibv_destroy_qp(fdq->qp));
 760                 claim_zero(ibv_destroy_cq(fdq->cq));
 761                 rte_free(fdq);
 762         }
 763 }
 764
 765 /**
 766  * Create a single drop queue for all drop flows.
 767  *
 768  * @param priv
 769  *   Pointer to private structure.
 770  *
 771  * @return
 772  *   0 on success, negative value otherwise.
 773  */
 774 static int
 775 mlx4_flow_create_drop_queue(struct priv *priv)
 776 {
 777         struct ibv_qp *qp;
 778         struct ibv_cq *cq;
 779         struct rte_flow_drop *fdq;
 780
 781         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 782         if (!fdq) {
 783                 ERROR("Cannot allocate memory for drop struct");
 784                 goto err;
 785         }
 786         cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 787         if (!cq) {
 788                 ERROR("Cannot create drop CQ");
 789                 goto err_create_cq;
 790         }
 791         qp = ibv_create_qp(priv->pd,
 792                            &(struct ibv_qp_init_attr){
 793                                 .send_cq = cq,
 794                                 .recv_cq = cq,
 795                                 .cap = {
 796                                         .max_recv_wr = 1,
 797                                         .max_recv_sge = 1,
 798                                 },
 799                                 .qp_type = IBV_QPT_RAW_PACKET,
 800                            });
 801         if (!qp) {
 802                 ERROR("Cannot create drop QP");
 803                 goto err_create_qp;
 804         }
 805         *fdq = (struct rte_flow_drop){
 806                 .qp = qp,
 807                 .cq = cq,
 808         };
 809         priv->flow_drop_queue = fdq;
 810         return 0;
 811 err_create_qp:
 812         claim_zero(ibv_destroy_cq(cq));
 813 err_create_cq:
 814         rte_free(fdq);
 815 err:
 816         return -1;
 817 }
 818
 819 /**
 820  * Toggle a configured flow rule.
 821  *
 822  * @param priv
 823  *   Pointer to private structure.
 824  * @param flow
 825  *   Flow rule handle to toggle.
 826  * @param enable
 827  *   Whether associated Verbs flow must be created or removed.
 828  * @param[out] error
 829  *   Perform verbose error reporting if not NULL.
 830  *
 831  * @return
 832  *   0 on success, a negative errno value otherwise and rte_errno is set.
 833  */
 834 static int
 835 mlx4_flow_toggle(struct priv *priv,
 836                  struct rte_flow *flow,
 837                  int enable,
 838                  struct rte_flow_error *error)
 839 {
 840         struct ibv_qp *qp = NULL;
 841         const char *msg;
 842         int err;
 843
 844         if (!enable) {
 845                 if (!flow->ibv_flow)
 846                         return 0;
 847                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 848                 flow->ibv_flow = NULL;
 849                 return 0;
 850         }
 851         if (flow->ibv_flow)
 852                 return 0;
 853         assert(flow->queue ^ flow->drop);
 854         if (flow->queue) {
 855                 struct rxq *rxq;
 856
 857                 assert(flow->queue_id < priv->dev->data->nb_rx_queues);
 858                 rxq = priv->dev->data->rx_queues[flow->queue_id];
 859                 if (!rxq) {
 860                         err = EINVAL;
 861                         msg = "target queue must be configured first";
 862                         goto error;
 863                 }
 864                 qp = rxq->qp;
 865         }
 866         if (flow->drop) {
 867                 assert(priv->flow_drop_queue);
 868                 qp = priv->flow_drop_queue->qp;
 869         }
 870         assert(qp);
 871         assert(flow->ibv_attr);
 872         flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
 873         if (flow->ibv_flow)
 874                 return 0;
 875         err = errno;
 876         msg = "flow rule rejected by device";
 877 error:
 878         return rte_flow_error_set
 879                 (error, err, RTE_FLOW_ERROR_TYPE_HANDLE, flow, msg);
 880 }
 881
 882 /**
 883  * Create a flow.
 884  *
 885  * @see rte_flow_create()
 886  * @see rte_flow_ops
 887  */
 888 static struct rte_flow *
 889 mlx4_flow_create(struct rte_eth_dev *dev,
 890                  const struct rte_flow_attr *attr,
 891                  const struct rte_flow_item pattern[],
 892                  const struct rte_flow_action actions[],
 893                  struct rte_flow_error *error)
 894 {
 895         struct priv *priv = dev->data->dev_private;
 896         struct rte_flow *flow;
 897         int err;
 898
 899         err = mlx4_flow_prepare(priv, attr, pattern, actions, error, &flow);
 900         if (err)
 901                 return NULL;
 902         err = mlx4_flow_toggle(priv, flow, priv->started, error);
 903         if (!err) {
 904                 LIST_INSERT_HEAD(&priv->flows, flow, next);
 905                 return flow;
 906         }
 907         rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 908                            error->message);
 909         rte_free(flow);
 910         return NULL;
 911 }
 912
 913 /**
 914  * Configure isolated mode.
 915  *
 916  * @see rte_flow_isolate()
 917  * @see rte_flow_ops
 918  */
 919 static int
 920 mlx4_flow_isolate(struct rte_eth_dev *dev,
 921                   int enable,
 922                   struct rte_flow_error *error)
 923 {
 924         struct priv *priv = dev->data->dev_private;
 925
 926         if (!!enable == !!priv->isolated)
 927                 return 0;
 928         priv->isolated = !!enable;
 929         if (enable) {
 930                 mlx4_mac_addr_del(priv);
 931         } else if (mlx4_mac_addr_add(priv) < 0) {
 932                 priv->isolated = 1;
 933                 return rte_flow_error_set(error, rte_errno,
 934                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 935                                           NULL, "cannot leave isolated mode");
 936         }
 937         return 0;
 938 }
 939
 940 /**
 941  * Destroy a flow rule.
 942  *
 943  * @see rte_flow_destroy()
 944  * @see rte_flow_ops
 945  */
 946 static int
 947 mlx4_flow_destroy(struct rte_eth_dev *dev,
 948                   struct rte_flow *flow,
 949                   struct rte_flow_error *error)
 950 {
 951         struct priv *priv = dev->data->dev_private;
 952         int err = mlx4_flow_toggle(priv, flow, 0, error);
 953
 954         if (err)
 955                 return err;
 956         LIST_REMOVE(flow, next);
 957         rte_free(flow);
 958         return 0;
 959 }
 960
 961 /**
 962  * Destroy all flow rules.
 963  *
 964  * @see rte_flow_flush()
 965  * @see rte_flow_ops
 966  */
 967 static int
 968 mlx4_flow_flush(struct rte_eth_dev *dev,
 969                 struct rte_flow_error *error)
 970 {
 971         struct priv *priv = dev->data->dev_private;
 972
 973         while (!LIST_EMPTY(&priv->flows)) {
 974                 struct rte_flow *flow;
 975
 976                 flow = LIST_FIRST(&priv->flows);
 977                 mlx4_flow_destroy(dev, flow, error);
 978         }
 979         return 0;
 980 }
 981
 982 /**
 983  * Disable flow rules.
 984  *
 985  * @param priv
 986  *   Pointer to private structure.
 987  */
 988 void
 989 mlx4_flow_stop(struct priv *priv)
 990 {
 991         struct rte_flow *flow;
 992
 993         for (flow = LIST_FIRST(&priv->flows);
 994              flow;
 995              flow = LIST_NEXT(flow, next)) {
 996                 claim_zero(mlx4_flow_toggle(priv, flow, 0, NULL));
 997         }
 998         mlx4_flow_destroy_drop_queue(priv);
 999 }
1000
1001 /**
1002  * Enable flow rules.
1003  *
1004  * @param priv
1005  *   Pointer to private structure.
1006  *
1007  * @return
1008  *   0 on success, a negative errno value otherwise and rte_errno is set.
1009  */
1010 int
1011 mlx4_flow_start(struct priv *priv)
1012 {
1013         int ret;
1014         struct rte_flow *flow;
1015
1016         ret = mlx4_flow_create_drop_queue(priv);
1017         if (ret)
1018                 return -1;
1019         for (flow = LIST_FIRST(&priv->flows);
1020              flow;
1021              flow = LIST_NEXT(flow, next)) {
1022                 ret = mlx4_flow_toggle(priv, flow, 1, NULL);
1023                 if (unlikely(ret)) {
1024                         mlx4_flow_stop(priv);
1025                         return ret;
1026                 }
1027         }
1028         return 0;
1029 }
1030
1031 static const struct rte_flow_ops mlx4_flow_ops = {
1032         .validate = mlx4_flow_validate,
1033         .create = mlx4_flow_create,
1034         .destroy = mlx4_flow_destroy,
1035         .flush = mlx4_flow_flush,
1036         .isolate = mlx4_flow_isolate,
1037 };
1038
1039 /**
1040  * Manage filter operations.
1041  *
1042  * @param dev
1043  *   Pointer to Ethernet device structure.
1044  * @param filter_type
1045  *   Filter type.
1046  * @param filter_op
1047  *   Operation to perform.
1048  * @param arg
1049  *   Pointer to operation-specific structure.
1050  *
1051  * @return
1052  *   0 on success, negative errno value otherwise and rte_errno is set.
1053  */
1054 int
1055 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1056                  enum rte_filter_type filter_type,
1057                  enum rte_filter_op filter_op,
1058                  void *arg)
1059 {
1060         switch (filter_type) {
1061         case RTE_ETH_FILTER_GENERIC:
1062                 if (filter_op != RTE_ETH_FILTER_GET)
1063                         break;
1064                 *(const void **)arg = &mlx4_flow_ops;
1065                 return 0;
1066         default:
1067                 ERROR("%p: filter type (%d) not supported",
1068                       (void *)dev, filter_type);
1069                 break;
1070         }
1071         rte_errno = ENOTSUP;
1072         return -rte_errno;
1073 }