drivers/net/mlx5/mlx5_fdir.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2015 6WIND S.A.
   5  *   Copyright 2015 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stddef.h>
  35 #include <assert.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 /* Verbs header. */
  41 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  42 #ifdef PEDANTIC
  43 #pragma GCC diagnostic ignored "-pedantic"
  44 #endif
  45 #include <infiniband/verbs.h>
  46 #ifdef PEDANTIC
  47 #pragma GCC diagnostic error "-pedantic"
  48 #endif
  49
  50 /* DPDK headers don't like -pedantic. */
  51 #ifdef PEDANTIC
  52 #pragma GCC diagnostic ignored "-pedantic"
  53 #endif
  54 #include <rte_ether.h>
  55 #include <rte_malloc.h>
  56 #include <rte_ethdev.h>
  57 #include <rte_common.h>
  58 #ifdef PEDANTIC
  59 #pragma GCC diagnostic error "-pedantic"
  60 #endif
  61
  62 #include "mlx5.h"
  63 #include "mlx5_rxtx.h"
  64
  65 struct fdir_flow_desc {
  66         uint16_t dst_port;
  67         uint16_t src_port;
  68         uint32_t src_ip[4];
  69         uint32_t dst_ip[4];
  70         uint8_t mac[6];
  71         uint16_t vlan_tag;
  72         enum hash_rxq_type type;
  73 };
  74
  75 struct mlx5_fdir_filter {
  76         LIST_ENTRY(mlx5_fdir_filter) next;
  77         uint16_t queue; /* Queue assigned to if FDIR match. */
  78         struct fdir_flow_desc desc;
  79         struct ibv_exp_flow *flow;
  80 };
  81
  82 LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
  83
  84 /**
  85  * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
  86  *
  87  * @param[in] fdir_filter
  88  *   DPDK filter structure to convert.
  89  * @param[out] desc
  90  *   Resulting mlx5 filter descriptor.
  91  * @param mode
  92  *   Flow director mode.
  93  */
  94 static void
  95 fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
  96                          struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
  97 {
  98         /* Initialize descriptor. */
  99         memset(desc, 0, sizeof(*desc));
 100
 101         /* Set VLAN ID. */
 102         desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
 103
 104         /* Set MAC address. */
 105         if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 106                 rte_memcpy(desc->mac,
 107                            fdir_filter->input.flow.mac_vlan_flow.mac_addr.
 108                                 addr_bytes,
 109                            sizeof(desc->mac));
 110                 desc->type = HASH_RXQ_ETH;
 111                 return;
 112         }
 113
 114         /* Set mode */
 115         switch (fdir_filter->input.flow_type) {
 116         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 117                 desc->type = HASH_RXQ_UDPV4;
 118                 break;
 119         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 120                 desc->type = HASH_RXQ_TCPV4;
 121                 break;
 122         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 123                 desc->type = HASH_RXQ_IPV4;
 124                 break;
 125 #ifdef HAVE_FLOW_SPEC_IPV6
 126         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 127                 desc->type = HASH_RXQ_UDPV6;
 128                 break;
 129         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 130                 desc->type = HASH_RXQ_TCPV6;
 131                 break;
 132         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 133                 desc->type = HASH_RXQ_IPV6;
 134                 break;
 135 #endif /* HAVE_FLOW_SPEC_IPV6 */
 136         default:
 137                 break;
 138         }
 139
 140         /* Set flow values */
 141         switch (fdir_filter->input.flow_type) {
 142         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 143         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 144                 desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
 145                 desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
 146         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 147                 desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
 148                 desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
 149                 break;
 150 #ifdef HAVE_FLOW_SPEC_IPV6
 151         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 152         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 153                 desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
 154                 desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
 155                 /* Fall through. */
 156         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 157                 rte_memcpy(desc->src_ip,
 158                            fdir_filter->input.flow.ipv6_flow.src_ip,
 159                            sizeof(desc->src_ip));
 160                 rte_memcpy(desc->dst_ip,
 161                            fdir_filter->input.flow.ipv6_flow.dst_ip,
 162                            sizeof(desc->dst_ip));
 163                 break;
 164 #endif /* HAVE_FLOW_SPEC_IPV6 */
 165         default:
 166                 break;
 167         }
 168 }
 169
 170 /**
 171  * Check if two flow descriptors overlap according to configured mask.
 172  *
 173  * @param priv
 174  *   Private structure that provides flow director mask.
 175  * @param desc1
 176  *   First flow descriptor to compare.
 177  * @param desc2
 178  *   Second flow descriptor to compare.
 179  *
 180  * @return
 181  *   Nonzero if descriptors overlap.
 182  */
 183 static int
 184 priv_fdir_overlap(const struct priv *priv,
 185                   const struct fdir_flow_desc *desc1,
 186                   const struct fdir_flow_desc *desc2)
 187 {
 188         const struct rte_eth_fdir_masks *mask =
 189                 &priv->dev->data->dev_conf.fdir_conf.mask;
 190         unsigned int i;
 191
 192         if (desc1->type != desc2->type)
 193                 return 0;
 194         /* Ignore non masked bits. */
 195         for (i = 0; i != RTE_DIM(desc1->mac); ++i)
 196                 if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
 197                     (desc2->mac[i] & mask->mac_addr_byte_mask))
 198                         return 0;
 199         if (((desc1->src_port & mask->src_port_mask) !=
 200              (desc2->src_port & mask->src_port_mask)) ||
 201             ((desc1->dst_port & mask->dst_port_mask) !=
 202              (desc2->dst_port & mask->dst_port_mask)))
 203                 return 0;
 204         switch (desc1->type) {
 205         case HASH_RXQ_IPV4:
 206         case HASH_RXQ_UDPV4:
 207         case HASH_RXQ_TCPV4:
 208                 if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
 209                      (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
 210                     ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
 211                      (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
 212                         return 0;
 213                 break;
 214 #ifdef HAVE_FLOW_SPEC_IPV6
 215         case HASH_RXQ_IPV6:
 216         case HASH_RXQ_UDPV6:
 217         case HASH_RXQ_TCPV6:
 218                 for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
 219                         if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
 220                              (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
 221                             ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
 222                              (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
 223                                 return 0;
 224                 break;
 225 #endif /* HAVE_FLOW_SPEC_IPV6 */
 226         default:
 227                 break;
 228         }
 229         return 1;
 230 }
 231
 232 /**
 233  * Create flow director steering rule for a specific filter.
 234  *
 235  * @param priv
 236  *   Private structure.
 237  * @param mlx5_fdir_filter
 238  *   Filter to create a steering rule for.
 239  * @param fdir_queue
 240  *   Flow director queue for matching packets.
 241  *
 242  * @return
 243  *   0 on success, errno value on failure.
 244  */
 245 static int
 246 priv_fdir_flow_add(struct priv *priv,
 247                    struct mlx5_fdir_filter *mlx5_fdir_filter,
 248                    struct fdir_queue *fdir_queue)
 249 {
 250         struct ibv_exp_flow *flow;
 251         struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
 252         enum rte_fdir_mode fdir_mode =
 253                 priv->dev->data->dev_conf.fdir_conf.mode;
 254         struct rte_eth_fdir_masks *mask =
 255                 &priv->dev->data->dev_conf.fdir_conf.mask;
 256         FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
 257         struct ibv_exp_flow_attr *attr = &data->attr;
 258         uintptr_t spec_offset = (uintptr_t)&data->spec;
 259         struct ibv_exp_flow_spec_eth *spec_eth;
 260         struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
 261 #ifdef HAVE_FLOW_SPEC_IPV6
 262         struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
 263 #endif /* HAVE_FLOW_SPEC_IPV6 */
 264         struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
 265         struct mlx5_fdir_filter *iter_fdir_filter;
 266         unsigned int i;
 267
 268         /* Abort if an existing flow overlaps this one to avoid packet
 269          * duplication, even if it targets another queue. */
 270         LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
 271                 if ((iter_fdir_filter != mlx5_fdir_filter) &&
 272                     (iter_fdir_filter->flow != NULL) &&
 273                     (priv_fdir_overlap(priv,
 274                                        &mlx5_fdir_filter->desc,
 275                                        &iter_fdir_filter->desc)))
 276                         return EEXIST;
 277
 278         /*
 279          * No padding must be inserted by the compiler between attr and spec.
 280          * This layout is expected by libibverbs.
 281          */
 282         assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
 283         priv_flow_attr(priv, attr, sizeof(data), desc->type);
 284
 285         /* Set Ethernet spec */
 286         spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
 287
 288         /* The first specification must be Ethernet. */
 289         assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
 290         assert(spec_eth->size == sizeof(*spec_eth));
 291
 292         /* VLAN ID */
 293         spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
 294         spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
 295
 296         /* Update priority */
 297         attr->priority = 2;
 298
 299         if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 300                 /* MAC Address */
 301                 for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
 302                         spec_eth->val.dst_mac[i] =
 303                                 desc->mac[i] & mask->mac_addr_byte_mask;
 304                         spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
 305                 }
 306                 goto create_flow;
 307         }
 308
 309         switch (desc->type) {
 310         case HASH_RXQ_IPV4:
 311         case HASH_RXQ_UDPV4:
 312         case HASH_RXQ_TCPV4:
 313                 spec_offset += spec_eth->size;
 314
 315                 /* Set IP spec */
 316                 spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
 317
 318                 /* The second specification must be IP. */
 319                 assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
 320                 assert(spec_ipv4->size == sizeof(*spec_ipv4));
 321
 322                 spec_ipv4->val.src_ip =
 323                         desc->src_ip[0] & mask->ipv4_mask.src_ip;
 324                 spec_ipv4->val.dst_ip =
 325                         desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
 326                 spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
 327                 spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
 328
 329                 /* Update priority */
 330                 attr->priority = 1;
 331
 332                 if (desc->type == HASH_RXQ_IPV4)
 333                         goto create_flow;
 334
 335                 spec_offset += spec_ipv4->size;
 336                 break;
 337 #ifdef HAVE_FLOW_SPEC_IPV6
 338         case HASH_RXQ_IPV6:
 339         case HASH_RXQ_UDPV6:
 340         case HASH_RXQ_TCPV6:
 341                 spec_offset += spec_eth->size;
 342
 343                 /* Set IP spec */
 344                 spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
 345
 346                 /* The second specification must be IP. */
 347                 assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
 348                 assert(spec_ipv6->size == sizeof(*spec_ipv6));
 349
 350                 for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
 351                         ((uint32_t *)spec_ipv6->val.src_ip)[i] =
 352                                 desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
 353                         ((uint32_t *)spec_ipv6->val.dst_ip)[i] =
 354                                 desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
 355                 }
 356                 rte_memcpy(spec_ipv6->mask.src_ip,
 357                            mask->ipv6_mask.src_ip,
 358                            sizeof(spec_ipv6->mask.src_ip));
 359                 rte_memcpy(spec_ipv6->mask.dst_ip,
 360                            mask->ipv6_mask.dst_ip,
 361                            sizeof(spec_ipv6->mask.dst_ip));
 362
 363                 /* Update priority */
 364                 attr->priority = 1;
 365
 366                 if (desc->type == HASH_RXQ_IPV6)
 367                         goto create_flow;
 368
 369                 spec_offset += spec_ipv6->size;
 370                 break;
 371 #endif /* HAVE_FLOW_SPEC_IPV6 */
 372         default:
 373                 ERROR("invalid flow attribute type");
 374                 return EINVAL;
 375         }
 376
 377         /* Set TCP/UDP flow specification. */
 378         spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
 379
 380         /* The third specification must be TCP/UDP. */
 381         assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
 382                spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
 383         assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
 384
 385         spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
 386         spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
 387         spec_tcp_udp->mask.src_port = mask->src_port_mask;
 388         spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
 389
 390         /* Update priority */
 391         attr->priority = 0;
 392
 393 create_flow:
 394
 395         errno = 0;
 396         flow = ibv_exp_create_flow(fdir_queue->qp, attr);
 397         if (flow == NULL) {
 398                 /* It's not clear whether errno is always set in this case. */
 399                 ERROR("%p: flow director configuration failed, errno=%d: %s",
 400                       (void *)priv, errno,
 401                       (errno ? strerror(errno) : "Unknown error"));
 402                 if (errno)
 403                         return errno;
 404                 return EINVAL;
 405         }
 406
 407         DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
 408         mlx5_fdir_filter->flow = flow;
 409         return 0;
 410 }
 411
 412 /**
 413  * Get flow director queue for a specific RX queue, create it in case
 414  * it does not exist.
 415  *
 416  * @param priv
 417  *   Private structure.
 418  * @param idx
 419  *   RX queue index.
 420  *
 421  * @return
 422  *   Related flow director queue on success, NULL otherwise.
 423  */
 424 static struct fdir_queue *
 425 priv_get_fdir_queue(struct priv *priv, uint16_t idx)
 426 {
 427         struct fdir_queue *fdir_queue = &(*priv->rxqs)[idx]->fdir_queue;
 428         struct ibv_exp_rwq_ind_table *ind_table = NULL;
 429         struct ibv_qp *qp = NULL;
 430         struct ibv_exp_rwq_ind_table_init_attr ind_init_attr;
 431         struct ibv_exp_rx_hash_conf hash_conf;
 432         struct ibv_exp_qp_init_attr qp_init_attr;
 433         int err = 0;
 434
 435         /* Return immediately if it has already been created. */
 436         if (fdir_queue->qp != NULL)
 437                 return fdir_queue;
 438
 439         ind_init_attr = (struct ibv_exp_rwq_ind_table_init_attr){
 440                 .pd = priv->pd,
 441                 .log_ind_tbl_size = 0,
 442                 .ind_tbl = &((*priv->rxqs)[idx]->wq),
 443                 .comp_mask = 0,
 444         };
 445
 446         errno = 0;
 447         ind_table = ibv_exp_create_rwq_ind_table(priv->ctx,
 448                                                  &ind_init_attr);
 449         if (ind_table == NULL) {
 450                 /* Not clear whether errno is set. */
 451                 err = (errno ? errno : EINVAL);
 452                 ERROR("RX indirection table creation failed with error %d: %s",
 453                       err, strerror(err));
 454                 goto error;
 455         }
 456
 457         /* Create fdir_queue qp. */
 458         hash_conf = (struct ibv_exp_rx_hash_conf){
 459                 .rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
 460                 .rx_hash_key_len = rss_hash_default_key_len,
 461                 .rx_hash_key = rss_hash_default_key,
 462                 .rx_hash_fields_mask = 0,
 463                 .rwq_ind_tbl = ind_table,
 464         };
 465         qp_init_attr = (struct ibv_exp_qp_init_attr){
 466                 .max_inl_recv = 0, /* Currently not supported. */
 467                 .qp_type = IBV_QPT_RAW_PACKET,
 468                 .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
 469                               IBV_EXP_QP_INIT_ATTR_RX_HASH),
 470                 .pd = priv->pd,
 471                 .rx_hash_conf = &hash_conf,
 472                 .port_num = priv->port,
 473         };
 474
 475         qp = ibv_exp_create_qp(priv->ctx, &qp_init_attr);
 476         if (qp == NULL) {
 477                 err = (errno ? errno : EINVAL);
 478                 ERROR("hash RX QP creation failure: %s", strerror(err));
 479                 goto error;
 480         }
 481
 482         fdir_queue->ind_table = ind_table;
 483         fdir_queue->qp = qp;
 484
 485         return fdir_queue;
 486
 487 error:
 488         if (qp != NULL)
 489                 claim_zero(ibv_destroy_qp(qp));
 490
 491         if (ind_table != NULL)
 492                 claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table));
 493
 494         return NULL;
 495 }
 496
 497 /**
 498  * Enable flow director filter and create steering rules.
 499  *
 500  * @param priv
 501  *   Private structure.
 502  * @param mlx5_fdir_filter
 503  *   Filter to create steering rule for.
 504  *
 505  * @return
 506  *   0 on success, errno value on failure.
 507  */
 508 static int
 509 priv_fdir_filter_enable(struct priv *priv,
 510                         struct mlx5_fdir_filter *mlx5_fdir_filter)
 511 {
 512         struct fdir_queue *fdir_queue;
 513
 514         /* Check if flow already exists. */
 515         if (mlx5_fdir_filter->flow != NULL)
 516                 return 0;
 517
 518         /* Get fdir_queue for specific queue. */
 519         fdir_queue = priv_get_fdir_queue(priv, mlx5_fdir_filter->queue);
 520
 521         if (fdir_queue == NULL) {
 522                 ERROR("failed to create flow director rxq for queue %d",
 523                       mlx5_fdir_filter->queue);
 524                 return EINVAL;
 525         }
 526
 527         /* Create flow */
 528         return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
 529 }
 530
 531 /**
 532  * Initialize flow director filters list.
 533  *
 534  * @param priv
 535  *   Private structure.
 536  *
 537  * @return
 538  *   0 on success, errno value on failure.
 539  */
 540 int
 541 fdir_init_filters_list(struct priv *priv)
 542 {
 543         /* Filter list initialization should be done only once. */
 544         if (priv->fdir_filter_list)
 545                 return 0;
 546
 547         /* Create filters list. */
 548         priv->fdir_filter_list =
 549                 rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
 550
 551         if (priv->fdir_filter_list == NULL) {
 552                 int err = ENOMEM;
 553
 554                 ERROR("cannot allocate flow director filter list: %s",
 555                       strerror(err));
 556                 return err;
 557         }
 558
 559         LIST_INIT(priv->fdir_filter_list);
 560
 561         return 0;
 562 }
 563
 564 /**
 565  * Flush all filters.
 566  *
 567  * @param priv
 568  *   Private structure.
 569  */
 570 static void
 571 priv_fdir_filter_flush(struct priv *priv)
 572 {
 573         struct mlx5_fdir_filter *mlx5_fdir_filter;
 574
 575         while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
 576                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 577
 578                 DEBUG("%p: flushing flow director filter %p",
 579                       (void *)priv, (void *)mlx5_fdir_filter);
 580                 LIST_REMOVE(mlx5_fdir_filter, next);
 581                 if (flow != NULL)
 582                         claim_zero(ibv_exp_destroy_flow(flow));
 583                 rte_free(mlx5_fdir_filter);
 584         }
 585 }
 586
 587 /**
 588  * Remove all flow director filters and delete list.
 589  *
 590  * @param priv
 591  *   Private structure.
 592  */
 593 void
 594 priv_fdir_delete_filters_list(struct priv *priv)
 595 {
 596         priv_fdir_filter_flush(priv);
 597         rte_free(priv->fdir_filter_list);
 598         priv->fdir_filter_list = NULL;
 599 }
 600
 601 /**
 602  * Disable flow director, remove all steering rules.
 603  *
 604  * @param priv
 605  *   Private structure.
 606  */
 607 void
 608 priv_fdir_disable(struct priv *priv)
 609 {
 610         unsigned int i;
 611         struct mlx5_fdir_filter *mlx5_fdir_filter;
 612         struct fdir_queue *fdir_queue;
 613
 614         /* Run on every flow director filter and destroy flow handle. */
 615         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 616                 struct ibv_exp_flow *flow;
 617
 618                 /* Only valid elements should be in the list */
 619                 assert(mlx5_fdir_filter != NULL);
 620                 flow = mlx5_fdir_filter->flow;
 621
 622                 /* Destroy flow handle */
 623                 if (flow != NULL) {
 624                         claim_zero(ibv_exp_destroy_flow(flow));
 625                         mlx5_fdir_filter->flow = NULL;
 626                 }
 627         }
 628
 629         /* Run on every RX queue to destroy related flow director QP and
 630          * indirection table. */
 631         for (i = 0; (i != priv->rxqs_n); i++) {
 632                 fdir_queue = &(*priv->rxqs)[i]->fdir_queue;
 633
 634                 if (fdir_queue->qp != NULL) {
 635                         claim_zero(ibv_destroy_qp(fdir_queue->qp));
 636                         fdir_queue->qp = NULL;
 637                 }
 638
 639                 if (fdir_queue->ind_table != NULL) {
 640                         claim_zero(ibv_exp_destroy_rwq_ind_table
 641                                    (fdir_queue->ind_table));
 642                         fdir_queue->ind_table = NULL;
 643                 }
 644         }
 645 }
 646
 647 /**
 648  * Enable flow director, create steering rules.
 649  *
 650  * @param priv
 651  *   Private structure.
 652  */
 653 void
 654 priv_fdir_enable(struct priv *priv)
 655 {
 656         struct mlx5_fdir_filter *mlx5_fdir_filter;
 657
 658         /* Run on every fdir filter and create flow handle */
 659         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 660                 /* Only valid elements should be in the list */
 661                 assert(mlx5_fdir_filter != NULL);
 662
 663                 priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 664         }
 665 }
 666
 667 /**
 668  * Find specific filter in list.
 669  *
 670  * @param priv
 671  *   Private structure.
 672  * @param fdir_filter
 673  *   Flow director filter to find.
 674  *
 675  * @return
 676  *   Filter element if found, otherwise NULL.
 677  */
 678 static struct mlx5_fdir_filter *
 679 priv_find_filter_in_list(struct priv *priv,
 680                          const struct rte_eth_fdir_filter *fdir_filter)
 681 {
 682         struct fdir_flow_desc desc;
 683         struct mlx5_fdir_filter *mlx5_fdir_filter;
 684         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 685
 686         /* Get flow director filter to look for. */
 687         fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
 688
 689         /* Look for the requested element. */
 690         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 691                 /* Only valid elements should be in the list. */
 692                 assert(mlx5_fdir_filter != NULL);
 693
 694                 /* Return matching filter. */
 695                 if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
 696                         return mlx5_fdir_filter;
 697         }
 698
 699         /* Filter not found */
 700         return NULL;
 701 }
 702
 703 /**
 704  * Add new flow director filter and store it in list.
 705  *
 706  * @param priv
 707  *   Private structure.
 708  * @param fdir_filter
 709  *   Flow director filter to add.
 710  *
 711  * @return
 712  *   0 on success, errno value on failure.
 713  */
 714 static int
 715 priv_fdir_filter_add(struct priv *priv,
 716                      const struct rte_eth_fdir_filter *fdir_filter)
 717 {
 718         struct mlx5_fdir_filter *mlx5_fdir_filter;
 719         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 720         int err = 0;
 721
 722         /* Validate queue number. */
 723         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 724                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 725                 return EINVAL;
 726         }
 727
 728         /* Duplicate filters are currently unsupported. */
 729         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 730         if (mlx5_fdir_filter != NULL) {
 731                 ERROR("filter already exists");
 732                 return EINVAL;
 733         }
 734
 735         /* Create new flow director filter. */
 736         mlx5_fdir_filter =
 737                 rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
 738         if (mlx5_fdir_filter == NULL) {
 739                 err = ENOMEM;
 740                 ERROR("cannot allocate flow director filter: %s",
 741                       strerror(err));
 742                 return err;
 743         }
 744
 745         /* Set queue. */
 746         mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 747
 748         /* Convert to mlx5 filter descriptor. */
 749         fdir_filter_to_flow_desc(fdir_filter,
 750                                  &mlx5_fdir_filter->desc, fdir_mode);
 751
 752         /* Insert new filter into list. */
 753         LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
 754
 755         DEBUG("%p: flow director filter %p added",
 756               (void *)priv, (void *)mlx5_fdir_filter);
 757
 758         /* Enable filter immediately if device is started. */
 759         if (priv->started)
 760                 err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 761
 762         return err;
 763 }
 764
 765 /**
 766  * Update queue for specific filter.
 767  *
 768  * @param priv
 769  *   Private structure.
 770  * @param fdir_filter
 771  *   Filter to be updated.
 772  *
 773  * @return
 774  *   0 on success, errno value on failure.
 775  */
 776 static int
 777 priv_fdir_filter_update(struct priv *priv,
 778                         const struct rte_eth_fdir_filter *fdir_filter)
 779 {
 780         struct mlx5_fdir_filter *mlx5_fdir_filter;
 781
 782         /* Validate queue number. */
 783         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 784                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 785                 return EINVAL;
 786         }
 787
 788         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 789         if (mlx5_fdir_filter != NULL) {
 790                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 791                 int err = 0;
 792
 793                 /* Update queue number. */
 794                 mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 795
 796                 /* Destroy flow handle. */
 797                 if (flow != NULL) {
 798                         claim_zero(ibv_exp_destroy_flow(flow));
 799                         mlx5_fdir_filter->flow = NULL;
 800                 }
 801                 DEBUG("%p: flow director filter %p updated",
 802                       (void *)priv, (void *)mlx5_fdir_filter);
 803
 804                 /* Enable filter if device is started. */
 805                 if (priv->started)
 806                         err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 807
 808                 return err;
 809         }
 810
 811         /* Filter not found, create it. */
 812         DEBUG("%p: filter not found for update, creating new filter",
 813               (void *)priv);
 814         return priv_fdir_filter_add(priv, fdir_filter);
 815 }
 816
 817 /**
 818  * Delete specific filter.
 819  *
 820  * @param priv
 821  *   Private structure.
 822  * @param fdir_filter
 823  *   Filter to be deleted.
 824  *
 825  * @return
 826  *   0 on success, errno value on failure.
 827  */
 828 static int
 829 priv_fdir_filter_delete(struct priv *priv,
 830                         const struct rte_eth_fdir_filter *fdir_filter)
 831 {
 832         struct mlx5_fdir_filter *mlx5_fdir_filter;
 833
 834         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 835         if (mlx5_fdir_filter != NULL) {
 836                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 837
 838                 /* Remove element from list. */
 839                 LIST_REMOVE(mlx5_fdir_filter, next);
 840
 841                 /* Destroy flow handle. */
 842                 if (flow != NULL) {
 843                         claim_zero(ibv_exp_destroy_flow(flow));
 844                         mlx5_fdir_filter->flow = NULL;
 845                 }
 846
 847                 DEBUG("%p: flow director filter %p deleted",
 848                       (void *)priv, (void *)mlx5_fdir_filter);
 849
 850                 /* Delete filter. */
 851                 rte_free(mlx5_fdir_filter);
 852
 853                 return 0;
 854         }
 855
 856         ERROR("%p: flow director delete failed, cannot find filter",
 857               (void *)priv);
 858         return EINVAL;
 859 }
 860
 861 /**
 862  * Get flow director information.
 863  *
 864  * @param priv
 865  *   Private structure.
 866  * @param[out] fdir_info
 867  *   Resulting flow director information.
 868  */
 869 static void
 870 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
 871 {
 872         struct rte_eth_fdir_masks *mask =
 873                 &priv->dev->data->dev_conf.fdir_conf.mask;
 874
 875         fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
 876         fdir_info->guarant_spc = 0;
 877
 878         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
 879
 880         fdir_info->max_flexpayload = 0;
 881         fdir_info->flow_types_mask[0] = 0;
 882
 883         fdir_info->flex_payload_unit = 0;
 884         fdir_info->max_flex_payload_segment_num = 0;
 885         fdir_info->flex_payload_limit = 0;
 886         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
 887 }
 888
 889 /**
 890  * Deal with flow director operations.
 891  *
 892  * @param priv
 893  *   Pointer to private structure.
 894  * @param filter_op
 895  *   Operation to perform.
 896  * @param arg
 897  *   Pointer to operation-specific structure.
 898  *
 899  * @return
 900  *   0 on success, errno value on failure.
 901  */
 902 static int
 903 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
 904 {
 905         enum rte_fdir_mode fdir_mode =
 906                 priv->dev->data->dev_conf.fdir_conf.mode;
 907         int ret = 0;
 908
 909         if (filter_op == RTE_ETH_FILTER_NOP)
 910                 return 0;
 911
 912         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
 913             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 914                 ERROR("%p: flow director mode %d not supported",
 915                       (void *)priv, fdir_mode);
 916                 return EINVAL;
 917         }
 918
 919         switch (filter_op) {
 920         case RTE_ETH_FILTER_ADD:
 921                 ret = priv_fdir_filter_add(priv, arg);
 922                 break;
 923         case RTE_ETH_FILTER_UPDATE:
 924                 ret = priv_fdir_filter_update(priv, arg);
 925                 break;
 926         case RTE_ETH_FILTER_DELETE:
 927                 ret = priv_fdir_filter_delete(priv, arg);
 928                 break;
 929         case RTE_ETH_FILTER_FLUSH:
 930                 priv_fdir_filter_flush(priv);
 931                 break;
 932         case RTE_ETH_FILTER_INFO:
 933                 priv_fdir_info_get(priv, arg);
 934                 break;
 935         default:
 936                 DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
 937                 ret = EINVAL;
 938                 break;
 939         }
 940         return ret;
 941 }
 942
 943 /**
 944  * Manage filter operations.
 945  *
 946  * @param dev
 947  *   Pointer to Ethernet device structure.
 948  * @param filter_type
 949  *   Filter type.
 950  * @param filter_op
 951  *   Operation to perform.
 952  * @param arg
 953  *   Pointer to operation-specific structure.
 954  *
 955  * @return
 956  *   0 on success, negative errno value on failure.
 957  */
 958 int
 959 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 960                      enum rte_filter_type filter_type,
 961                      enum rte_filter_op filter_op,
 962                      void *arg)
 963 {
 964         int ret = -EINVAL;
 965         struct priv *priv = dev->data->dev_private;
 966
 967         switch (filter_type) {
 968         case RTE_ETH_FILTER_FDIR:
 969                 priv_lock(priv);
 970                 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
 971                 priv_unlock(priv);
 972                 break;
 973         default:
 974                 ERROR("%p: filter type (%d) not supported",
 975                       (void *)dev, filter_type);
 976                 break;
 977         }
 978
 979         return ret;
 980 }