drivers/net/mlx5/mlx5_fdir.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2015 6WIND S.A.
   5  *   Copyright 2015 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stddef.h>
  35 #include <assert.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 /* Verbs header. */
  41 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  42 #ifdef PEDANTIC
  43 #pragma GCC diagnostic ignored "-Wpedantic"
  44 #endif
  45 #include <infiniband/verbs.h>
  46 #ifdef PEDANTIC
  47 #pragma GCC diagnostic error "-Wpedantic"
  48 #endif
  49
  50 /* DPDK headers don't like -pedantic. */
  51 #ifdef PEDANTIC
  52 #pragma GCC diagnostic ignored "-Wpedantic"
  53 #endif
  54 #include <rte_ether.h>
  55 #include <rte_malloc.h>
  56 #include <rte_ethdev.h>
  57 #include <rte_common.h>
  58 #ifdef PEDANTIC
  59 #pragma GCC diagnostic error "-Wpedantic"
  60 #endif
  61
  62 #include "mlx5.h"
  63 #include "mlx5_rxtx.h"
  64
  65 struct fdir_flow_desc {
  66         uint16_t dst_port;
  67         uint16_t src_port;
  68         uint32_t src_ip[4];
  69         uint32_t dst_ip[4];
  70         uint8_t mac[6];
  71         uint16_t vlan_tag;
  72         enum hash_rxq_type type;
  73 };
  74
  75 struct mlx5_fdir_filter {
  76         LIST_ENTRY(mlx5_fdir_filter) next;
  77         uint16_t queue; /* Queue assigned to if FDIR match. */
  78         enum rte_eth_fdir_behavior behavior;
  79         struct fdir_flow_desc desc;
  80         struct ibv_exp_flow *flow;
  81 };
  82
  83 LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
  84
  85 /**
  86  * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
  87  *
  88  * @param[in] fdir_filter
  89  *   DPDK filter structure to convert.
  90  * @param[out] desc
  91  *   Resulting mlx5 filter descriptor.
  92  * @param mode
  93  *   Flow director mode.
  94  */
  95 static void
  96 fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
  97                          struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
  98 {
  99         /* Initialize descriptor. */
 100         memset(desc, 0, sizeof(*desc));
 101
 102         /* Set VLAN ID. */
 103         desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
 104
 105         /* Set MAC address. */
 106         if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 107                 rte_memcpy(desc->mac,
 108                            fdir_filter->input.flow.mac_vlan_flow.mac_addr.
 109                                 addr_bytes,
 110                            sizeof(desc->mac));
 111                 desc->type = HASH_RXQ_ETH;
 112                 return;
 113         }
 114
 115         /* Set mode */
 116         switch (fdir_filter->input.flow_type) {
 117         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 118                 desc->type = HASH_RXQ_UDPV4;
 119                 break;
 120         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 121                 desc->type = HASH_RXQ_TCPV4;
 122                 break;
 123         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 124                 desc->type = HASH_RXQ_IPV4;
 125                 break;
 126         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 127                 desc->type = HASH_RXQ_UDPV6;
 128                 break;
 129         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 130                 desc->type = HASH_RXQ_TCPV6;
 131                 break;
 132         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 133                 desc->type = HASH_RXQ_IPV6;
 134                 break;
 135         default:
 136                 break;
 137         }
 138
 139         /* Set flow values */
 140         switch (fdir_filter->input.flow_type) {
 141         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 142         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 143                 desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
 144                 desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
 145         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 146                 desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
 147                 desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
 148                 break;
 149         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 150         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 151                 desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
 152                 desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
 153                 /* Fall through. */
 154         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 155                 rte_memcpy(desc->src_ip,
 156                            fdir_filter->input.flow.ipv6_flow.src_ip,
 157                            sizeof(desc->src_ip));
 158                 rte_memcpy(desc->dst_ip,
 159                            fdir_filter->input.flow.ipv6_flow.dst_ip,
 160                            sizeof(desc->dst_ip));
 161                 break;
 162         default:
 163                 break;
 164         }
 165 }
 166
 167 /**
 168  * Check if two flow descriptors overlap according to configured mask.
 169  *
 170  * @param priv
 171  *   Private structure that provides flow director mask.
 172  * @param desc1
 173  *   First flow descriptor to compare.
 174  * @param desc2
 175  *   Second flow descriptor to compare.
 176  *
 177  * @return
 178  *   Nonzero if descriptors overlap.
 179  */
 180 static int
 181 priv_fdir_overlap(const struct priv *priv,
 182                   const struct fdir_flow_desc *desc1,
 183                   const struct fdir_flow_desc *desc2)
 184 {
 185         const struct rte_eth_fdir_masks *mask =
 186                 &priv->dev->data->dev_conf.fdir_conf.mask;
 187         unsigned int i;
 188
 189         if (desc1->type != desc2->type)
 190                 return 0;
 191         /* Ignore non masked bits. */
 192         for (i = 0; i != RTE_DIM(desc1->mac); ++i)
 193                 if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
 194                     (desc2->mac[i] & mask->mac_addr_byte_mask))
 195                         return 0;
 196         if (((desc1->src_port & mask->src_port_mask) !=
 197              (desc2->src_port & mask->src_port_mask)) ||
 198             ((desc1->dst_port & mask->dst_port_mask) !=
 199              (desc2->dst_port & mask->dst_port_mask)))
 200                 return 0;
 201         switch (desc1->type) {
 202         case HASH_RXQ_IPV4:
 203         case HASH_RXQ_UDPV4:
 204         case HASH_RXQ_TCPV4:
 205                 if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
 206                      (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
 207                     ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
 208                      (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
 209                         return 0;
 210                 break;
 211         case HASH_RXQ_IPV6:
 212         case HASH_RXQ_UDPV6:
 213         case HASH_RXQ_TCPV6:
 214                 for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
 215                         if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
 216                              (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
 217                             ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
 218                              (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
 219                                 return 0;
 220                 break;
 221         default:
 222                 break;
 223         }
 224         return 1;
 225 }
 226
 227 /**
 228  * Create flow director steering rule for a specific filter.
 229  *
 230  * @param priv
 231  *   Private structure.
 232  * @param mlx5_fdir_filter
 233  *   Filter to create a steering rule for.
 234  * @param fdir_queue
 235  *   Flow director queue for matching packets.
 236  *
 237  * @return
 238  *   0 on success, errno value on failure.
 239  */
 240 static int
 241 priv_fdir_flow_add(struct priv *priv,
 242                    struct mlx5_fdir_filter *mlx5_fdir_filter,
 243                    struct fdir_queue *fdir_queue)
 244 {
 245         struct ibv_exp_flow *flow;
 246         struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
 247         enum rte_fdir_mode fdir_mode =
 248                 priv->dev->data->dev_conf.fdir_conf.mode;
 249         struct rte_eth_fdir_masks *mask =
 250                 &priv->dev->data->dev_conf.fdir_conf.mask;
 251         FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
 252         struct ibv_exp_flow_attr *attr = &data->attr;
 253         uintptr_t spec_offset = (uintptr_t)&data->spec;
 254         struct ibv_exp_flow_spec_eth *spec_eth;
 255         struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
 256         struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
 257         struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
 258         struct mlx5_fdir_filter *iter_fdir_filter;
 259         unsigned int i;
 260
 261         /* Abort if an existing flow overlaps this one to avoid packet
 262          * duplication, even if it targets another queue. */
 263         LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
 264                 if ((iter_fdir_filter != mlx5_fdir_filter) &&
 265                     (iter_fdir_filter->flow != NULL) &&
 266                     (priv_fdir_overlap(priv,
 267                                        &mlx5_fdir_filter->desc,
 268                                        &iter_fdir_filter->desc)))
 269                         return EEXIST;
 270
 271         /*
 272          * No padding must be inserted by the compiler between attr and spec.
 273          * This layout is expected by libibverbs.
 274          */
 275         assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
 276         priv_flow_attr(priv, attr, sizeof(data), desc->type);
 277
 278         /* Set Ethernet spec */
 279         spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
 280
 281         /* The first specification must be Ethernet. */
 282         assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
 283         assert(spec_eth->size == sizeof(*spec_eth));
 284
 285         /* VLAN ID */
 286         spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
 287         spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
 288
 289         /* Update priority */
 290         attr->priority = 2;
 291
 292         if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 293                 /* MAC Address */
 294                 for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
 295                         spec_eth->val.dst_mac[i] =
 296                                 desc->mac[i] & mask->mac_addr_byte_mask;
 297                         spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
 298                 }
 299                 goto create_flow;
 300         }
 301
 302         switch (desc->type) {
 303         case HASH_RXQ_IPV4:
 304         case HASH_RXQ_UDPV4:
 305         case HASH_RXQ_TCPV4:
 306                 spec_offset += spec_eth->size;
 307
 308                 /* Set IP spec */
 309                 spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
 310
 311                 /* The second specification must be IP. */
 312                 assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
 313                 assert(spec_ipv4->size == sizeof(*spec_ipv4));
 314
 315                 spec_ipv4->val.src_ip =
 316                         desc->src_ip[0] & mask->ipv4_mask.src_ip;
 317                 spec_ipv4->val.dst_ip =
 318                         desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
 319                 spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
 320                 spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
 321
 322                 /* Update priority */
 323                 attr->priority = 1;
 324
 325                 if (desc->type == HASH_RXQ_IPV4)
 326                         goto create_flow;
 327
 328                 spec_offset += spec_ipv4->size;
 329                 break;
 330         case HASH_RXQ_IPV6:
 331         case HASH_RXQ_UDPV6:
 332         case HASH_RXQ_TCPV6:
 333                 spec_offset += spec_eth->size;
 334
 335                 /* Set IP spec */
 336                 spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
 337
 338                 /* The second specification must be IP. */
 339                 assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
 340                 assert(spec_ipv6->size == sizeof(*spec_ipv6));
 341
 342                 for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
 343                         ((uint32_t *)spec_ipv6->val.src_ip)[i] =
 344                                 desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
 345                         ((uint32_t *)spec_ipv6->val.dst_ip)[i] =
 346                                 desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
 347                 }
 348                 rte_memcpy(spec_ipv6->mask.src_ip,
 349                            mask->ipv6_mask.src_ip,
 350                            sizeof(spec_ipv6->mask.src_ip));
 351                 rte_memcpy(spec_ipv6->mask.dst_ip,
 352                            mask->ipv6_mask.dst_ip,
 353                            sizeof(spec_ipv6->mask.dst_ip));
 354
 355                 /* Update priority */
 356                 attr->priority = 1;
 357
 358                 if (desc->type == HASH_RXQ_IPV6)
 359                         goto create_flow;
 360
 361                 spec_offset += spec_ipv6->size;
 362                 break;
 363         default:
 364                 ERROR("invalid flow attribute type");
 365                 return EINVAL;
 366         }
 367
 368         /* Set TCP/UDP flow specification. */
 369         spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
 370
 371         /* The third specification must be TCP/UDP. */
 372         assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
 373                spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
 374         assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
 375
 376         spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
 377         spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
 378         spec_tcp_udp->mask.src_port = mask->src_port_mask;
 379         spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
 380
 381         /* Update priority */
 382         attr->priority = 0;
 383
 384 create_flow:
 385
 386         errno = 0;
 387         flow = ibv_exp_create_flow(fdir_queue->qp, attr);
 388         if (flow == NULL) {
 389                 /* It's not clear whether errno is always set in this case. */
 390                 ERROR("%p: flow director configuration failed, errno=%d: %s",
 391                       (void *)priv, errno,
 392                       (errno ? strerror(errno) : "Unknown error"));
 393                 if (errno)
 394                         return errno;
 395                 return EINVAL;
 396         }
 397
 398         DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
 399         mlx5_fdir_filter->flow = flow;
 400         return 0;
 401 }
 402
 403 /**
 404  * Destroy a flow director queue.
 405  *
 406  * @param fdir_queue
 407  *   Flow director queue to be destroyed.
 408  */
 409 void
 410 priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
 411 {
 412         struct mlx5_fdir_filter *fdir_filter;
 413
 414         /* Disable filter flows still applying to this queue. */
 415         LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) {
 416                 unsigned int idx = fdir_filter->queue;
 417                 struct rxq_ctrl *rxq_ctrl =
 418                         container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 419
 420                 assert(idx < priv->rxqs_n);
 421                 if (fdir_queue == rxq_ctrl->fdir_queue &&
 422                     fdir_filter->flow != NULL) {
 423                         claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
 424                         fdir_filter->flow = NULL;
 425                 }
 426         }
 427         assert(fdir_queue->qp);
 428         claim_zero(ibv_destroy_qp(fdir_queue->qp));
 429         assert(fdir_queue->ind_table);
 430         claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
 431         if (fdir_queue->wq)
 432                 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
 433         if (fdir_queue->cq)
 434                 claim_zero(ibv_destroy_cq(fdir_queue->cq));
 435 #ifndef NDEBUG
 436         memset(fdir_queue, 0x2a, sizeof(*fdir_queue));
 437 #endif
 438         rte_free(fdir_queue);
 439 }
 440
 441 /**
 442  * Create a flow director queue.
 443  *
 444  * @param priv
 445  *   Private structure.
 446  * @param wq
 447  *   Work queue to route matched packets to, NULL if one needs to
 448  *   be created.
 449  *
 450  * @return
 451  *   Related flow director queue on success, NULL otherwise.
 452  */
 453 static struct fdir_queue *
 454 priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
 455                        unsigned int socket)
 456 {
 457         struct fdir_queue *fdir_queue;
 458
 459         fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue),
 460                                        0, socket);
 461         if (!fdir_queue) {
 462                 ERROR("cannot allocate flow director queue");
 463                 return NULL;
 464         }
 465         assert(priv->pd);
 466         assert(priv->ctx);
 467         if (!wq) {
 468                 fdir_queue->cq = ibv_exp_create_cq(
 469                         priv->ctx, 1, NULL, NULL, 0,
 470                         &(struct ibv_exp_cq_init_attr){
 471                                 .comp_mask = 0,
 472                         });
 473                 if (!fdir_queue->cq) {
 474                         ERROR("cannot create flow director CQ");
 475                         goto error;
 476                 }
 477                 fdir_queue->wq = ibv_exp_create_wq(
 478                         priv->ctx,
 479                         &(struct ibv_exp_wq_init_attr){
 480                                 .wq_type = IBV_EXP_WQT_RQ,
 481                                 .max_recv_wr = 1,
 482                                 .max_recv_sge = 1,
 483                                 .pd = priv->pd,
 484                                 .cq = fdir_queue->cq,
 485                         });
 486                 if (!fdir_queue->wq) {
 487                         ERROR("cannot create flow director WQ");
 488                         goto error;
 489                 }
 490                 wq = fdir_queue->wq;
 491         }
 492         fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
 493                 priv->ctx,
 494                 &(struct ibv_exp_rwq_ind_table_init_attr){
 495                         .pd = priv->pd,
 496                         .log_ind_tbl_size = 0,
 497                         .ind_tbl = &wq,
 498                         .comp_mask = 0,
 499                 });
 500         if (!fdir_queue->ind_table) {
 501                 ERROR("cannot create flow director indirection table");
 502                 goto error;
 503         }
 504         fdir_queue->qp = ibv_exp_create_qp(
 505                 priv->ctx,
 506                 &(struct ibv_exp_qp_init_attr){
 507                         .qp_type = IBV_QPT_RAW_PACKET,
 508                         .comp_mask =
 509                                 IBV_EXP_QP_INIT_ATTR_PD |
 510                                 IBV_EXP_QP_INIT_ATTR_PORT |
 511                                 IBV_EXP_QP_INIT_ATTR_RX_HASH,
 512                         .pd = priv->pd,
 513                         .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
 514                                 .rx_hash_function =
 515                                         IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
 516                                 .rx_hash_key_len = rss_hash_default_key_len,
 517                                 .rx_hash_key = rss_hash_default_key,
 518                                 .rx_hash_fields_mask = 0,
 519                                 .rwq_ind_tbl = fdir_queue->ind_table,
 520                         },
 521                         .port_num = priv->port,
 522                 });
 523         if (!fdir_queue->qp) {
 524                 ERROR("cannot create flow director hash RX QP");
 525                 goto error;
 526         }
 527         return fdir_queue;
 528 error:
 529         assert(fdir_queue);
 530         assert(!fdir_queue->qp);
 531         if (fdir_queue->ind_table)
 532                 claim_zero(ibv_exp_destroy_rwq_ind_table
 533                            (fdir_queue->ind_table));
 534         if (fdir_queue->wq)
 535                 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
 536         if (fdir_queue->cq)
 537                 claim_zero(ibv_destroy_cq(fdir_queue->cq));
 538         rte_free(fdir_queue);
 539         return NULL;
 540 }
 541
 542 /**
 543  * Get flow director queue for a specific RX queue, create it in case
 544  * it does not exist.
 545  *
 546  * @param priv
 547  *   Private structure.
 548  * @param idx
 549  *   RX queue index.
 550  *
 551  * @return
 552  *   Related flow director queue on success, NULL otherwise.
 553  */
 554 static struct fdir_queue *
 555 priv_get_fdir_queue(struct priv *priv, uint16_t idx)
 556 {
 557         struct rxq_ctrl *rxq_ctrl =
 558                 container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 559         struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue;
 560
 561         assert(rxq_ctrl->wq);
 562         if (fdir_queue == NULL) {
 563                 fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq,
 564                                                     rxq_ctrl->socket);
 565                 rxq_ctrl->fdir_queue = fdir_queue;
 566         }
 567         return fdir_queue;
 568 }
 569
 570 /**
 571  * Get or flow director drop queue. Create it if it does not exist.
 572  *
 573  * @param priv
 574  *   Private structure.
 575  *
 576  * @return
 577  *   Flow director drop queue on success, NULL otherwise.
 578  */
 579 static struct fdir_queue *
 580 priv_get_fdir_drop_queue(struct priv *priv)
 581 {
 582         struct fdir_queue *fdir_queue = priv->fdir_drop_queue;
 583
 584         if (fdir_queue == NULL) {
 585                 unsigned int socket = SOCKET_ID_ANY;
 586
 587                 /* Select a known NUMA socket if possible. */
 588                 if (priv->rxqs_n && (*priv->rxqs)[0])
 589                         socket = container_of((*priv->rxqs)[0],
 590                                               struct rxq_ctrl, rxq)->socket;
 591                 fdir_queue = priv_fdir_queue_create(priv, NULL, socket);
 592                 priv->fdir_drop_queue = fdir_queue;
 593         }
 594         return fdir_queue;
 595 }
 596
 597 /**
 598  * Enable flow director filter and create steering rules.
 599  *
 600  * @param priv
 601  *   Private structure.
 602  * @param mlx5_fdir_filter
 603  *   Filter to create steering rule for.
 604  *
 605  * @return
 606  *   0 on success, errno value on failure.
 607  */
 608 static int
 609 priv_fdir_filter_enable(struct priv *priv,
 610                         struct mlx5_fdir_filter *mlx5_fdir_filter)
 611 {
 612         struct fdir_queue *fdir_queue;
 613
 614         /* Check if flow already exists. */
 615         if (mlx5_fdir_filter->flow != NULL)
 616                 return 0;
 617
 618         /* Get fdir_queue for specific queue. */
 619         if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT)
 620                 fdir_queue = priv_get_fdir_drop_queue(priv);
 621         else
 622                 fdir_queue = priv_get_fdir_queue(priv,
 623                                                  mlx5_fdir_filter->queue);
 624
 625         if (fdir_queue == NULL) {
 626                 ERROR("failed to create flow director rxq for queue %d",
 627                       mlx5_fdir_filter->queue);
 628                 return EINVAL;
 629         }
 630
 631         /* Create flow */
 632         return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
 633 }
 634
 635 /**
 636  * Initialize flow director filters list.
 637  *
 638  * @param priv
 639  *   Private structure.
 640  *
 641  * @return
 642  *   0 on success, errno value on failure.
 643  */
 644 int
 645 fdir_init_filters_list(struct priv *priv)
 646 {
 647         /* Filter list initialization should be done only once. */
 648         if (priv->fdir_filter_list)
 649                 return 0;
 650
 651         /* Create filters list. */
 652         priv->fdir_filter_list =
 653                 rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
 654
 655         if (priv->fdir_filter_list == NULL) {
 656                 int err = ENOMEM;
 657
 658                 ERROR("cannot allocate flow director filter list: %s",
 659                       strerror(err));
 660                 return err;
 661         }
 662
 663         LIST_INIT(priv->fdir_filter_list);
 664
 665         return 0;
 666 }
 667
 668 /**
 669  * Flush all filters.
 670  *
 671  * @param priv
 672  *   Private structure.
 673  */
 674 static void
 675 priv_fdir_filter_flush(struct priv *priv)
 676 {
 677         struct mlx5_fdir_filter *mlx5_fdir_filter;
 678
 679         while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
 680                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 681
 682                 DEBUG("%p: flushing flow director filter %p",
 683                       (void *)priv, (void *)mlx5_fdir_filter);
 684                 LIST_REMOVE(mlx5_fdir_filter, next);
 685                 if (flow != NULL)
 686                         claim_zero(ibv_exp_destroy_flow(flow));
 687                 rte_free(mlx5_fdir_filter);
 688         }
 689 }
 690
 691 /**
 692  * Remove all flow director filters and delete list.
 693  *
 694  * @param priv
 695  *   Private structure.
 696  */
 697 void
 698 priv_fdir_delete_filters_list(struct priv *priv)
 699 {
 700         priv_fdir_filter_flush(priv);
 701         rte_free(priv->fdir_filter_list);
 702         priv->fdir_filter_list = NULL;
 703 }
 704
 705 /**
 706  * Disable flow director, remove all steering rules.
 707  *
 708  * @param priv
 709  *   Private structure.
 710  */
 711 void
 712 priv_fdir_disable(struct priv *priv)
 713 {
 714         unsigned int i;
 715         struct mlx5_fdir_filter *mlx5_fdir_filter;
 716
 717         /* Run on every flow director filter and destroy flow handle. */
 718         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 719                 struct ibv_exp_flow *flow;
 720
 721                 /* Only valid elements should be in the list */
 722                 assert(mlx5_fdir_filter != NULL);
 723                 flow = mlx5_fdir_filter->flow;
 724
 725                 /* Destroy flow handle */
 726                 if (flow != NULL) {
 727                         claim_zero(ibv_exp_destroy_flow(flow));
 728                         mlx5_fdir_filter->flow = NULL;
 729                 }
 730         }
 731
 732         /* Destroy flow director context in each RX queue. */
 733         for (i = 0; (i != priv->rxqs_n); i++) {
 734                 struct rxq_ctrl *rxq_ctrl =
 735                         container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
 736
 737                 if (!rxq_ctrl->fdir_queue)
 738                         continue;
 739                 priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue);
 740                 rxq_ctrl->fdir_queue = NULL;
 741         }
 742         if (priv->fdir_drop_queue) {
 743                 priv_fdir_queue_destroy(priv, priv->fdir_drop_queue);
 744                 priv->fdir_drop_queue = NULL;
 745         }
 746 }
 747
 748 /**
 749  * Enable flow director, create steering rules.
 750  *
 751  * @param priv
 752  *   Private structure.
 753  */
 754 void
 755 priv_fdir_enable(struct priv *priv)
 756 {
 757         struct mlx5_fdir_filter *mlx5_fdir_filter;
 758
 759         /* Run on every fdir filter and create flow handle */
 760         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 761                 /* Only valid elements should be in the list */
 762                 assert(mlx5_fdir_filter != NULL);
 763
 764                 priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 765         }
 766 }
 767
 768 /**
 769  * Find specific filter in list.
 770  *
 771  * @param priv
 772  *   Private structure.
 773  * @param fdir_filter
 774  *   Flow director filter to find.
 775  *
 776  * @return
 777  *   Filter element if found, otherwise NULL.
 778  */
 779 static struct mlx5_fdir_filter *
 780 priv_find_filter_in_list(struct priv *priv,
 781                          const struct rte_eth_fdir_filter *fdir_filter)
 782 {
 783         struct fdir_flow_desc desc;
 784         struct mlx5_fdir_filter *mlx5_fdir_filter;
 785         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 786
 787         /* Get flow director filter to look for. */
 788         fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
 789
 790         /* Look for the requested element. */
 791         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 792                 /* Only valid elements should be in the list. */
 793                 assert(mlx5_fdir_filter != NULL);
 794
 795                 /* Return matching filter. */
 796                 if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
 797                         return mlx5_fdir_filter;
 798         }
 799
 800         /* Filter not found */
 801         return NULL;
 802 }
 803
 804 /**
 805  * Add new flow director filter and store it in list.
 806  *
 807  * @param priv
 808  *   Private structure.
 809  * @param fdir_filter
 810  *   Flow director filter to add.
 811  *
 812  * @return
 813  *   0 on success, errno value on failure.
 814  */
 815 static int
 816 priv_fdir_filter_add(struct priv *priv,
 817                      const struct rte_eth_fdir_filter *fdir_filter)
 818 {
 819         struct mlx5_fdir_filter *mlx5_fdir_filter;
 820         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 821         int err = 0;
 822
 823         /* Validate queue number. */
 824         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 825                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 826                 return EINVAL;
 827         }
 828
 829         /* Duplicate filters are currently unsupported. */
 830         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 831         if (mlx5_fdir_filter != NULL) {
 832                 ERROR("filter already exists");
 833                 return EINVAL;
 834         }
 835
 836         /* Create new flow director filter. */
 837         mlx5_fdir_filter =
 838                 rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
 839         if (mlx5_fdir_filter == NULL) {
 840                 err = ENOMEM;
 841                 ERROR("cannot allocate flow director filter: %s",
 842                       strerror(err));
 843                 return err;
 844         }
 845
 846         /* Set action parameters. */
 847         mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 848         mlx5_fdir_filter->behavior = fdir_filter->action.behavior;
 849
 850         /* Convert to mlx5 filter descriptor. */
 851         fdir_filter_to_flow_desc(fdir_filter,
 852                                  &mlx5_fdir_filter->desc, fdir_mode);
 853
 854         /* Insert new filter into list. */
 855         LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
 856
 857         DEBUG("%p: flow director filter %p added",
 858               (void *)priv, (void *)mlx5_fdir_filter);
 859
 860         /* Enable filter immediately if device is started. */
 861         if (priv->started)
 862                 err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 863
 864         return err;
 865 }
 866
 867 /**
 868  * Update queue for specific filter.
 869  *
 870  * @param priv
 871  *   Private structure.
 872  * @param fdir_filter
 873  *   Filter to be updated.
 874  *
 875  * @return
 876  *   0 on success, errno value on failure.
 877  */
 878 static int
 879 priv_fdir_filter_update(struct priv *priv,
 880                         const struct rte_eth_fdir_filter *fdir_filter)
 881 {
 882         struct mlx5_fdir_filter *mlx5_fdir_filter;
 883
 884         /* Validate queue number. */
 885         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 886                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 887                 return EINVAL;
 888         }
 889
 890         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 891         if (mlx5_fdir_filter != NULL) {
 892                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 893                 int err = 0;
 894
 895                 /* Update queue number. */
 896                 mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 897
 898                 /* Destroy flow handle. */
 899                 if (flow != NULL) {
 900                         claim_zero(ibv_exp_destroy_flow(flow));
 901                         mlx5_fdir_filter->flow = NULL;
 902                 }
 903                 DEBUG("%p: flow director filter %p updated",
 904                       (void *)priv, (void *)mlx5_fdir_filter);
 905
 906                 /* Enable filter if device is started. */
 907                 if (priv->started)
 908                         err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 909
 910                 return err;
 911         }
 912
 913         /* Filter not found, create it. */
 914         DEBUG("%p: filter not found for update, creating new filter",
 915               (void *)priv);
 916         return priv_fdir_filter_add(priv, fdir_filter);
 917 }
 918
 919 /**
 920  * Delete specific filter.
 921  *
 922  * @param priv
 923  *   Private structure.
 924  * @param fdir_filter
 925  *   Filter to be deleted.
 926  *
 927  * @return
 928  *   0 on success, errno value on failure.
 929  */
 930 static int
 931 priv_fdir_filter_delete(struct priv *priv,
 932                         const struct rte_eth_fdir_filter *fdir_filter)
 933 {
 934         struct mlx5_fdir_filter *mlx5_fdir_filter;
 935
 936         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 937         if (mlx5_fdir_filter != NULL) {
 938                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 939
 940                 /* Remove element from list. */
 941                 LIST_REMOVE(mlx5_fdir_filter, next);
 942
 943                 /* Destroy flow handle. */
 944                 if (flow != NULL) {
 945                         claim_zero(ibv_exp_destroy_flow(flow));
 946                         mlx5_fdir_filter->flow = NULL;
 947                 }
 948
 949                 DEBUG("%p: flow director filter %p deleted",
 950                       (void *)priv, (void *)mlx5_fdir_filter);
 951
 952                 /* Delete filter. */
 953                 rte_free(mlx5_fdir_filter);
 954
 955                 return 0;
 956         }
 957
 958         ERROR("%p: flow director delete failed, cannot find filter",
 959               (void *)priv);
 960         return EINVAL;
 961 }
 962
 963 /**
 964  * Get flow director information.
 965  *
 966  * @param priv
 967  *   Private structure.
 968  * @param[out] fdir_info
 969  *   Resulting flow director information.
 970  */
 971 static void
 972 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
 973 {
 974         struct rte_eth_fdir_masks *mask =
 975                 &priv->dev->data->dev_conf.fdir_conf.mask;
 976
 977         fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
 978         fdir_info->guarant_spc = 0;
 979
 980         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
 981
 982         fdir_info->max_flexpayload = 0;
 983         fdir_info->flow_types_mask[0] = 0;
 984
 985         fdir_info->flex_payload_unit = 0;
 986         fdir_info->max_flex_payload_segment_num = 0;
 987         fdir_info->flex_payload_limit = 0;
 988         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
 989 }
 990
 991 /**
 992  * Deal with flow director operations.
 993  *
 994  * @param priv
 995  *   Pointer to private structure.
 996  * @param filter_op
 997  *   Operation to perform.
 998  * @param arg
 999  *   Pointer to operation-specific structure.
1000  *
1001  * @return
1002  *   0 on success, errno value on failure.
1003  */
1004 static int
1005 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
1006 {
1007         enum rte_fdir_mode fdir_mode =
1008                 priv->dev->data->dev_conf.fdir_conf.mode;
1009         int ret = 0;
1010
1011         if (filter_op == RTE_ETH_FILTER_NOP)
1012                 return 0;
1013
1014         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
1015             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
1016                 ERROR("%p: flow director mode %d not supported",
1017                       (void *)priv, fdir_mode);
1018                 return EINVAL;
1019         }
1020
1021         switch (filter_op) {
1022         case RTE_ETH_FILTER_ADD:
1023                 ret = priv_fdir_filter_add(priv, arg);
1024                 break;
1025         case RTE_ETH_FILTER_UPDATE:
1026                 ret = priv_fdir_filter_update(priv, arg);
1027                 break;
1028         case RTE_ETH_FILTER_DELETE:
1029                 ret = priv_fdir_filter_delete(priv, arg);
1030                 break;
1031         case RTE_ETH_FILTER_FLUSH:
1032                 priv_fdir_filter_flush(priv);
1033                 break;
1034         case RTE_ETH_FILTER_INFO:
1035                 priv_fdir_info_get(priv, arg);
1036                 break;
1037         default:
1038                 DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
1039                 ret = EINVAL;
1040                 break;
1041         }
1042         return ret;
1043 }
1044
1045 /**
1046  * Manage filter operations.
1047  *
1048  * @param dev
1049  *   Pointer to Ethernet device structure.
1050  * @param filter_type
1051  *   Filter type.
1052  * @param filter_op
1053  *   Operation to perform.
1054  * @param arg
1055  *   Pointer to operation-specific structure.
1056  *
1057  * @return
1058  *   0 on success, negative errno value on failure.
1059  */
1060 int
1061 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
1062                      enum rte_filter_type filter_type,
1063                      enum rte_filter_op filter_op,
1064                      void *arg)
1065 {
1066         int ret = EINVAL;
1067         struct priv *priv = dev->data->dev_private;
1068
1069         switch (filter_type) {
1070         case RTE_ETH_FILTER_FDIR:
1071                 priv_lock(priv);
1072                 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
1073                 priv_unlock(priv);
1074                 break;
1075         default:
1076                 ERROR("%p: filter type (%d) not supported",
1077                       (void *)dev, filter_type);
1078                 break;
1079         }
1080
1081         return -ret;
1082 }