net/af_packet: make qdisc bypass configurable
[dpdk.git] / drivers / net / mlx4 / mlx4_flow.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright 2017 6WIND S.A.
5  *   Copyright 2017 Mellanox
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of 6WIND S.A. nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 /**
35  * @file
36  * Flow API operations for mlx4 driver.
37  */
38
39 #include <arpa/inet.h>
40 #include <assert.h>
41 #include <errno.h>
42 #include <stddef.h>
43 #include <stdint.h>
44 #include <string.h>
45 #include <sys/queue.h>
46
47 /* Verbs headers do not support -pedantic. */
48 #ifdef PEDANTIC
49 #pragma GCC diagnostic ignored "-Wpedantic"
50 #endif
51 #include <infiniband/verbs.h>
52 #ifdef PEDANTIC
53 #pragma GCC diagnostic error "-Wpedantic"
54 #endif
55
56 #include <rte_errno.h>
57 #include <rte_eth_ctrl.h>
58 #include <rte_ethdev.h>
59 #include <rte_flow.h>
60 #include <rte_flow_driver.h>
61 #include <rte_malloc.h>
62
63 /* PMD headers. */
64 #include "mlx4.h"
65 #include "mlx4_flow.h"
66 #include "mlx4_rxtx.h"
67 #include "mlx4_utils.h"
68
69 /** Static initializer for items. */
70 #define ITEMS(...) \
71         (const enum rte_flow_item_type []){ \
72                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
73         }
74
75 /** Structure to generate a simple graph of layers supported by the NIC. */
76 struct mlx4_flow_items {
77         /** List of possible actions for these items. */
78         const enum rte_flow_action_type *const actions;
79         /** Bit-masks corresponding to the possibilities for the item. */
80         const void *mask;
81         /**
82          * Default bit-masks to use when item->mask is not provided. When
83          * \default_mask is also NULL, the full supported bit-mask (\mask) is
84          * used instead.
85          */
86         const void *default_mask;
87         /** Bit-masks size in bytes. */
88         const unsigned int mask_sz;
89         /**
90          * Check support for a given item.
91          *
92          * @param item[in]
93          *   Item specification.
94          * @param mask[in]
95          *   Bit-masks covering supported fields to compare with spec,
96          *   last and mask in
97          *   \item.
98          * @param size
99          *   Bit-Mask size in bytes.
100          *
101          * @return
102          *   0 on success, negative value otherwise.
103          */
104         int (*validate)(const struct rte_flow_item *item,
105                         const uint8_t *mask, unsigned int size);
106         /**
107          * Conversion function from rte_flow to NIC specific flow.
108          *
109          * @param item
110          *   rte_flow item to convert.
111          * @param default_mask
112          *   Default bit-masks to use when item->mask is not provided.
113          * @param data
114          *   Internal structure to store the conversion.
115          *
116          * @return
117          *   0 on success, negative value otherwise.
118          */
119         int (*convert)(const struct rte_flow_item *item,
120                        const void *default_mask,
121                        void *data);
122         /** Size in bytes of the destination structure. */
123         const unsigned int dst_sz;
124         /** List of possible following items.  */
125         const enum rte_flow_item_type *const items;
126 };
127
128 struct rte_flow_drop {
129         struct ibv_qp *qp; /**< Verbs queue pair. */
130         struct ibv_cq *cq; /**< Verbs completion queue. */
131 };
132
133 /** Valid action for this PMD. */
134 static const enum rte_flow_action_type valid_actions[] = {
135         RTE_FLOW_ACTION_TYPE_DROP,
136         RTE_FLOW_ACTION_TYPE_QUEUE,
137         RTE_FLOW_ACTION_TYPE_END,
138 };
139
140 /**
141  * Convert Ethernet item to Verbs specification.
142  *
143  * @param item[in]
144  *   Item specification.
145  * @param default_mask[in]
146  *   Default bit-masks to use when item->mask is not provided.
147  * @param data[in, out]
148  *   User structure.
149  */
150 static int
151 mlx4_flow_create_eth(const struct rte_flow_item *item,
152                      const void *default_mask,
153                      void *data)
154 {
155         const struct rte_flow_item_eth *spec = item->spec;
156         const struct rte_flow_item_eth *mask = item->mask;
157         struct mlx4_flow *flow = (struct mlx4_flow *)data;
158         struct ibv_flow_spec_eth *eth;
159         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
160         unsigned int i;
161
162         ++flow->ibv_attr->num_of_specs;
163         flow->ibv_attr->priority = 2;
164         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
165         *eth = (struct ibv_flow_spec_eth) {
166                 .type = IBV_FLOW_SPEC_ETH,
167                 .size = eth_size,
168         };
169         if (!spec) {
170                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
171                 return 0;
172         }
173         if (!mask)
174                 mask = default_mask;
175         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
176         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
177         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
178         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
179         /* Remove unwanted bits from values. */
180         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
181                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
182                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
183         }
184         return 0;
185 }
186
187 /**
188  * Convert VLAN item to Verbs specification.
189  *
190  * @param item[in]
191  *   Item specification.
192  * @param default_mask[in]
193  *   Default bit-masks to use when item->mask is not provided.
194  * @param data[in, out]
195  *   User structure.
196  */
197 static int
198 mlx4_flow_create_vlan(const struct rte_flow_item *item,
199                       const void *default_mask,
200                       void *data)
201 {
202         const struct rte_flow_item_vlan *spec = item->spec;
203         const struct rte_flow_item_vlan *mask = item->mask;
204         struct mlx4_flow *flow = (struct mlx4_flow *)data;
205         struct ibv_flow_spec_eth *eth;
206         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
207
208         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
209         if (!spec)
210                 return 0;
211         if (!mask)
212                 mask = default_mask;
213         eth->val.vlan_tag = spec->tci;
214         eth->mask.vlan_tag = mask->tci;
215         eth->val.vlan_tag &= eth->mask.vlan_tag;
216         return 0;
217 }
218
219 /**
220  * Convert IPv4 item to Verbs specification.
221  *
222  * @param item[in]
223  *   Item specification.
224  * @param default_mask[in]
225  *   Default bit-masks to use when item->mask is not provided.
226  * @param data[in, out]
227  *   User structure.
228  */
229 static int
230 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
231                       const void *default_mask,
232                       void *data)
233 {
234         const struct rte_flow_item_ipv4 *spec = item->spec;
235         const struct rte_flow_item_ipv4 *mask = item->mask;
236         struct mlx4_flow *flow = (struct mlx4_flow *)data;
237         struct ibv_flow_spec_ipv4 *ipv4;
238         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
239
240         ++flow->ibv_attr->num_of_specs;
241         flow->ibv_attr->priority = 1;
242         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
243         *ipv4 = (struct ibv_flow_spec_ipv4) {
244                 .type = IBV_FLOW_SPEC_IPV4,
245                 .size = ipv4_size,
246         };
247         if (!spec)
248                 return 0;
249         ipv4->val = (struct ibv_flow_ipv4_filter) {
250                 .src_ip = spec->hdr.src_addr,
251                 .dst_ip = spec->hdr.dst_addr,
252         };
253         if (!mask)
254                 mask = default_mask;
255         ipv4->mask = (struct ibv_flow_ipv4_filter) {
256                 .src_ip = mask->hdr.src_addr,
257                 .dst_ip = mask->hdr.dst_addr,
258         };
259         /* Remove unwanted bits from values. */
260         ipv4->val.src_ip &= ipv4->mask.src_ip;
261         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
262         return 0;
263 }
264
265 /**
266  * Convert UDP item to Verbs specification.
267  *
268  * @param item[in]
269  *   Item specification.
270  * @param default_mask[in]
271  *   Default bit-masks to use when item->mask is not provided.
272  * @param data[in, out]
273  *   User structure.
274  */
275 static int
276 mlx4_flow_create_udp(const struct rte_flow_item *item,
277                      const void *default_mask,
278                      void *data)
279 {
280         const struct rte_flow_item_udp *spec = item->spec;
281         const struct rte_flow_item_udp *mask = item->mask;
282         struct mlx4_flow *flow = (struct mlx4_flow *)data;
283         struct ibv_flow_spec_tcp_udp *udp;
284         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
285
286         ++flow->ibv_attr->num_of_specs;
287         flow->ibv_attr->priority = 0;
288         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
289         *udp = (struct ibv_flow_spec_tcp_udp) {
290                 .type = IBV_FLOW_SPEC_UDP,
291                 .size = udp_size,
292         };
293         if (!spec)
294                 return 0;
295         udp->val.dst_port = spec->hdr.dst_port;
296         udp->val.src_port = spec->hdr.src_port;
297         if (!mask)
298                 mask = default_mask;
299         udp->mask.dst_port = mask->hdr.dst_port;
300         udp->mask.src_port = mask->hdr.src_port;
301         /* Remove unwanted bits from values. */
302         udp->val.src_port &= udp->mask.src_port;
303         udp->val.dst_port &= udp->mask.dst_port;
304         return 0;
305 }
306
307 /**
308  * Convert TCP item to Verbs specification.
309  *
310  * @param item[in]
311  *   Item specification.
312  * @param default_mask[in]
313  *   Default bit-masks to use when item->mask is not provided.
314  * @param data[in, out]
315  *   User structure.
316  */
317 static int
318 mlx4_flow_create_tcp(const struct rte_flow_item *item,
319                      const void *default_mask,
320                      void *data)
321 {
322         const struct rte_flow_item_tcp *spec = item->spec;
323         const struct rte_flow_item_tcp *mask = item->mask;
324         struct mlx4_flow *flow = (struct mlx4_flow *)data;
325         struct ibv_flow_spec_tcp_udp *tcp;
326         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
327
328         ++flow->ibv_attr->num_of_specs;
329         flow->ibv_attr->priority = 0;
330         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
331         *tcp = (struct ibv_flow_spec_tcp_udp) {
332                 .type = IBV_FLOW_SPEC_TCP,
333                 .size = tcp_size,
334         };
335         if (!spec)
336                 return 0;
337         tcp->val.dst_port = spec->hdr.dst_port;
338         tcp->val.src_port = spec->hdr.src_port;
339         if (!mask)
340                 mask = default_mask;
341         tcp->mask.dst_port = mask->hdr.dst_port;
342         tcp->mask.src_port = mask->hdr.src_port;
343         /* Remove unwanted bits from values. */
344         tcp->val.src_port &= tcp->mask.src_port;
345         tcp->val.dst_port &= tcp->mask.dst_port;
346         return 0;
347 }
348
349 /**
350  * Check support for a given item.
351  *
352  * @param item[in]
353  *   Item specification.
354  * @param mask[in]
355  *   Bit-masks covering supported fields to compare with spec, last and mask in
356  *   \item.
357  * @param size
358  *   Bit-Mask size in bytes.
359  *
360  * @return
361  *   0 on success, negative value otherwise.
362  */
363 static int
364 mlx4_flow_item_validate(const struct rte_flow_item *item,
365                         const uint8_t *mask, unsigned int size)
366 {
367         int ret = 0;
368
369         if (!item->spec && (item->mask || item->last))
370                 return -1;
371         if (item->spec && !item->mask) {
372                 unsigned int i;
373                 const uint8_t *spec = item->spec;
374
375                 for (i = 0; i < size; ++i)
376                         if ((spec[i] | mask[i]) != mask[i])
377                                 return -1;
378         }
379         if (item->last && !item->mask) {
380                 unsigned int i;
381                 const uint8_t *spec = item->last;
382
383                 for (i = 0; i < size; ++i)
384                         if ((spec[i] | mask[i]) != mask[i])
385                                 return -1;
386         }
387         if (item->spec && item->last) {
388                 uint8_t spec[size];
389                 uint8_t last[size];
390                 const uint8_t *apply = mask;
391                 unsigned int i;
392
393                 if (item->mask)
394                         apply = item->mask;
395                 for (i = 0; i < size; ++i) {
396                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
397                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
398                 }
399                 ret = memcmp(spec, last, size);
400         }
401         return ret;
402 }
403
404 static int
405 mlx4_flow_validate_eth(const struct rte_flow_item *item,
406                        const uint8_t *mask, unsigned int size)
407 {
408         if (item->mask) {
409                 const struct rte_flow_item_eth *mask = item->mask;
410
411                 if (mask->dst.addr_bytes[0] != 0xff ||
412                                 mask->dst.addr_bytes[1] != 0xff ||
413                                 mask->dst.addr_bytes[2] != 0xff ||
414                                 mask->dst.addr_bytes[3] != 0xff ||
415                                 mask->dst.addr_bytes[4] != 0xff ||
416                                 mask->dst.addr_bytes[5] != 0xff)
417                         return -1;
418         }
419         return mlx4_flow_item_validate(item, mask, size);
420 }
421
422 static int
423 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
424                         const uint8_t *mask, unsigned int size)
425 {
426         if (item->mask) {
427                 const struct rte_flow_item_vlan *mask = item->mask;
428
429                 if (mask->tci != 0 &&
430                     ntohs(mask->tci) != 0x0fff)
431                         return -1;
432         }
433         return mlx4_flow_item_validate(item, mask, size);
434 }
435
436 static int
437 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
438                         const uint8_t *mask, unsigned int size)
439 {
440         if (item->mask) {
441                 const struct rte_flow_item_ipv4 *mask = item->mask;
442
443                 if (mask->hdr.src_addr != 0 &&
444                     mask->hdr.src_addr != 0xffffffff)
445                         return -1;
446                 if (mask->hdr.dst_addr != 0 &&
447                     mask->hdr.dst_addr != 0xffffffff)
448                         return -1;
449         }
450         return mlx4_flow_item_validate(item, mask, size);
451 }
452
453 static int
454 mlx4_flow_validate_udp(const struct rte_flow_item *item,
455                        const uint8_t *mask, unsigned int size)
456 {
457         if (item->mask) {
458                 const struct rte_flow_item_udp *mask = item->mask;
459
460                 if (mask->hdr.src_port != 0 &&
461                     mask->hdr.src_port != 0xffff)
462                         return -1;
463                 if (mask->hdr.dst_port != 0 &&
464                     mask->hdr.dst_port != 0xffff)
465                         return -1;
466         }
467         return mlx4_flow_item_validate(item, mask, size);
468 }
469
470 static int
471 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
472                        const uint8_t *mask, unsigned int size)
473 {
474         if (item->mask) {
475                 const struct rte_flow_item_tcp *mask = item->mask;
476
477                 if (mask->hdr.src_port != 0 &&
478                     mask->hdr.src_port != 0xffff)
479                         return -1;
480                 if (mask->hdr.dst_port != 0 &&
481                     mask->hdr.dst_port != 0xffff)
482                         return -1;
483         }
484         return mlx4_flow_item_validate(item, mask, size);
485 }
486
487 /** Graph of supported items and associated actions. */
488 static const struct mlx4_flow_items mlx4_flow_items[] = {
489         [RTE_FLOW_ITEM_TYPE_END] = {
490                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
491         },
492         [RTE_FLOW_ITEM_TYPE_ETH] = {
493                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
494                                RTE_FLOW_ITEM_TYPE_IPV4),
495                 .actions = valid_actions,
496                 .mask = &(const struct rte_flow_item_eth){
497                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
498                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
499                 },
500                 .default_mask = &rte_flow_item_eth_mask,
501                 .mask_sz = sizeof(struct rte_flow_item_eth),
502                 .validate = mlx4_flow_validate_eth,
503                 .convert = mlx4_flow_create_eth,
504                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
505         },
506         [RTE_FLOW_ITEM_TYPE_VLAN] = {
507                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
508                 .actions = valid_actions,
509                 .mask = &(const struct rte_flow_item_vlan){
510                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
511 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
512                         .tci = 0x0fff,
513 #else
514                         .tci = 0xff0f,
515 #endif
516                 },
517                 .mask_sz = sizeof(struct rte_flow_item_vlan),
518                 .validate = mlx4_flow_validate_vlan,
519                 .convert = mlx4_flow_create_vlan,
520                 .dst_sz = 0,
521         },
522         [RTE_FLOW_ITEM_TYPE_IPV4] = {
523                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
524                                RTE_FLOW_ITEM_TYPE_TCP),
525                 .actions = valid_actions,
526                 .mask = &(const struct rte_flow_item_ipv4){
527                         .hdr = {
528                                 .src_addr = -1,
529                                 .dst_addr = -1,
530                         },
531                 },
532                 .default_mask = &rte_flow_item_ipv4_mask,
533                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
534                 .validate = mlx4_flow_validate_ipv4,
535                 .convert = mlx4_flow_create_ipv4,
536                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
537         },
538         [RTE_FLOW_ITEM_TYPE_UDP] = {
539                 .actions = valid_actions,
540                 .mask = &(const struct rte_flow_item_udp){
541                         .hdr = {
542                                 .src_port = -1,
543                                 .dst_port = -1,
544                         },
545                 },
546                 .default_mask = &rte_flow_item_udp_mask,
547                 .mask_sz = sizeof(struct rte_flow_item_udp),
548                 .validate = mlx4_flow_validate_udp,
549                 .convert = mlx4_flow_create_udp,
550                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
551         },
552         [RTE_FLOW_ITEM_TYPE_TCP] = {
553                 .actions = valid_actions,
554                 .mask = &(const struct rte_flow_item_tcp){
555                         .hdr = {
556                                 .src_port = -1,
557                                 .dst_port = -1,
558                         },
559                 },
560                 .default_mask = &rte_flow_item_tcp_mask,
561                 .mask_sz = sizeof(struct rte_flow_item_tcp),
562                 .validate = mlx4_flow_validate_tcp,
563                 .convert = mlx4_flow_create_tcp,
564                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
565         },
566 };
567
568 /**
569  * Make sure a flow rule is supported and initialize associated structure.
570  *
571  * @param priv
572  *   Pointer to private structure.
573  * @param[in] attr
574  *   Flow rule attributes.
575  * @param[in] items
576  *   Pattern specification (list terminated by the END pattern item).
577  * @param[in] actions
578  *   Associated actions (list terminated by the END action).
579  * @param[out] error
580  *   Perform verbose error reporting if not NULL.
581  * @param[in, out] flow
582  *   Flow structure to update.
583  *
584  * @return
585  *   0 on success, a negative errno value otherwise and rte_errno is set.
586  */
587 static int
588 mlx4_flow_prepare(struct priv *priv,
589                   const struct rte_flow_attr *attr,
590                   const struct rte_flow_item items[],
591                   const struct rte_flow_action actions[],
592                   struct rte_flow_error *error,
593                   struct mlx4_flow *flow)
594 {
595         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
596         struct mlx4_flow_action action = {
597                 .queue = 0,
598                 .drop = 0,
599         };
600
601         (void)priv;
602         if (attr->group) {
603                 rte_flow_error_set(error, ENOTSUP,
604                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
605                                    NULL,
606                                    "groups are not supported");
607                 return -rte_errno;
608         }
609         if (attr->priority) {
610                 rte_flow_error_set(error, ENOTSUP,
611                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
612                                    NULL,
613                                    "priorities are not supported");
614                 return -rte_errno;
615         }
616         if (attr->egress) {
617                 rte_flow_error_set(error, ENOTSUP,
618                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
619                                    NULL,
620                                    "egress is not supported");
621                 return -rte_errno;
622         }
623         if (!attr->ingress) {
624                 rte_flow_error_set(error, ENOTSUP,
625                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
626                                    NULL,
627                                    "only ingress is supported");
628                 return -rte_errno;
629         }
630         /* Go over items list. */
631         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
632                 const struct mlx4_flow_items *token = NULL;
633                 unsigned int i;
634                 int err;
635
636                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
637                         continue;
638                 /*
639                  * The nic can support patterns with NULL eth spec only
640                  * if eth is a single item in a rule.
641                  */
642                 if (!items->spec &&
643                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
644                         const struct rte_flow_item *next = items + 1;
645
646                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
647                                 rte_flow_error_set(error, ENOTSUP,
648                                                    RTE_FLOW_ERROR_TYPE_ITEM,
649                                                    items,
650                                                    "the rule requires"
651                                                    " an Ethernet spec");
652                                 return -rte_errno;
653                         }
654                 }
655                 for (i = 0;
656                      cur_item->items &&
657                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
658                      ++i) {
659                         if (cur_item->items[i] == items->type) {
660                                 token = &mlx4_flow_items[items->type];
661                                 break;
662                         }
663                 }
664                 if (!token)
665                         goto exit_item_not_supported;
666                 cur_item = token;
667                 err = cur_item->validate(items,
668                                         (const uint8_t *)cur_item->mask,
669                                          cur_item->mask_sz);
670                 if (err)
671                         goto exit_item_not_supported;
672                 if (flow->ibv_attr && cur_item->convert) {
673                         err = cur_item->convert(items,
674                                                 (cur_item->default_mask ?
675                                                  cur_item->default_mask :
676                                                  cur_item->mask),
677                                                  flow);
678                         if (err)
679                                 goto exit_item_not_supported;
680                 }
681                 flow->offset += cur_item->dst_sz;
682         }
683         /* Go over actions list */
684         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
685                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
686                         continue;
687                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
688                         action.drop = 1;
689                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
690                         const struct rte_flow_action_queue *queue =
691                                 (const struct rte_flow_action_queue *)
692                                 actions->conf;
693
694                         if (!queue || (queue->index >
695                                        (priv->dev->data->nb_rx_queues - 1)))
696                                 goto exit_action_not_supported;
697                         action.queue = 1;
698                 } else {
699                         goto exit_action_not_supported;
700                 }
701         }
702         if (!action.queue && !action.drop) {
703                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
704                                    NULL, "no valid action");
705                 return -rte_errno;
706         }
707         return 0;
708 exit_item_not_supported:
709         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
710                            items, "item not supported");
711         return -rte_errno;
712 exit_action_not_supported:
713         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
714                            actions, "action not supported");
715         return -rte_errno;
716 }
717
718 /**
719  * Validate a flow supported by the NIC.
720  *
721  * @see rte_flow_validate()
722  * @see rte_flow_ops
723  */
724 static int
725 mlx4_flow_validate(struct rte_eth_dev *dev,
726                    const struct rte_flow_attr *attr,
727                    const struct rte_flow_item items[],
728                    const struct rte_flow_action actions[],
729                    struct rte_flow_error *error)
730 {
731         struct priv *priv = dev->data->dev_private;
732         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
733
734         return mlx4_flow_prepare(priv, attr, items, actions, error, &flow);
735 }
736
737 /**
738  * Destroy a drop queue.
739  *
740  * @param priv
741  *   Pointer to private structure.
742  */
743 static void
744 mlx4_flow_destroy_drop_queue(struct priv *priv)
745 {
746         if (priv->flow_drop_queue) {
747                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
748
749                 priv->flow_drop_queue = NULL;
750                 claim_zero(ibv_destroy_qp(fdq->qp));
751                 claim_zero(ibv_destroy_cq(fdq->cq));
752                 rte_free(fdq);
753         }
754 }
755
756 /**
757  * Create a single drop queue for all drop flows.
758  *
759  * @param priv
760  *   Pointer to private structure.
761  *
762  * @return
763  *   0 on success, negative value otherwise.
764  */
765 static int
766 mlx4_flow_create_drop_queue(struct priv *priv)
767 {
768         struct ibv_qp *qp;
769         struct ibv_cq *cq;
770         struct rte_flow_drop *fdq;
771
772         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
773         if (!fdq) {
774                 ERROR("Cannot allocate memory for drop struct");
775                 goto err;
776         }
777         cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
778         if (!cq) {
779                 ERROR("Cannot create drop CQ");
780                 goto err_create_cq;
781         }
782         qp = ibv_create_qp(priv->pd,
783                            &(struct ibv_qp_init_attr){
784                                 .send_cq = cq,
785                                 .recv_cq = cq,
786                                 .cap = {
787                                         .max_recv_wr = 1,
788                                         .max_recv_sge = 1,
789                                 },
790                                 .qp_type = IBV_QPT_RAW_PACKET,
791                            });
792         if (!qp) {
793                 ERROR("Cannot create drop QP");
794                 goto err_create_qp;
795         }
796         *fdq = (struct rte_flow_drop){
797                 .qp = qp,
798                 .cq = cq,
799         };
800         priv->flow_drop_queue = fdq;
801         return 0;
802 err_create_qp:
803         claim_zero(ibv_destroy_cq(cq));
804 err_create_cq:
805         rte_free(fdq);
806 err:
807         return -1;
808 }
809
810 /**
811  * Complete flow rule creation.
812  *
813  * @param priv
814  *   Pointer to private structure.
815  * @param ibv_attr
816  *   Verbs flow attributes.
817  * @param action
818  *   Target action structure.
819  * @param[out] error
820  *   Perform verbose error reporting if not NULL.
821  *
822  * @return
823  *   A flow if the rule could be created.
824  */
825 static struct rte_flow *
826 mlx4_flow_create_action_queue(struct priv *priv,
827                               struct ibv_flow_attr *ibv_attr,
828                               struct mlx4_flow_action *action,
829                               struct rte_flow_error *error)
830 {
831         struct ibv_qp *qp;
832         struct rte_flow *rte_flow;
833
834         assert(priv->pd);
835         assert(priv->ctx);
836         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
837         if (!rte_flow) {
838                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
839                                    NULL, "cannot allocate flow memory");
840                 return NULL;
841         }
842         if (action->drop) {
843                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
844         } else {
845                 struct rxq *rxq = priv->dev->data->rx_queues[action->queue_id];
846
847                 qp = rxq->qp;
848                 rte_flow->qp = qp;
849         }
850         rte_flow->ibv_attr = ibv_attr;
851         if (!priv->started)
852                 return rte_flow;
853         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
854         if (!rte_flow->ibv_flow) {
855                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
856                                    NULL, "flow rule creation failure");
857                 goto error;
858         }
859         return rte_flow;
860 error:
861         rte_free(rte_flow);
862         return NULL;
863 }
864
865 /**
866  * Create a flow.
867  *
868  * @see rte_flow_create()
869  * @see rte_flow_ops
870  */
871 static struct rte_flow *
872 mlx4_flow_create(struct rte_eth_dev *dev,
873                  const struct rte_flow_attr *attr,
874                  const struct rte_flow_item items[],
875                  const struct rte_flow_action actions[],
876                  struct rte_flow_error *error)
877 {
878         struct priv *priv = dev->data->dev_private;
879         struct rte_flow *rte_flow;
880         struct mlx4_flow_action action;
881         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
882         int err;
883
884         err = mlx4_flow_prepare(priv, attr, items, actions, error, &flow);
885         if (err)
886                 return NULL;
887         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
888         if (!flow.ibv_attr) {
889                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
890                                    NULL, "cannot allocate ibv_attr memory");
891                 return NULL;
892         }
893         flow.offset = sizeof(struct ibv_flow_attr);
894         *flow.ibv_attr = (struct ibv_flow_attr){
895                 .comp_mask = 0,
896                 .type = IBV_FLOW_ATTR_NORMAL,
897                 .size = sizeof(struct ibv_flow_attr),
898                 .priority = attr->priority,
899                 .num_of_specs = 0,
900                 .port = priv->port,
901                 .flags = 0,
902         };
903         claim_zero(mlx4_flow_prepare(priv, attr, items, actions,
904                                      error, &flow));
905         action = (struct mlx4_flow_action){
906                 .queue = 0,
907                 .drop = 0,
908         };
909         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
910                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
911                         continue;
912                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
913                         action.queue = 1;
914                         action.queue_id =
915                                 ((const struct rte_flow_action_queue *)
916                                  actions->conf)->index;
917                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
918                         action.drop = 1;
919                 } else {
920                         rte_flow_error_set(error, ENOTSUP,
921                                            RTE_FLOW_ERROR_TYPE_ACTION,
922                                            actions, "unsupported action");
923                         goto exit;
924                 }
925         }
926         rte_flow = mlx4_flow_create_action_queue(priv, flow.ibv_attr,
927                                                  &action, error);
928         if (rte_flow) {
929                 LIST_INSERT_HEAD(&priv->flows, rte_flow, next);
930                 DEBUG("Flow created %p", (void *)rte_flow);
931                 return rte_flow;
932         }
933 exit:
934         rte_free(flow.ibv_attr);
935         return NULL;
936 }
937
938 /**
939  * Configure isolated mode.
940  *
941  * @see rte_flow_isolate()
942  * @see rte_flow_ops
943  */
944 static int
945 mlx4_flow_isolate(struct rte_eth_dev *dev,
946                   int enable,
947                   struct rte_flow_error *error)
948 {
949         struct priv *priv = dev->data->dev_private;
950
951         if (!!enable == !!priv->isolated)
952                 return 0;
953         priv->isolated = !!enable;
954         if (enable) {
955                 mlx4_mac_addr_del(priv);
956         } else if (mlx4_mac_addr_add(priv) < 0) {
957                 priv->isolated = 1;
958                 return -rte_flow_error_set(error, rte_errno,
959                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
960                                            NULL, "cannot leave isolated mode");
961         }
962         return 0;
963 }
964
965 /**
966  * Destroy a flow.
967  *
968  * @see rte_flow_destroy()
969  * @see rte_flow_ops
970  */
971 static int
972 mlx4_flow_destroy(struct rte_eth_dev *dev,
973                   struct rte_flow *flow,
974                   struct rte_flow_error *error)
975 {
976         (void)dev;
977         (void)error;
978         LIST_REMOVE(flow, next);
979         if (flow->ibv_flow)
980                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
981         rte_free(flow->ibv_attr);
982         DEBUG("Flow destroyed %p", (void *)flow);
983         rte_free(flow);
984         return 0;
985 }
986
987 /**
988  * Destroy all flows.
989  *
990  * @see rte_flow_flush()
991  * @see rte_flow_ops
992  */
993 static int
994 mlx4_flow_flush(struct rte_eth_dev *dev,
995                 struct rte_flow_error *error)
996 {
997         struct priv *priv = dev->data->dev_private;
998
999         while (!LIST_EMPTY(&priv->flows)) {
1000                 struct rte_flow *flow;
1001
1002                 flow = LIST_FIRST(&priv->flows);
1003                 mlx4_flow_destroy(dev, flow, error);
1004         }
1005         return 0;
1006 }
1007
1008 /**
1009  * Remove all flows.
1010  *
1011  * Called by dev_stop() to remove all flows.
1012  *
1013  * @param priv
1014  *   Pointer to private structure.
1015  */
1016 void
1017 mlx4_flow_stop(struct priv *priv)
1018 {
1019         struct rte_flow *flow;
1020
1021         for (flow = LIST_FIRST(&priv->flows);
1022              flow;
1023              flow = LIST_NEXT(flow, next)) {
1024                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1025                 flow->ibv_flow = NULL;
1026                 DEBUG("Flow %p removed", (void *)flow);
1027         }
1028         mlx4_flow_destroy_drop_queue(priv);
1029 }
1030
1031 /**
1032  * Add all flows.
1033  *
1034  * @param priv
1035  *   Pointer to private structure.
1036  *
1037  * @return
1038  *   0 on success, a errno value otherwise and rte_errno is set.
1039  */
1040 int
1041 mlx4_flow_start(struct priv *priv)
1042 {
1043         int ret;
1044         struct ibv_qp *qp;
1045         struct rte_flow *flow;
1046
1047         ret = mlx4_flow_create_drop_queue(priv);
1048         if (ret)
1049                 return -1;
1050         for (flow = LIST_FIRST(&priv->flows);
1051              flow;
1052              flow = LIST_NEXT(flow, next)) {
1053                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1054                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1055                 if (!flow->ibv_flow) {
1056                         DEBUG("Flow %p cannot be applied", (void *)flow);
1057                         rte_errno = EINVAL;
1058                         return rte_errno;
1059                 }
1060                 DEBUG("Flow %p applied", (void *)flow);
1061         }
1062         return 0;
1063 }
1064
1065 static const struct rte_flow_ops mlx4_flow_ops = {
1066         .validate = mlx4_flow_validate,
1067         .create = mlx4_flow_create,
1068         .destroy = mlx4_flow_destroy,
1069         .flush = mlx4_flow_flush,
1070         .isolate = mlx4_flow_isolate,
1071 };
1072
1073 /**
1074  * Manage filter operations.
1075  *
1076  * @param dev
1077  *   Pointer to Ethernet device structure.
1078  * @param filter_type
1079  *   Filter type.
1080  * @param filter_op
1081  *   Operation to perform.
1082  * @param arg
1083  *   Pointer to operation-specific structure.
1084  *
1085  * @return
1086  *   0 on success, negative errno value otherwise and rte_errno is set.
1087  */
1088 int
1089 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1090                  enum rte_filter_type filter_type,
1091                  enum rte_filter_op filter_op,
1092                  void *arg)
1093 {
1094         switch (filter_type) {
1095         case RTE_ETH_FILTER_GENERIC:
1096                 if (filter_op != RTE_ETH_FILTER_GET)
1097                         break;
1098                 *(const void **)arg = &mlx4_flow_ops;
1099                 return 0;
1100         default:
1101                 ERROR("%p: filter type (%d) not supported",
1102                       (void *)dev, filter_type);
1103                 break;
1104         }
1105         rte_errno = ENOTSUP;
1106         return -rte_errno;
1107 }