net/cxgbe: add flow operations to offload VLAN actions
[dpdk.git] / drivers / net / cxgbe / cxgbe_flow.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 Chelsio Communications.
3  * All rights reserved.
4  */
5 #include "common.h"
6 #include "cxgbe_flow.h"
7
8 #define __CXGBE_FILL_FS(__v, __m, fs, elem, e) \
9 do { \
10         if (!((fs)->val.elem || (fs)->mask.elem)) { \
11                 (fs)->val.elem = (__v); \
12                 (fs)->mask.elem = (__m); \
13         } else { \
14                 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, \
15                                           NULL, "a filter can be specified" \
16                                           " only once"); \
17         } \
18 } while (0)
19
20 #define __CXGBE_FILL_FS_MEMCPY(__v, __m, fs, elem) \
21 do { \
22         memcpy(&(fs)->val.elem, &(__v), sizeof(__v)); \
23         memcpy(&(fs)->mask.elem, &(__m), sizeof(__m)); \
24 } while (0)
25
26 #define CXGBE_FILL_FS(v, m, elem) \
27         __CXGBE_FILL_FS(v, m, fs, elem, e)
28
29 #define CXGBE_FILL_FS_MEMCPY(v, m, elem) \
30         __CXGBE_FILL_FS_MEMCPY(v, m, fs, elem)
31
32 static int
33 cxgbe_validate_item(const struct rte_flow_item *i, struct rte_flow_error *e)
34 {
35         /* rte_flow specification does not allow it. */
36         if (!i->spec && (i->mask ||  i->last))
37                 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
38                                    i, "last or mask given without spec");
39         /*
40          * We don't support it.
41          * Although, we can support values in last as 0's or last == spec.
42          * But this will not provide user with any additional functionality
43          * and will only increase the complexity for us.
44          */
45         if (i->last)
46                 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
47                                    i, "last is not supported by chelsio pmd");
48         return 0;
49 }
50
51 static void
52 cxgbe_fill_filter_region(struct adapter *adap,
53                          struct ch_filter_specification *fs)
54 {
55         struct tp_params *tp = &adap->params.tp;
56         u64 hash_filter_mask = tp->hash_filter_mask;
57         u64 ntuple_mask = 0;
58
59         fs->cap = 0;
60
61         if (!is_hashfilter(adap))
62                 return;
63
64         if (fs->type) {
65                 uint8_t biton[16] = {0xff, 0xff, 0xff, 0xff,
66                                      0xff, 0xff, 0xff, 0xff,
67                                      0xff, 0xff, 0xff, 0xff,
68                                      0xff, 0xff, 0xff, 0xff};
69                 uint8_t bitoff[16] = {0};
70
71                 if (!memcmp(fs->val.lip, bitoff, sizeof(bitoff)) ||
72                     !memcmp(fs->val.fip, bitoff, sizeof(bitoff)) ||
73                     memcmp(fs->mask.lip, biton, sizeof(biton)) ||
74                     memcmp(fs->mask.fip, biton, sizeof(biton)))
75                         return;
76         } else {
77                 uint32_t biton  = 0xffffffff;
78                 uint32_t bitoff = 0x0U;
79
80                 if (!memcmp(fs->val.lip, &bitoff, sizeof(bitoff)) ||
81                     !memcmp(fs->val.fip, &bitoff, sizeof(bitoff)) ||
82                     memcmp(fs->mask.lip, &biton, sizeof(biton)) ||
83                     memcmp(fs->mask.fip, &biton, sizeof(biton)))
84                         return;
85         }
86
87         if (!fs->val.lport || fs->mask.lport != 0xffff)
88                 return;
89         if (!fs->val.fport || fs->mask.fport != 0xffff)
90                 return;
91
92         if (tp->protocol_shift >= 0)
93                 ntuple_mask |= (u64)fs->mask.proto << tp->protocol_shift;
94         if (tp->ethertype_shift >= 0)
95                 ntuple_mask |= (u64)fs->mask.ethtype << tp->ethertype_shift;
96         if (tp->port_shift >= 0)
97                 ntuple_mask |= (u64)fs->mask.iport << tp->port_shift;
98
99         if (ntuple_mask != hash_filter_mask)
100                 return;
101
102         fs->cap = 1;    /* use hash region */
103 }
104
105 static int
106 ch_rte_parsetype_port(const void *dmask, const struct rte_flow_item *item,
107                       struct ch_filter_specification *fs,
108                       struct rte_flow_error *e)
109 {
110         const struct rte_flow_item_phy_port *val = item->spec;
111         const struct rte_flow_item_phy_port *umask = item->mask;
112         const struct rte_flow_item_phy_port *mask;
113
114         mask = umask ? umask : (const struct rte_flow_item_phy_port *)dmask;
115
116         if (val->index > 0x7)
117                 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
118                                           item,
119                                           "port index upto 0x7 is supported");
120
121         CXGBE_FILL_FS(val->index, mask->index, iport);
122
123         return 0;
124 }
125
126 static int
127 ch_rte_parsetype_udp(const void *dmask, const struct rte_flow_item *item,
128                      struct ch_filter_specification *fs,
129                      struct rte_flow_error *e)
130 {
131         const struct rte_flow_item_udp *val = item->spec;
132         const struct rte_flow_item_udp *umask = item->mask;
133         const struct rte_flow_item_udp *mask;
134
135         mask = umask ? umask : (const struct rte_flow_item_udp *)dmask;
136
137         if (mask->hdr.dgram_len || mask->hdr.dgram_cksum)
138                 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
139                                           item,
140                                           "udp: only src/dst port supported");
141
142         CXGBE_FILL_FS(IPPROTO_UDP, 0xff, proto);
143         if (!val)
144                 return 0;
145         CXGBE_FILL_FS(be16_to_cpu(val->hdr.src_port),
146                       be16_to_cpu(mask->hdr.src_port), fport);
147         CXGBE_FILL_FS(be16_to_cpu(val->hdr.dst_port),
148                       be16_to_cpu(mask->hdr.dst_port), lport);
149         return 0;
150 }
151
152 static int
153 ch_rte_parsetype_tcp(const void *dmask, const struct rte_flow_item *item,
154                      struct ch_filter_specification *fs,
155                      struct rte_flow_error *e)
156 {
157         const struct rte_flow_item_tcp *val = item->spec;
158         const struct rte_flow_item_tcp *umask = item->mask;
159         const struct rte_flow_item_tcp *mask;
160
161         mask = umask ? umask : (const struct rte_flow_item_tcp *)dmask;
162
163         if (mask->hdr.sent_seq || mask->hdr.recv_ack || mask->hdr.data_off ||
164             mask->hdr.tcp_flags || mask->hdr.rx_win || mask->hdr.cksum ||
165             mask->hdr.tcp_urp)
166                 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
167                                           item,
168                                           "tcp: only src/dst port supported");
169
170         CXGBE_FILL_FS(IPPROTO_TCP, 0xff, proto);
171         if (!val)
172                 return 0;
173         CXGBE_FILL_FS(be16_to_cpu(val->hdr.src_port),
174                       be16_to_cpu(mask->hdr.src_port), fport);
175         CXGBE_FILL_FS(be16_to_cpu(val->hdr.dst_port),
176                       be16_to_cpu(mask->hdr.dst_port), lport);
177         return 0;
178 }
179
180 static int
181 ch_rte_parsetype_ipv4(const void *dmask, const struct rte_flow_item *item,
182                       struct ch_filter_specification *fs,
183                       struct rte_flow_error *e)
184 {
185         const struct rte_flow_item_ipv4 *val = item->spec;
186         const struct rte_flow_item_ipv4 *umask = item->mask;
187         const struct rte_flow_item_ipv4 *mask;
188
189         mask = umask ? umask : (const struct rte_flow_item_ipv4 *)dmask;
190
191         if (mask->hdr.time_to_live || mask->hdr.type_of_service)
192                 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
193                                           item, "ttl/tos are not supported");
194
195         fs->type = FILTER_TYPE_IPV4;
196         CXGBE_FILL_FS(ETHER_TYPE_IPv4, 0xffff, ethtype);
197         if (!val)
198                 return 0; /* ipv4 wild card */
199
200         CXGBE_FILL_FS(val->hdr.next_proto_id, mask->hdr.next_proto_id, proto);
201         CXGBE_FILL_FS_MEMCPY(val->hdr.dst_addr, mask->hdr.dst_addr, lip);
202         CXGBE_FILL_FS_MEMCPY(val->hdr.src_addr, mask->hdr.src_addr, fip);
203
204         return 0;
205 }
206
207 static int
208 ch_rte_parsetype_ipv6(const void *dmask, const struct rte_flow_item *item,
209                       struct ch_filter_specification *fs,
210                       struct rte_flow_error *e)
211 {
212         const struct rte_flow_item_ipv6 *val = item->spec;
213         const struct rte_flow_item_ipv6 *umask = item->mask;
214         const struct rte_flow_item_ipv6 *mask;
215
216         mask = umask ? umask : (const struct rte_flow_item_ipv6 *)dmask;
217
218         if (mask->hdr.vtc_flow ||
219             mask->hdr.payload_len || mask->hdr.hop_limits)
220                 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
221                                           item,
222                                           "tc/flow/hop are not supported");
223
224         fs->type = FILTER_TYPE_IPV6;
225         CXGBE_FILL_FS(ETHER_TYPE_IPv6, 0xffff, ethtype);
226         if (!val)
227                 return 0; /* ipv6 wild card */
228
229         CXGBE_FILL_FS(val->hdr.proto, mask->hdr.proto, proto);
230         CXGBE_FILL_FS_MEMCPY(val->hdr.dst_addr, mask->hdr.dst_addr, lip);
231         CXGBE_FILL_FS_MEMCPY(val->hdr.src_addr, mask->hdr.src_addr, fip);
232
233         return 0;
234 }
235
236 static int
237 cxgbe_rtef_parse_attr(struct rte_flow *flow, const struct rte_flow_attr *attr,
238                       struct rte_flow_error *e)
239 {
240         if (attr->egress)
241                 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR,
242                                           attr, "attribute:<egress> is"
243                                           " not supported !");
244         if (attr->group > 0)
245                 return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR,
246                                           attr, "group parameter is"
247                                           " not supported.");
248
249         flow->fidx = attr->priority ? attr->priority - 1 : FILTER_ID_MAX;
250
251         return 0;
252 }
253
254 static inline int check_rxq(struct rte_eth_dev *dev, uint16_t rxq)
255 {
256         struct port_info *pi = ethdev2pinfo(dev);
257
258         if (rxq > pi->n_rx_qsets)
259                 return -EINVAL;
260         return 0;
261 }
262
263 static int cxgbe_validate_fidxondel(struct filter_entry *f, unsigned int fidx)
264 {
265         struct adapter *adap = ethdev2adap(f->dev);
266         struct ch_filter_specification fs = f->fs;
267
268         if (fidx >= adap->tids.nftids) {
269                 dev_err(adap, "invalid flow index %d.\n", fidx);
270                 return -EINVAL;
271         }
272         if (!is_filter_set(&adap->tids, fidx, fs.type)) {
273                 dev_err(adap, "Already free fidx:%d f:%p\n", fidx, f);
274                 return -EINVAL;
275         }
276
277         return 0;
278 }
279
280 static int
281 cxgbe_validate_fidxonadd(struct ch_filter_specification *fs,
282                          struct adapter *adap, unsigned int fidx)
283 {
284         if (is_filter_set(&adap->tids, fidx, fs->type)) {
285                 dev_err(adap, "filter index: %d is busy.\n", fidx);
286                 return -EBUSY;
287         }
288         if (fidx >= adap->tids.nftids) {
289                 dev_err(adap, "filter index (%u) >= max(%u)\n",
290                         fidx, adap->tids.nftids);
291                 return -ERANGE;
292         }
293
294         return 0;
295 }
296
297 static int
298 cxgbe_verify_fidx(struct rte_flow *flow, unsigned int fidx, uint8_t del)
299 {
300         if (flow->fs.cap)
301                 return 0; /* Hash filters */
302         return del ? cxgbe_validate_fidxondel(flow->f, fidx) :
303                 cxgbe_validate_fidxonadd(&flow->fs,
304                                          ethdev2adap(flow->dev), fidx);
305 }
306
307 static int cxgbe_get_fidx(struct rte_flow *flow, unsigned int *fidx)
308 {
309         struct ch_filter_specification *fs = &flow->fs;
310         struct adapter *adap = ethdev2adap(flow->dev);
311
312         /* For tcam get the next available slot, if default value specified */
313         if (flow->fidx == FILTER_ID_MAX) {
314                 int idx;
315
316                 idx = cxgbe_alloc_ftid(adap, fs->type);
317                 if (idx < 0) {
318                         dev_err(adap, "unable to get a filter index in tcam\n");
319                         return -ENOMEM;
320                 }
321                 *fidx = (unsigned int)idx;
322         } else {
323                 *fidx = flow->fidx;
324         }
325
326         return 0;
327 }
328
329 static int
330 ch_rte_parse_atype_switch(const struct rte_flow_action *a,
331                           struct ch_filter_specification *fs,
332                           struct rte_flow_error *e)
333 {
334         const struct rte_flow_action_of_set_vlan_vid *vlanid;
335         const struct rte_flow_action_of_push_vlan *pushvlan;
336         const struct rte_flow_action_phy_port *port;
337
338         switch (a->type) {
339         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
340                 vlanid = (const struct rte_flow_action_of_set_vlan_vid *)
341                           a->conf;
342                 fs->newvlan = VLAN_REWRITE;
343                 fs->vlan = vlanid->vlan_vid;
344                 break;
345         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
346                 pushvlan = (const struct rte_flow_action_of_push_vlan *)
347                             a->conf;
348                 if (pushvlan->ethertype != ETHER_TYPE_VLAN)
349                         return rte_flow_error_set(e, EINVAL,
350                                                   RTE_FLOW_ERROR_TYPE_ACTION, a,
351                                                   "only ethertype 0x8100 "
352                                                   "supported for push vlan.");
353                 fs->newvlan = VLAN_INSERT;
354                 break;
355         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
356                 fs->newvlan = VLAN_REMOVE;
357                 break;
358         case RTE_FLOW_ACTION_TYPE_PHY_PORT:
359                 port = (const struct rte_flow_action_phy_port *)a->conf;
360                 fs->eport = port->index;
361                 break;
362         default:
363                 /* We are not supposed to come here */
364                 return rte_flow_error_set(e, EINVAL,
365                                           RTE_FLOW_ERROR_TYPE_ACTION, a,
366                                           "Action not supported");
367         }
368
369         return 0;
370 }
371
372 static int
373 cxgbe_rtef_parse_actions(struct rte_flow *flow,
374                          const struct rte_flow_action action[],
375                          struct rte_flow_error *e)
376 {
377         struct ch_filter_specification *fs = &flow->fs;
378         const struct rte_flow_action_queue *q;
379         const struct rte_flow_action *a;
380         char abit = 0;
381         int ret;
382
383         for (a = action; a->type != RTE_FLOW_ACTION_TYPE_END; a++) {
384                 switch (a->type) {
385                 case RTE_FLOW_ACTION_TYPE_VOID:
386                         continue;
387                 case RTE_FLOW_ACTION_TYPE_DROP:
388                         if (abit++)
389                                 return rte_flow_error_set(e, EINVAL,
390                                                 RTE_FLOW_ERROR_TYPE_ACTION, a,
391                                                 "specify only 1 pass/drop");
392                         fs->action = FILTER_DROP;
393                         break;
394                 case RTE_FLOW_ACTION_TYPE_QUEUE:
395                         q = (const struct rte_flow_action_queue *)a->conf;
396                         if (!q)
397                                 return rte_flow_error_set(e, EINVAL,
398                                                 RTE_FLOW_ERROR_TYPE_ACTION, q,
399                                                 "specify rx queue index");
400                         if (check_rxq(flow->dev, q->index))
401                                 return rte_flow_error_set(e, EINVAL,
402                                                 RTE_FLOW_ERROR_TYPE_ACTION, q,
403                                                 "Invalid rx queue");
404                         if (abit++)
405                                 return rte_flow_error_set(e, EINVAL,
406                                                 RTE_FLOW_ERROR_TYPE_ACTION, a,
407                                                 "specify only 1 pass/drop");
408                         fs->action = FILTER_PASS;
409                         fs->dirsteer = 1;
410                         fs->iq = q->index;
411                         break;
412                 case RTE_FLOW_ACTION_TYPE_COUNT:
413                         fs->hitcnts = 1;
414                         break;
415                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
416                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
417                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
418                 case RTE_FLOW_ACTION_TYPE_PHY_PORT:
419                         /* We allow multiple switch actions, but switch is
420                          * not compatible with either queue or drop
421                          */
422                         if (abit++ && fs->action != FILTER_SWITCH)
423                                 return rte_flow_error_set(e, EINVAL,
424                                                 RTE_FLOW_ERROR_TYPE_ACTION, a,
425                                                 "overlapping action specified");
426                         ret = ch_rte_parse_atype_switch(a, fs, e);
427                         if (ret)
428                                 return ret;
429                         fs->action = FILTER_SWITCH;
430                         break;
431                 default:
432                         /* Not supported action : return error */
433                         return rte_flow_error_set(e, ENOTSUP,
434                                                   RTE_FLOW_ERROR_TYPE_ACTION,
435                                                   a, "Action not supported");
436                 }
437         }
438
439         return 0;
440 }
441
442 struct chrte_fparse parseitem[] = {
443                 [RTE_FLOW_ITEM_TYPE_PHY_PORT] = {
444                 .fptr = ch_rte_parsetype_port,
445                 .dmask = &(const struct rte_flow_item_phy_port){
446                         .index = 0x7,
447                 }
448         },
449
450         [RTE_FLOW_ITEM_TYPE_IPV4] = {
451                 .fptr  = ch_rte_parsetype_ipv4,
452                 .dmask = &rte_flow_item_ipv4_mask,
453         },
454
455         [RTE_FLOW_ITEM_TYPE_IPV6] = {
456                 .fptr  = ch_rte_parsetype_ipv6,
457                 .dmask = &rte_flow_item_ipv6_mask,
458         },
459
460         [RTE_FLOW_ITEM_TYPE_UDP] = {
461                 .fptr  = ch_rte_parsetype_udp,
462                 .dmask = &rte_flow_item_udp_mask,
463         },
464
465         [RTE_FLOW_ITEM_TYPE_TCP] = {
466                 .fptr  = ch_rte_parsetype_tcp,
467                 .dmask = &rte_flow_item_tcp_mask,
468         },
469 };
470
471 static int
472 cxgbe_rtef_parse_items(struct rte_flow *flow,
473                        const struct rte_flow_item items[],
474                        struct rte_flow_error *e)
475 {
476         struct adapter *adap = ethdev2adap(flow->dev);
477         const struct rte_flow_item *i;
478         char repeat[ARRAY_SIZE(parseitem)] = {0};
479
480         for (i = items; i->type != RTE_FLOW_ITEM_TYPE_END; i++) {
481                 struct chrte_fparse *idx;
482                 int ret;
483
484                 if (i->type >= ARRAY_SIZE(parseitem))
485                         return rte_flow_error_set(e, ENOTSUP,
486                                                   RTE_FLOW_ERROR_TYPE_ITEM,
487                                                   i, "Item not supported");
488
489                 switch (i->type) {
490                 case RTE_FLOW_ITEM_TYPE_VOID:
491                         continue;
492                 default:
493                         /* check if item is repeated */
494                         if (repeat[i->type])
495                                 return rte_flow_error_set(e, ENOTSUP,
496                                                 RTE_FLOW_ERROR_TYPE_ITEM, i,
497                                                 "parse items cannot be repeated (except void)");
498                         repeat[i->type] = 1;
499
500                         /* validate the item */
501                         ret = cxgbe_validate_item(i, e);
502                         if (ret)
503                                 return ret;
504
505                         idx = &flow->item_parser[i->type];
506                         if (!idx || !idx->fptr) {
507                                 return rte_flow_error_set(e, ENOTSUP,
508                                                 RTE_FLOW_ERROR_TYPE_ITEM, i,
509                                                 "Item not supported");
510                         } else {
511                                 ret = idx->fptr(idx->dmask, i, &flow->fs, e);
512                                 if (ret)
513                                         return ret;
514                         }
515                 }
516         }
517
518         cxgbe_fill_filter_region(adap, &flow->fs);
519
520         return 0;
521 }
522
523 static int
524 cxgbe_flow_parse(struct rte_flow *flow,
525                  const struct rte_flow_attr *attr,
526                  const struct rte_flow_item item[],
527                  const struct rte_flow_action action[],
528                  struct rte_flow_error *e)
529 {
530         int ret;
531
532         /* parse user request into ch_filter_specification */
533         ret = cxgbe_rtef_parse_attr(flow, attr, e);
534         if (ret)
535                 return ret;
536         ret = cxgbe_rtef_parse_items(flow, item, e);
537         if (ret)
538                 return ret;
539         return cxgbe_rtef_parse_actions(flow, action, e);
540 }
541
542 static int __cxgbe_flow_create(struct rte_eth_dev *dev, struct rte_flow *flow)
543 {
544         struct ch_filter_specification *fs = &flow->fs;
545         struct adapter *adap = ethdev2adap(dev);
546         struct tid_info *t = &adap->tids;
547         struct filter_ctx ctx;
548         unsigned int fidx;
549         int err;
550
551         if (cxgbe_get_fidx(flow, &fidx))
552                 return -ENOMEM;
553         if (cxgbe_verify_fidx(flow, fidx, 0))
554                 return -1;
555
556         t4_init_completion(&ctx.completion);
557         /* go create the filter */
558         err = cxgbe_set_filter(dev, fidx, fs, &ctx);
559         if (err) {
560                 dev_err(adap, "Error %d while creating filter.\n", err);
561                 return err;
562         }
563
564         /* Poll the FW for reply */
565         err = cxgbe_poll_for_completion(&adap->sge.fw_evtq,
566                                         CXGBE_FLOW_POLL_US,
567                                         CXGBE_FLOW_POLL_CNT,
568                                         &ctx.completion);
569         if (err) {
570                 dev_err(adap, "Filter set operation timed out (%d)\n", err);
571                 return err;
572         }
573         if (ctx.result) {
574                 dev_err(adap, "Hardware error %d while creating the filter.\n",
575                         ctx.result);
576                 return ctx.result;
577         }
578
579         if (fs->cap) { /* to destroy the filter */
580                 flow->fidx = ctx.tid;
581                 flow->f = lookup_tid(t, ctx.tid);
582         } else {
583                 flow->fidx = fidx;
584                 flow->f = &adap->tids.ftid_tab[fidx];
585         }
586
587         return 0;
588 }
589
590 static struct rte_flow *
591 cxgbe_flow_create(struct rte_eth_dev *dev,
592                   const struct rte_flow_attr *attr,
593                   const struct rte_flow_item item[],
594                   const struct rte_flow_action action[],
595                   struct rte_flow_error *e)
596 {
597         struct rte_flow *flow;
598         int ret;
599
600         flow = t4_os_alloc(sizeof(struct rte_flow));
601         if (!flow) {
602                 rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
603                                    NULL, "Unable to allocate memory for"
604                                    " filter_entry");
605                 return NULL;
606         }
607
608         flow->item_parser = parseitem;
609         flow->dev = dev;
610
611         if (cxgbe_flow_parse(flow, attr, item, action, e)) {
612                 t4_os_free(flow);
613                 return NULL;
614         }
615
616         /* go, interact with cxgbe_filter */
617         ret = __cxgbe_flow_create(dev, flow);
618         if (ret) {
619                 rte_flow_error_set(e, ret, RTE_FLOW_ERROR_TYPE_HANDLE,
620                                    NULL, "Unable to create flow rule");
621                 t4_os_free(flow);
622                 return NULL;
623         }
624
625         flow->f->private = flow; /* Will be used during flush */
626
627         return flow;
628 }
629
630 static int __cxgbe_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
631 {
632         struct adapter *adap = ethdev2adap(dev);
633         struct filter_entry *f = flow->f;
634         struct ch_filter_specification *fs;
635         struct filter_ctx ctx;
636         int err;
637
638         fs = &f->fs;
639         if (cxgbe_verify_fidx(flow, flow->fidx, 1))
640                 return -1;
641
642         t4_init_completion(&ctx.completion);
643         err = cxgbe_del_filter(dev, flow->fidx, fs, &ctx);
644         if (err) {
645                 dev_err(adap, "Error %d while deleting filter.\n", err);
646                 return err;
647         }
648
649         /* Poll the FW for reply */
650         err = cxgbe_poll_for_completion(&adap->sge.fw_evtq,
651                                         CXGBE_FLOW_POLL_US,
652                                         CXGBE_FLOW_POLL_CNT,
653                                         &ctx.completion);
654         if (err) {
655                 dev_err(adap, "Filter delete operation timed out (%d)\n", err);
656                 return err;
657         }
658         if (ctx.result) {
659                 dev_err(adap, "Hardware error %d while deleting the filter.\n",
660                         ctx.result);
661                 return ctx.result;
662         }
663
664         return 0;
665 }
666
667 static int
668 cxgbe_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
669                    struct rte_flow_error *e)
670 {
671         int ret;
672
673         ret = __cxgbe_flow_destroy(dev, flow);
674         if (ret)
675                 return rte_flow_error_set(e, ret, RTE_FLOW_ERROR_TYPE_HANDLE,
676                                           flow, "error destroying filter.");
677         t4_os_free(flow);
678         return 0;
679 }
680
681 static int __cxgbe_flow_query(struct rte_flow *flow, u64 *count,
682                               u64 *byte_count)
683 {
684         struct adapter *adap = ethdev2adap(flow->dev);
685         struct ch_filter_specification fs = flow->f->fs;
686         unsigned int fidx = flow->fidx;
687         int ret = 0;
688
689         ret = cxgbe_get_filter_count(adap, fidx, count, fs.cap, 0);
690         if (ret)
691                 return ret;
692         return cxgbe_get_filter_count(adap, fidx, byte_count, fs.cap, 1);
693 }
694
695 static int
696 cxgbe_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
697                  const struct rte_flow_action *action, void *data,
698                  struct rte_flow_error *e)
699 {
700         struct ch_filter_specification fs;
701         struct rte_flow_query_count *c;
702         struct filter_entry *f;
703         int ret;
704
705         RTE_SET_USED(dev);
706
707         f = flow->f;
708         fs = f->fs;
709
710         if (action->type != RTE_FLOW_ACTION_TYPE_COUNT)
711                 return rte_flow_error_set(e, ENOTSUP,
712                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
713                                           "only count supported for query");
714
715         /*
716          * This is a valid operation, Since we are allowed to do chelsio
717          * specific operations in rte side of our code but not vise-versa
718          *
719          * So, fs can be queried/modified here BUT rte_flow_query_count
720          * cannot be worked on by the lower layer since we want to maintain
721          * it as rte_flow agnostic.
722          */
723         if (!fs.hitcnts)
724                 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
725                                           &fs, "filter hit counters were not"
726                                           " enabled during filter creation");
727
728         c = (struct rte_flow_query_count *)data;
729         ret = __cxgbe_flow_query(flow, &c->hits, &c->bytes);
730         if (ret)
731                 return rte_flow_error_set(e, -ret, RTE_FLOW_ERROR_TYPE_ACTION,
732                                           f, "cxgbe pmd failed to"
733                                           " perform query");
734
735         /* Query was successful */
736         c->bytes_set = 1;
737         c->hits_set = 1;
738
739         return 0; /* success / partial_success */
740 }
741
742 static int
743 cxgbe_flow_validate(struct rte_eth_dev *dev,
744                     const struct rte_flow_attr *attr,
745                     const struct rte_flow_item item[],
746                     const struct rte_flow_action action[],
747                     struct rte_flow_error *e)
748 {
749         struct adapter *adap = ethdev2adap(dev);
750         struct rte_flow *flow;
751         unsigned int fidx;
752         int ret;
753
754         flow = t4_os_alloc(sizeof(struct rte_flow));
755         if (!flow)
756                 return rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
757                                 NULL,
758                                 "Unable to allocate memory for filter_entry");
759
760         flow->item_parser = parseitem;
761         flow->dev = dev;
762
763         ret = cxgbe_flow_parse(flow, attr, item, action, e);
764         if (ret) {
765                 t4_os_free(flow);
766                 return ret;
767         }
768
769         if (validate_filter(adap, &flow->fs)) {
770                 t4_os_free(flow);
771                 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
772                                 NULL,
773                                 "validation failed. Check f/w config file.");
774         }
775
776         if (cxgbe_get_fidx(flow, &fidx)) {
777                 t4_os_free(flow);
778                 return rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
779                                           NULL, "no memory in tcam.");
780         }
781
782         if (cxgbe_verify_fidx(flow, fidx, 0)) {
783                 t4_os_free(flow);
784                 return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
785                                           NULL, "validation failed");
786         }
787
788         t4_os_free(flow);
789         return 0;
790 }
791
792 /*
793  * @ret : > 0 filter destroyed succsesfully
794  *        < 0 error destroying filter
795  *        == 1 filter not active / not found
796  */
797 static int
798 cxgbe_check_n_destroy(struct filter_entry *f, struct rte_eth_dev *dev,
799                       struct rte_flow_error *e)
800 {
801         if (f && (f->valid || f->pending) &&
802             f->dev == dev && /* Only if user has asked for this port */
803              f->private) /* We (rte_flow) created this filter */
804                 return cxgbe_flow_destroy(dev, (struct rte_flow *)f->private,
805                                           e);
806         return 1;
807 }
808
809 static int cxgbe_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *e)
810 {
811         struct adapter *adap = ethdev2adap(dev);
812         unsigned int i;
813         int ret = 0;
814
815         if (adap->tids.ftid_tab) {
816                 struct filter_entry *f = &adap->tids.ftid_tab[0];
817
818                 for (i = 0; i < adap->tids.nftids; i++, f++) {
819                         ret = cxgbe_check_n_destroy(f, dev, e);
820                         if (ret < 0)
821                                 goto out;
822                 }
823         }
824
825         if (is_hashfilter(adap) && adap->tids.tid_tab) {
826                 struct filter_entry *f;
827
828                 for (i = adap->tids.hash_base; i <= adap->tids.ntids; i++) {
829                         f = (struct filter_entry *)adap->tids.tid_tab[i];
830
831                         ret = cxgbe_check_n_destroy(f, dev, e);
832                         if (ret < 0)
833                                 goto out;
834                 }
835         }
836
837 out:
838         return ret >= 0 ? 0 : ret;
839 }
840
841 static const struct rte_flow_ops cxgbe_flow_ops = {
842         .validate       = cxgbe_flow_validate,
843         .create         = cxgbe_flow_create,
844         .destroy        = cxgbe_flow_destroy,
845         .flush          = cxgbe_flow_flush,
846         .query          = cxgbe_flow_query,
847         .isolate        = NULL,
848 };
849
850 int
851 cxgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
852                       enum rte_filter_type filter_type,
853                       enum rte_filter_op filter_op,
854                       void *arg)
855 {
856         int ret = 0;
857
858         RTE_SET_USED(dev);
859         switch (filter_type) {
860         case RTE_ETH_FILTER_GENERIC:
861                 if (filter_op != RTE_ETH_FILTER_GET)
862                         return -EINVAL;
863                 *(const void **)arg = &cxgbe_flow_ops;
864                 break;
865         default:
866                 ret = -ENOTSUP;
867                 break;
868         }
869         return ret;
870 }