143754a454ab8751a5a424214113a8b49089d735
[dpdk.git] / lib / librte_pmd_i40e / i40e_fdir.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <sys/queue.h>
35 #include <stdio.h>
36 #include <errno.h>
37 #include <stdint.h>
38 #include <string.h>
39 #include <unistd.h>
40 #include <stdarg.h>
41
42 #include <rte_ether.h>
43 #include <rte_ethdev.h>
44 #include <rte_log.h>
45 #include <rte_memzone.h>
46 #include <rte_malloc.h>
47 #include <rte_ip.h>
48 #include <rte_udp.h>
49 #include <rte_tcp.h>
50 #include <rte_sctp.h>
51
52 #include "i40e_logs.h"
53 #include "i40e/i40e_type.h"
54 #include "i40e_ethdev.h"
55 #include "i40e_rxtx.h"
56
57 #define I40E_FDIR_MZ_NAME          "FDIR_MEMZONE"
58 #ifndef IPV6_ADDR_LEN
59 #define IPV6_ADDR_LEN              16
60 #endif
61
62 #define I40E_FDIR_PKT_LEN                   512
63 #define I40E_FDIR_IP_DEFAULT_LEN            420
64 #define I40E_FDIR_IP_DEFAULT_TTL            0x40
65 #define I40E_FDIR_IP_DEFAULT_VERSION_IHL    0x45
66 #define I40E_FDIR_TCP_DEFAULT_DATAOFF       0x50
67 #define I40E_FDIR_IPv6_DEFAULT_VTC_FLOW     0x60300000
68 #define I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS   0xFF
69 #define I40E_FDIR_IPv6_PAYLOAD_LEN          380
70 #define I40E_FDIR_UDP_DEFAULT_LEN           400
71
72 /* Wait count and interval for fdir filter programming */
73 #define I40E_FDIR_WAIT_COUNT       10
74 #define I40E_FDIR_WAIT_INTERVAL_US 1000
75
76 /* Wait count and interval for fdir filter flush */
77 #define I40E_FDIR_FLUSH_RETRY       50
78 #define I40E_FDIR_FLUSH_INTERVAL_MS 5
79
80 #define I40E_COUNTER_PF           2
81 /* Statistic counter index for one pf */
82 #define I40E_COUNTER_INDEX_FDIR(pf_id)   (0 + (pf_id) * I40E_COUNTER_PF)
83 #define I40E_FLX_OFFSET_IN_FIELD_VECTOR   50
84
85 static int i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq);
86 static int i40e_fdir_construct_pkt(struct i40e_pf *pf,
87                                      const struct rte_eth_fdir_input *fdir_input,
88                                      unsigned char *raw_pkt);
89 static int i40e_add_del_fdir_filter(struct rte_eth_dev *dev,
90                             const struct rte_eth_fdir_filter *filter,
91                             bool add);
92 static int i40e_fdir_filter_programming(struct i40e_pf *pf,
93                         enum i40e_filter_pctype pctype,
94                         const struct rte_eth_fdir_filter *filter,
95                         bool add);
96 static int i40e_fdir_flush(struct rte_eth_dev *dev);
97
98 static int
99 i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq)
100 {
101         struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
102         struct i40e_hmc_obj_rxq rx_ctx;
103         int err = I40E_SUCCESS;
104
105         memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq));
106         /* Init the RX queue in hardware */
107         rx_ctx.dbuff = I40E_RXBUF_SZ_1024 >> I40E_RXQ_CTX_DBUFF_SHIFT;
108         rx_ctx.hbuff = 0;
109         rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
110         rx_ctx.qlen = rxq->nb_rx_desc;
111 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
112         rx_ctx.dsize = 1;
113 #endif
114         rx_ctx.dtype = i40e_header_split_none;
115         rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE;
116         rx_ctx.rxmax = ETHER_MAX_LEN;
117         rx_ctx.tphrdesc_ena = 1;
118         rx_ctx.tphwdesc_ena = 1;
119         rx_ctx.tphdata_ena = 1;
120         rx_ctx.tphhead_ena = 1;
121         rx_ctx.lrxqthresh = 2;
122         rx_ctx.crcstrip = 0;
123         rx_ctx.l2tsel = 1;
124         rx_ctx.showiv = 1;
125         rx_ctx.prefena = 1;
126
127         err = i40e_clear_lan_rx_queue_context(hw, rxq->reg_idx);
128         if (err != I40E_SUCCESS) {
129                 PMD_DRV_LOG(ERR, "Failed to clear FDIR RX queue context.");
130                 return err;
131         }
132         err = i40e_set_lan_rx_queue_context(hw, rxq->reg_idx, &rx_ctx);
133         if (err != I40E_SUCCESS) {
134                 PMD_DRV_LOG(ERR, "Failed to set FDIR RX queue context.");
135                 return err;
136         }
137         rxq->qrx_tail = hw->hw_addr +
138                 I40E_QRX_TAIL(rxq->vsi->base_queue);
139
140         rte_wmb();
141         /* Init the RX tail regieter. */
142         I40E_PCI_REG_WRITE(rxq->qrx_tail, 0);
143         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
144
145         return err;
146 }
147
148 /*
149  * i40e_fdir_setup - reserve and initialize the Flow Director resources
150  * @pf: board private structure
151  */
152 int
153 i40e_fdir_setup(struct i40e_pf *pf)
154 {
155         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
156         struct i40e_vsi *vsi;
157         int err = I40E_SUCCESS;
158         char z_name[RTE_MEMZONE_NAMESIZE];
159         const struct rte_memzone *mz = NULL;
160         struct rte_eth_dev *eth_dev = pf->adapter->eth_dev;
161
162         PMD_DRV_LOG(INFO, "FDIR HW Capabilities: num_filters_guaranteed = %u,"
163                         " num_filters_best_effort = %u.",
164                         hw->func_caps.fd_filters_guaranteed,
165                         hw->func_caps.fd_filters_best_effort);
166
167         vsi = pf->fdir.fdir_vsi;
168         if (vsi) {
169                 PMD_DRV_LOG(ERR, "FDIR vsi pointer needs "
170                                  "to be null before creation.");
171                 return I40E_ERR_BAD_PTR;
172         }
173         /* make new FDIR VSI */
174         vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->main_vsi, 0);
175         if (!vsi) {
176                 PMD_DRV_LOG(ERR, "Couldn't create FDIR VSI.");
177                 return I40E_ERR_NO_AVAILABLE_VSI;
178         }
179         pf->fdir.fdir_vsi = vsi;
180
181         /*Fdir tx queue setup*/
182         err = i40e_fdir_setup_tx_resources(pf);
183         if (err) {
184                 PMD_DRV_LOG(ERR, "Failed to setup FDIR TX resources.");
185                 goto fail_setup_tx;
186         }
187
188         /*Fdir rx queue setup*/
189         err = i40e_fdir_setup_rx_resources(pf);
190         if (err) {
191                 PMD_DRV_LOG(ERR, "Failed to setup FDIR RX resources.");
192                 goto fail_setup_rx;
193         }
194
195         err = i40e_tx_queue_init(pf->fdir.txq);
196         if (err) {
197                 PMD_DRV_LOG(ERR, "Failed to do FDIR TX initialization.");
198                 goto fail_mem;
199         }
200
201         /* need switch on before dev start*/
202         err = i40e_switch_tx_queue(hw, vsi->base_queue, TRUE);
203         if (err) {
204                 PMD_DRV_LOG(ERR, "Failed to do fdir TX switch on.");
205                 goto fail_mem;
206         }
207
208         /* Init the rx queue in hardware */
209         err = i40e_fdir_rx_queue_init(pf->fdir.rxq);
210         if (err) {
211                 PMD_DRV_LOG(ERR, "Failed to do FDIR RX initialization.");
212                 goto fail_mem;
213         }
214
215         /* switch on rx queue */
216         err = i40e_switch_rx_queue(hw, vsi->base_queue, TRUE);
217         if (err) {
218                 PMD_DRV_LOG(ERR, "Failed to do FDIR RX switch on.");
219                 goto fail_mem;
220         }
221
222         /* reserve memory for the fdir programming packet */
223         snprintf(z_name, sizeof(z_name), "%s_%s_%d",
224                         eth_dev->driver->pci_drv.name,
225                         I40E_FDIR_MZ_NAME,
226                         eth_dev->data->port_id);
227         mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN, SOCKET_ID_ANY);
228         if (!mz) {
229                 PMD_DRV_LOG(ERR, "Cannot init memzone for "
230                                  "flow director program packet.");
231                 err = I40E_ERR_NO_MEMORY;
232                 goto fail_mem;
233         }
234         pf->fdir.prg_pkt = mz->addr;
235 #ifdef RTE_LIBRTE_XEN_DOM0
236         pf->fdir.dma_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
237 #else
238         pf->fdir.dma_addr = (uint64_t)mz->phys_addr;
239 #endif
240         pf->fdir.match_counter_index = I40E_COUNTER_INDEX_FDIR(hw->pf_id);
241         PMD_DRV_LOG(INFO, "FDIR setup successfully, with programming queue %u.",
242                     vsi->base_queue);
243         return I40E_SUCCESS;
244
245 fail_mem:
246         i40e_dev_rx_queue_release(pf->fdir.rxq);
247         pf->fdir.rxq = NULL;
248 fail_setup_rx:
249         i40e_dev_tx_queue_release(pf->fdir.txq);
250         pf->fdir.txq = NULL;
251 fail_setup_tx:
252         i40e_vsi_release(vsi);
253         pf->fdir.fdir_vsi = NULL;
254         return err;
255 }
256
257 /*
258  * i40e_fdir_teardown - release the Flow Director resources
259  * @pf: board private structure
260  */
261 void
262 i40e_fdir_teardown(struct i40e_pf *pf)
263 {
264         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
265         struct i40e_vsi *vsi;
266
267         vsi = pf->fdir.fdir_vsi;
268         i40e_switch_tx_queue(hw, vsi->base_queue, FALSE);
269         i40e_switch_rx_queue(hw, vsi->base_queue, FALSE);
270         i40e_dev_rx_queue_release(pf->fdir.rxq);
271         pf->fdir.rxq = NULL;
272         i40e_dev_tx_queue_release(pf->fdir.txq);
273         pf->fdir.txq = NULL;
274         i40e_vsi_release(vsi);
275         pf->fdir.fdir_vsi = NULL;
276 }
277
278 /* check whether the flow director table in empty */
279 static inline int
280 i40e_fdir_empty(struct i40e_hw *hw)
281 {
282         uint32_t guarant_cnt, best_cnt;
283
284         guarant_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
285                                  I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
286                                  I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
287         best_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
288                               I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
289                               I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
290         if (best_cnt + guarant_cnt > 0)
291                 return -1;
292
293         return 0;
294 }
295
296 /*
297  * Initialize the configuration about bytes stream extracted as flexible payload
298  * and mask setting
299  */
300 static inline void
301 i40e_init_flx_pld(struct i40e_pf *pf)
302 {
303         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
304         uint8_t pctype;
305         int i, index;
306
307         /*
308          * Define the bytes stream extracted as flexible payload in
309          * field vector. By default, select 8 words from the beginning
310          * of payload as flexible payload.
311          */
312         for (i = I40E_FLXPLD_L2_IDX; i < I40E_MAX_FLXPLD_LAYER; i++) {
313                 index = i * I40E_MAX_FLXPLD_FIED;
314                 pf->fdir.flex_set[index].src_offset = 0;
315                 pf->fdir.flex_set[index].size = I40E_FDIR_MAX_FLEXWORD_NUM;
316                 pf->fdir.flex_set[index].dst_offset = 0;
317                 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(index), 0x0000C900);
318                 I40E_WRITE_REG(hw,
319                         I40E_PRTQF_FLX_PIT(index + 1), 0x0000FC29);/*non-used*/
320                 I40E_WRITE_REG(hw,
321                         I40E_PRTQF_FLX_PIT(index + 2), 0x0000FC2A);/*non-used*/
322         }
323
324         /* initialize the masks */
325         for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
326              pctype <= I40E_FILTER_PCTYPE_FRAG_IPV6; pctype++) {
327                 pf->fdir.flex_mask[pctype].word_mask = 0;
328                 I40E_WRITE_REG(hw, I40E_PRTQF_FD_FLXINSET(pctype), 0);
329                 for (i = 0; i < I40E_FDIR_BITMASK_NUM_WORD; i++) {
330                         pf->fdir.flex_mask[pctype].bitmask[i].offset = 0;
331                         pf->fdir.flex_mask[pctype].bitmask[i].mask = 0;
332                         I40E_WRITE_REG(hw, I40E_PRTQF_FD_MSK(pctype, i), 0);
333                 }
334         }
335 }
336
337 /*
338  * Configure flow director related setting
339  */
340 int
341 i40e_fdir_configure(struct rte_eth_dev *dev)
342 {
343         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
344         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
345         uint32_t val;
346         int ret = 0;
347
348         /*
349         * configuration need to be done before
350         * flow director filters are added
351         * If filters exist, flush them.
352         */
353         if (i40e_fdir_empty(hw) < 0) {
354                 ret = i40e_fdir_flush(dev);
355                 if (ret) {
356                         PMD_DRV_LOG(ERR, "failed to flush fdir table.");
357                         return ret;
358                 }
359         }
360
361         val = I40E_READ_REG(hw, I40E_PFQF_CTL_0);
362         if ((pf->flags & I40E_FLAG_FDIR) &&
363                 dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) {
364                 /* enable FDIR filter */
365                 val |= I40E_PFQF_CTL_0_FD_ENA_MASK;
366                 I40E_WRITE_REG(hw, I40E_PFQF_CTL_0, val);
367
368                 i40e_init_flx_pld(pf); /* set flex config to default value */
369         } else {
370                 /* disable FDIR filter */
371                 val &= ~I40E_PFQF_CTL_0_FD_ENA_MASK;
372                 I40E_WRITE_REG(hw, I40E_PFQF_CTL_0, val);
373                 pf->flags &= ~I40E_FLAG_FDIR;
374         }
375
376         return ret;
377 }
378
379 static inline void
380 i40e_fdir_fill_eth_ip_head(const struct rte_eth_fdir_input *fdir_input,
381                                unsigned char *raw_pkt)
382 {
383         struct ether_hdr *ether = (struct ether_hdr *)raw_pkt;
384         struct ipv4_hdr *ip;
385         struct ipv6_hdr *ip6;
386         static const uint8_t next_proto[] = {
387                 [RTE_ETH_FLOW_TYPE_UDPV4] = IPPROTO_UDP,
388                 [RTE_ETH_FLOW_TYPE_TCPV4] = IPPROTO_TCP,
389                 [RTE_ETH_FLOW_TYPE_SCTPV4] = IPPROTO_SCTP,
390                 [RTE_ETH_FLOW_TYPE_IPV4_OTHER] = IPPROTO_IP,
391                 [RTE_ETH_FLOW_TYPE_FRAG_IPV4] = IPPROTO_IP,
392                 [RTE_ETH_FLOW_TYPE_UDPV6] = IPPROTO_UDP,
393                 [RTE_ETH_FLOW_TYPE_TCPV6] = IPPROTO_TCP,
394                 [RTE_ETH_FLOW_TYPE_SCTPV6] = IPPROTO_SCTP,
395                 [RTE_ETH_FLOW_TYPE_IPV6_OTHER] = IPPROTO_NONE,
396                 [RTE_ETH_FLOW_TYPE_FRAG_IPV6] = IPPROTO_NONE,
397         };
398
399         switch (fdir_input->flow_type) {
400         case RTE_ETH_FLOW_TYPE_UDPV4:
401         case RTE_ETH_FLOW_TYPE_TCPV4:
402         case RTE_ETH_FLOW_TYPE_SCTPV4:
403         case RTE_ETH_FLOW_TYPE_IPV4_OTHER:
404         case RTE_ETH_FLOW_TYPE_FRAG_IPV4:
405                 ip = (struct ipv4_hdr *)(raw_pkt + sizeof(struct ether_hdr));
406
407                 ether->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4);
408                 ip->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL;
409                 /* set len to by default */
410                 ip->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN);
411                 ip->time_to_live = I40E_FDIR_IP_DEFAULT_TTL;
412                 /*
413                  * The source and destination fields in the transmitted packet
414                  * need to be presented in a reversed order with respect
415                  * to the expected received packets.
416                  */
417                 ip->src_addr = fdir_input->flow.ip4_flow.dst_ip;
418                 ip->dst_addr = fdir_input->flow.ip4_flow.src_ip;
419                 ip->next_proto_id = next_proto[fdir_input->flow_type];
420                 break;
421         case RTE_ETH_FLOW_TYPE_UDPV6:
422         case RTE_ETH_FLOW_TYPE_TCPV6:
423         case RTE_ETH_FLOW_TYPE_SCTPV6:
424         case RTE_ETH_FLOW_TYPE_IPV6_OTHER:
425         case RTE_ETH_FLOW_TYPE_FRAG_IPV6:
426                 ip6 = (struct ipv6_hdr *)(raw_pkt + sizeof(struct ether_hdr));
427
428                 ether->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6);
429                 ip6->vtc_flow =
430                         rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW);
431                 ip6->payload_len =
432                         rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
433                 ip6->hop_limits = I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
434
435                 /*
436                  * The source and destination fields in the transmitted packet
437                  * need to be presented in a reversed order with respect
438                  * to the expected received packets.
439                  */
440                 rte_memcpy(&(ip6->src_addr),
441                            &(fdir_input->flow.ip6_flow.dst_ip),
442                            IPV6_ADDR_LEN);
443                 rte_memcpy(&(ip6->dst_addr),
444                            &(fdir_input->flow.ip6_flow.src_ip),
445                            IPV6_ADDR_LEN);
446                 ip6->proto = next_proto[fdir_input->flow_type];
447                 break;
448         default:
449                 PMD_DRV_LOG(ERR, "unknown flow type %u.",
450                             fdir_input->flow_type);
451                 break;
452         }
453 }
454
455
456 /*
457  * i40e_fdir_construct_pkt - construct packet based on fields in input
458  * @pf: board private structure
459  * @fdir_input: input set of the flow director entry
460  * @raw_pkt: a packet to be constructed
461  */
462 static int
463 i40e_fdir_construct_pkt(struct i40e_pf *pf,
464                              const struct rte_eth_fdir_input *fdir_input,
465                              unsigned char *raw_pkt)
466 {
467         unsigned char *payload, *ptr;
468         struct udp_hdr *udp;
469         struct tcp_hdr *tcp;
470         struct sctp_hdr *sctp;
471         uint8_t size, dst = 0;
472         uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/
473
474         /* fill the ethernet and IP head */
475         i40e_fdir_fill_eth_ip_head(fdir_input, raw_pkt);
476
477         /* fill the L4 head */
478         switch (fdir_input->flow_type) {
479         case RTE_ETH_FLOW_TYPE_UDPV4:
480                 udp = (struct udp_hdr *)(raw_pkt + sizeof(struct ether_hdr) +
481                                 sizeof(struct ipv4_hdr));
482                 payload = (unsigned char *)udp + sizeof(struct udp_hdr);
483                 /*
484                  * The source and destination fields in the transmitted packet
485                  * need to be presented in a reversed order with respect
486                  * to the expected received packets.
487                  */
488                 udp->src_port = fdir_input->flow.udp4_flow.dst_port;
489                 udp->dst_port = fdir_input->flow.udp4_flow.src_port;
490                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
491                 break;
492
493         case RTE_ETH_FLOW_TYPE_TCPV4:
494                 tcp = (struct tcp_hdr *)(raw_pkt + sizeof(struct ether_hdr) +
495                                          sizeof(struct ipv4_hdr));
496                 payload = (unsigned char *)tcp + sizeof(struct tcp_hdr);
497                 /*
498                  * The source and destination fields in the transmitted packet
499                  * need to be presented in a reversed order with respect
500                  * to the expected received packets.
501                  */
502                 tcp->src_port = fdir_input->flow.tcp4_flow.dst_port;
503                 tcp->dst_port = fdir_input->flow.tcp4_flow.src_port;
504                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
505                 break;
506
507         case RTE_ETH_FLOW_TYPE_SCTPV4:
508                 sctp = (struct sctp_hdr *)(raw_pkt + sizeof(struct ether_hdr) +
509                                            sizeof(struct ipv4_hdr));
510                 payload = (unsigned char *)sctp + sizeof(struct sctp_hdr);
511                 sctp->tag = fdir_input->flow.sctp4_flow.verify_tag;
512                 break;
513
514         case RTE_ETH_FLOW_TYPE_IPV4_OTHER:
515         case RTE_ETH_FLOW_TYPE_FRAG_IPV4:
516                 payload = raw_pkt + sizeof(struct ether_hdr) +
517                           sizeof(struct ipv4_hdr);
518                 set_idx = I40E_FLXPLD_L3_IDX;
519                 break;
520
521         case RTE_ETH_FLOW_TYPE_UDPV6:
522                 udp = (struct udp_hdr *)(raw_pkt + sizeof(struct ether_hdr) +
523                                          sizeof(struct ipv6_hdr));
524                 payload = (unsigned char *)udp + sizeof(struct udp_hdr);
525                 /*
526                  * The source and destination fields in the transmitted packet
527                  * need to be presented in a reversed order with respect
528                  * to the expected received packets.
529                  */
530                 udp->src_port = fdir_input->flow.udp6_flow.dst_port;
531                 udp->dst_port = fdir_input->flow.udp6_flow.src_port;
532                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
533                 break;
534
535         case RTE_ETH_FLOW_TYPE_TCPV6:
536                 tcp = (struct tcp_hdr *)(raw_pkt + sizeof(struct ether_hdr) +
537                                          sizeof(struct ipv6_hdr));
538                 payload = (unsigned char *)tcp + sizeof(struct tcp_hdr);
539                 /*
540                  * The source and destination fields in the transmitted packet
541                  * need to be presented in a reversed order with respect
542                  * to the expected received packets.
543                  */
544                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
545                 tcp->src_port = fdir_input->flow.udp6_flow.dst_port;
546                 tcp->dst_port = fdir_input->flow.udp6_flow.src_port;
547                 break;
548
549         case RTE_ETH_FLOW_TYPE_SCTPV6:
550                 sctp = (struct sctp_hdr *)(raw_pkt + sizeof(struct ether_hdr) +
551                                            sizeof(struct ipv6_hdr));
552                 payload = (unsigned char *)sctp + sizeof(struct sctp_hdr);
553                 sctp->tag = fdir_input->flow.sctp6_flow.verify_tag;
554                 break;
555
556         case RTE_ETH_FLOW_TYPE_IPV6_OTHER:
557         case RTE_ETH_FLOW_TYPE_FRAG_IPV6:
558                 payload = raw_pkt + sizeof(struct ether_hdr) +
559                           sizeof(struct ipv6_hdr);
560                 set_idx = I40E_FLXPLD_L3_IDX;
561                 break;
562         default:
563                 PMD_DRV_LOG(ERR, "unknown flow type %u.", fdir_input->flow_type);
564                 return -EINVAL;
565         }
566
567         /* fill the flexbytes to payload */
568         for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
569                 pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i;
570                 size = pf->fdir.flex_set[pit_idx].size;
571                 if (size == 0)
572                         continue;
573                 dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t);
574                 ptr = payload +
575                         pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t);
576                 (void)rte_memcpy(ptr,
577                                  &fdir_input->flow_ext.flexbytes[dst],
578                                  size * sizeof(uint16_t));
579         }
580
581         return 0;
582 }
583
584 /* Construct the tx flags */
585 static inline uint64_t
586 i40e_build_ctob(uint32_t td_cmd,
587                 uint32_t td_offset,
588                 unsigned int size,
589                 uint32_t td_tag)
590 {
591         return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA |
592                         ((uint64_t)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
593                         ((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
594                         ((uint64_t)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
595                         ((uint64_t)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
596 }
597
598 /*
599  * check the programming status descriptor in rx queue.
600  * done after Programming Flow Director is programmed on
601  * tx queue
602  */
603 static inline int
604 i40e_check_fdir_programming_status(struct i40e_rx_queue *rxq)
605 {
606         volatile union i40e_rx_desc *rxdp;
607         uint64_t qword1;
608         uint32_t rx_status;
609         uint32_t len, id;
610         uint32_t error;
611         int ret = 0;
612
613         rxdp = &rxq->rx_ring[rxq->rx_tail];
614         qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
615         rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK)
616                         >> I40E_RXD_QW1_STATUS_SHIFT;
617
618         if (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
619                 len = qword1 >> I40E_RX_PROG_STATUS_DESC_LENGTH_SHIFT;
620                 id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
621                             I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
622
623                 if (len  == I40E_RX_PROG_STATUS_DESC_LENGTH &&
624                     id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) {
625                         error = (qword1 &
626                                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
627                                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
628                         if (error == (0x1 <<
629                                 I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
630                                 PMD_DRV_LOG(ERR, "Failed to add FDIR filter"
631                                             " (FD_ID %u): programming status"
632                                             " reported.",
633                                             rxdp->wb.qword0.hi_dword.fd_id);
634                                 ret = -1;
635                         } else if (error == (0x1 <<
636                                 I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
637                                 PMD_DRV_LOG(ERR, "Failed to delete FDIR filter"
638                                             " (FD_ID %u): programming status"
639                                             " reported.",
640                                             rxdp->wb.qword0.hi_dword.fd_id);
641                                 ret = -1;
642                         } else
643                                 PMD_DRV_LOG(ERR, "invalid programming status"
644                                             " reported, error = %u.", error);
645                 } else
646                         PMD_DRV_LOG(ERR, "unknown programming status"
647                                     " reported, len = %d, id = %u.", len, id);
648                 rxdp->wb.qword1.status_error_len = 0;
649                 rxq->rx_tail++;
650                 if (unlikely(rxq->rx_tail == rxq->nb_rx_desc))
651                         rxq->rx_tail = 0;
652         }
653         return ret;
654 }
655
656 /*
657  * i40e_add_del_fdir_filter - add or remove a flow director filter.
658  * @pf: board private structure
659  * @filter: fdir filter entry
660  * @add: 0 - delete, 1 - add
661  */
662 static int
663 i40e_add_del_fdir_filter(struct rte_eth_dev *dev,
664                             const struct rte_eth_fdir_filter *filter,
665                             bool add)
666 {
667         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
668         unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
669         enum i40e_filter_pctype pctype;
670         int ret = 0;
671
672         if (!(pf->flags & I40E_FLAG_FDIR)) {
673                 PMD_DRV_LOG(ERR, "FDIR is not enabled.");
674                 return -ENOTSUP;
675         }
676         if (!I40E_VALID_FLOW_TYPE(filter->input.flow_type)) {
677                 PMD_DRV_LOG(ERR, "invalid flow_type input.");
678                 return -EINVAL;
679         }
680         if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) {
681                 PMD_DRV_LOG(ERR, "Invalid queue ID");
682                 return -EINVAL;
683         }
684
685         memset(pkt, 0, I40E_FDIR_PKT_LEN);
686
687         ret = i40e_fdir_construct_pkt(pf, &filter->input, pkt);
688         if (ret < 0) {
689                 PMD_DRV_LOG(ERR, "construct packet for fdir fails.");
690                 return ret;
691         }
692         pctype = i40e_flowtype_to_pctype(filter->input.flow_type);
693         ret = i40e_fdir_filter_programming(pf, pctype, filter, add);
694         if (ret < 0) {
695                 PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).",
696                             pctype);
697                 return ret;
698         }
699         return ret;
700 }
701
702 /*
703  * i40e_fdir_filter_programming - Program a flow director filter rule.
704  * Is done by Flow Director Programming Descriptor followed by packet
705  * structure that contains the filter fields need to match.
706  * @pf: board private structure
707  * @pctype: pctype
708  * @filter: fdir filter entry
709  * @add: 0 - delelet, 1 - add
710  */
711 static int
712 i40e_fdir_filter_programming(struct i40e_pf *pf,
713                         enum i40e_filter_pctype pctype,
714                         const struct rte_eth_fdir_filter *filter,
715                         bool add)
716 {
717         struct i40e_tx_queue *txq = pf->fdir.txq;
718         struct i40e_rx_queue *rxq = pf->fdir.rxq;
719         const struct rte_eth_fdir_action *fdir_action = &filter->action;
720         volatile struct i40e_tx_desc *txdp;
721         volatile struct i40e_filter_program_desc *fdirdp;
722         uint32_t td_cmd;
723         uint16_t i;
724         uint8_t dest;
725
726         PMD_DRV_LOG(INFO, "filling filter programming descriptor.");
727         fdirdp = (volatile struct i40e_filter_program_desc *)
728                         (&(txq->tx_ring[txq->tx_tail]));
729
730         fdirdp->qindex_flex_ptype_vsi =
731                         rte_cpu_to_le_32((fdir_action->rx_queue <<
732                                           I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
733                                           I40E_TXD_FLTR_QW0_QINDEX_MASK);
734
735         fdirdp->qindex_flex_ptype_vsi |=
736                         rte_cpu_to_le_32((fdir_action->flex_off <<
737                                           I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
738                                           I40E_TXD_FLTR_QW0_FLEXOFF_MASK);
739
740         fdirdp->qindex_flex_ptype_vsi |=
741                         rte_cpu_to_le_32((pctype <<
742                                           I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
743                                           I40E_TXD_FLTR_QW0_PCTYPE_MASK);
744
745         /* Use LAN VSI Id by default */
746         fdirdp->qindex_flex_ptype_vsi |=
747                 rte_cpu_to_le_32((pf->main_vsi->vsi_id <<
748                                   I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
749                                   I40E_TXD_FLTR_QW0_DEST_VSI_MASK);
750
751         fdirdp->dtype_cmd_cntindex =
752                         rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG);
753
754         if (add)
755                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
756                                 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
757                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
758         else
759                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
760                                 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
761                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
762
763         if (fdir_action->behavior == RTE_ETH_FDIR_REJECT)
764                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
765         else
766                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX;
767         fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest <<
768                                 I40E_TXD_FLTR_QW1_DEST_SHIFT) &
769                                 I40E_TXD_FLTR_QW1_DEST_MASK);
770
771         fdirdp->dtype_cmd_cntindex |=
772                 rte_cpu_to_le_32((fdir_action->report_status<<
773                                 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
774                                 I40E_TXD_FLTR_QW1_FD_STATUS_MASK);
775
776         fdirdp->dtype_cmd_cntindex |=
777                         rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK);
778         fdirdp->dtype_cmd_cntindex |=
779                         rte_cpu_to_le_32((pf->fdir.match_counter_index <<
780                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
781                         I40E_TXD_FLTR_QW1_CNTINDEX_MASK);
782
783         fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id);
784
785         PMD_DRV_LOG(INFO, "filling transmit descriptor.");
786         txdp = &(txq->tx_ring[txq->tx_tail + 1]);
787         txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
788         td_cmd = I40E_TX_DESC_CMD_EOP |
789                  I40E_TX_DESC_CMD_RS  |
790                  I40E_TX_DESC_CMD_DUMMY;
791
792         txdp->cmd_type_offset_bsz =
793                 i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0);
794
795         txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */
796         if (txq->tx_tail >= txq->nb_tx_desc)
797                 txq->tx_tail = 0;
798         /* Update the tx tail register */
799         rte_wmb();
800         I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
801
802         for (i = 0; i < I40E_FDIR_WAIT_COUNT; i++) {
803                 rte_delay_us(I40E_FDIR_WAIT_INTERVAL_US);
804                 if (txdp->cmd_type_offset_bsz &
805                                 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
806                         break;
807         }
808         if (i >= I40E_FDIR_WAIT_COUNT) {
809                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter:"
810                             " time out to get DD on tx queue.");
811                 return -ETIMEDOUT;
812         }
813         /* totally delay 10 ms to check programming status*/
814         rte_delay_us((I40E_FDIR_WAIT_COUNT - i) * I40E_FDIR_WAIT_INTERVAL_US);
815         if (i40e_check_fdir_programming_status(rxq) < 0) {
816                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter:"
817                             " programming status reported.");
818                 return -ENOSYS;
819         }
820
821         return 0;
822 }
823
824 /*
825  * i40e_fdir_flush - clear all filters of Flow Director table
826  * @pf: board private structure
827  */
828 static int
829 i40e_fdir_flush(struct rte_eth_dev *dev)
830 {
831         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
832         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
833         uint32_t reg;
834         uint16_t guarant_cnt, best_cnt;
835         uint16_t i;
836
837         I40E_WRITE_REG(hw, I40E_PFQF_CTL_1, I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
838         I40E_WRITE_FLUSH(hw);
839
840         for (i = 0; i < I40E_FDIR_FLUSH_RETRY; i++) {
841                 rte_delay_ms(I40E_FDIR_FLUSH_INTERVAL_MS);
842                 reg = I40E_READ_REG(hw, I40E_PFQF_CTL_1);
843                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
844                         break;
845         }
846         if (i >= I40E_FDIR_FLUSH_RETRY) {
847                 PMD_DRV_LOG(ERR, "FD table did not flush, may need more time.");
848                 return -ETIMEDOUT;
849         }
850         guarant_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
851                                 I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
852                                 I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
853         best_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
854                                 I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
855                                 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
856         if (guarant_cnt != 0 || best_cnt != 0) {
857                 PMD_DRV_LOG(ERR, "Failed to flush FD table.");
858                 return -ENOSYS;
859         } else
860                 PMD_DRV_LOG(INFO, "FD table Flush success.");
861         return 0;
862 }
863
864 /*
865  * i40e_fdir_ctrl_func - deal with all operations on flow director.
866  * @pf: board private structure
867  * @filter_op:operation will be taken.
868  * @arg: a pointer to specific structure corresponding to the filter_op
869  */
870 int
871 i40e_fdir_ctrl_func(struct rte_eth_dev *dev,
872                        enum rte_filter_op filter_op,
873                        void *arg)
874 {
875         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
876         int ret = 0;
877
878         if (filter_op == RTE_ETH_FILTER_NOP) {
879                 if (!(pf->flags & I40E_FLAG_FDIR))
880                         ret = -ENOTSUP;
881                 return ret;
882         }
883
884         if (arg == NULL && filter_op != RTE_ETH_FILTER_FLUSH)
885                 return -EINVAL;
886
887         switch (filter_op) {
888         case RTE_ETH_FILTER_ADD:
889                 ret = i40e_add_del_fdir_filter(dev,
890                         (struct rte_eth_fdir_filter *)arg,
891                         TRUE);
892                 break;
893         case RTE_ETH_FILTER_DELETE:
894                 ret = i40e_add_del_fdir_filter(dev,
895                         (struct rte_eth_fdir_filter *)arg,
896                         FALSE);
897                 break;
898         case RTE_ETH_FILTER_FLUSH:
899                 ret = i40e_fdir_flush(dev);
900                 break;
901         default:
902                 PMD_DRV_LOG(ERR, "unknown operation %u.", filter_op);
903                 ret = -EINVAL;
904                 break;
905         }
906         return ret;
907 }