drivers/net/i40e/i40e_fdir.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_ether.h>
  14 #include <rte_ethdev_driver.h>
  15 #include <rte_log.h>
  16 #include <rte_memzone.h>
  17 #include <rte_malloc.h>
  18 #include <rte_arp.h>
  19 #include <rte_ip.h>
  20 #include <rte_udp.h>
  21 #include <rte_tcp.h>
  22 #include <rte_sctp.h>
  23 #include <rte_hash_crc.h>
  24 #include <rte_bitmap.h>
  25
  26 #include "i40e_logs.h"
  27 #include "base/i40e_type.h"
  28 #include "base/i40e_prototype.h"
  29 #include "i40e_ethdev.h"
  30 #include "i40e_rxtx.h"
  31
  32 #define I40E_FDIR_MZ_NAME          "FDIR_MEMZONE"
  33 #ifndef IPV6_ADDR_LEN
  34 #define IPV6_ADDR_LEN              16
  35 #endif
  36
  37 #ifndef IPPROTO_L2TP
  38 #define IPPROTO_L2TP              115
  39 #endif
  40
  41 #define I40E_FDIR_PKT_LEN                   512
  42 #define I40E_FDIR_IP_DEFAULT_LEN            420
  43 #define I40E_FDIR_IP_DEFAULT_TTL            0x40
  44 #define I40E_FDIR_IP_DEFAULT_VERSION_IHL    0x45
  45 #define I40E_FDIR_TCP_DEFAULT_DATAOFF       0x50
  46 #define I40E_FDIR_IPv6_DEFAULT_VTC_FLOW     0x60000000
  47
  48 #define I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS   0xFF
  49 #define I40E_FDIR_IPv6_PAYLOAD_LEN          380
  50 #define I40E_FDIR_UDP_DEFAULT_LEN           400
  51 #define I40E_FDIR_GTP_DEFAULT_LEN           384
  52 #define I40E_FDIR_INNER_IP_DEFAULT_LEN      384
  53 #define I40E_FDIR_INNER_IPV6_DEFAULT_LEN    344
  54
  55 #define I40E_FDIR_GTPC_DST_PORT             2123
  56 #define I40E_FDIR_GTPU_DST_PORT             2152
  57 #define I40E_FDIR_GTP_VER_FLAG_0X30         0x30
  58 #define I40E_FDIR_GTP_VER_FLAG_0X32         0x32
  59 #define I40E_FDIR_GTP_MSG_TYPE_0X01         0x01
  60 #define I40E_FDIR_GTP_MSG_TYPE_0XFF         0xFF
  61
  62 #define I40E_FDIR_ESP_DST_PORT              4500
  63
  64 /* Wait time for fdir filter programming */
  65 #define I40E_FDIR_MAX_WAIT_US 10000
  66
  67 /* Wait count and interval for fdir filter flush */
  68 #define I40E_FDIR_FLUSH_RETRY       50
  69 #define I40E_FDIR_FLUSH_INTERVAL_MS 5
  70
  71 #define I40E_COUNTER_PF           2
  72 /* Statistic counter index for one pf */
  73 #define I40E_COUNTER_INDEX_FDIR(pf_id)   (0 + (pf_id) * I40E_COUNTER_PF)
  74
  75 #define I40E_FDIR_FLOWS ( \
  76         (1ULL << RTE_ETH_FLOW_FRAG_IPV4) | \
  77         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \
  78         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \
  79         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \
  80         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \
  81         (1ULL << RTE_ETH_FLOW_FRAG_IPV6) | \
  82         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \
  83         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \
  84         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \
  85         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \
  86         (1ULL << RTE_ETH_FLOW_L2_PAYLOAD))
  87
  88 static int i40e_fdir_filter_programming(struct i40e_pf *pf,
  89                         enum i40e_filter_pctype pctype,
  90                         const struct rte_eth_fdir_filter *filter,
  91                         bool add);
  92 static int i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input,
  93                          struct i40e_fdir_filter *filter);
  94 static struct i40e_fdir_filter *
  95 i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info,
  96                         const struct i40e_fdir_input *input);
  97 static int i40e_sw_fdir_filter_insert(struct i40e_pf *pf,
  98                                    struct i40e_fdir_filter *filter);
  99 static int
 100 i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
 101                                   enum i40e_filter_pctype pctype,
 102                                   const struct i40e_fdir_filter_conf *filter,
 103                                   bool add);
 104
 105 static int
 106 i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq)
 107 {
 108         struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
 109         struct i40e_hmc_obj_rxq rx_ctx;
 110         int err = I40E_SUCCESS;
 111
 112         memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq));
 113         /* Init the RX queue in hardware */
 114         rx_ctx.dbuff = I40E_RXBUF_SZ_1024 >> I40E_RXQ_CTX_DBUFF_SHIFT;
 115         rx_ctx.hbuff = 0;
 116         rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
 117         rx_ctx.qlen = rxq->nb_rx_desc;
 118 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
 119         rx_ctx.dsize = 1;
 120 #endif
 121         rx_ctx.dtype = i40e_header_split_none;
 122         rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE;
 123         rx_ctx.rxmax = RTE_ETHER_MAX_LEN;
 124         rx_ctx.tphrdesc_ena = 1;
 125         rx_ctx.tphwdesc_ena = 1;
 126         rx_ctx.tphdata_ena = 1;
 127         rx_ctx.tphhead_ena = 1;
 128         rx_ctx.lrxqthresh = 2;
 129         rx_ctx.crcstrip = 0;
 130         rx_ctx.l2tsel = 1;
 131         rx_ctx.showiv = 0;
 132         rx_ctx.prefena = 1;
 133
 134         err = i40e_clear_lan_rx_queue_context(hw, rxq->reg_idx);
 135         if (err != I40E_SUCCESS) {
 136                 PMD_DRV_LOG(ERR, "Failed to clear FDIR RX queue context.");
 137                 return err;
 138         }
 139         err = i40e_set_lan_rx_queue_context(hw, rxq->reg_idx, &rx_ctx);
 140         if (err != I40E_SUCCESS) {
 141                 PMD_DRV_LOG(ERR, "Failed to set FDIR RX queue context.");
 142                 return err;
 143         }
 144         rxq->qrx_tail = hw->hw_addr +
 145                 I40E_QRX_TAIL(rxq->vsi->base_queue);
 146
 147         rte_wmb();
 148         /* Init the RX tail regieter. */
 149         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 150
 151         return err;
 152 }
 153
 154 /*
 155  * i40e_fdir_setup - reserve and initialize the Flow Director resources
 156  * @pf: board private structure
 157  */
 158 int
 159 i40e_fdir_setup(struct i40e_pf *pf)
 160 {
 161         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 162         struct i40e_vsi *vsi;
 163         int err = I40E_SUCCESS;
 164         char z_name[RTE_MEMZONE_NAMESIZE];
 165         const struct rte_memzone *mz = NULL;
 166         struct rte_eth_dev *eth_dev = pf->adapter->eth_dev;
 167
 168         if ((pf->flags & I40E_FLAG_FDIR) == 0) {
 169                 PMD_INIT_LOG(ERR, "HW doesn't support FDIR");
 170                 return I40E_NOT_SUPPORTED;
 171         }
 172
 173         PMD_DRV_LOG(INFO, "FDIR HW Capabilities: num_filters_guaranteed = %u,"
 174                         " num_filters_best_effort = %u.",
 175                         hw->func_caps.fd_filters_guaranteed,
 176                         hw->func_caps.fd_filters_best_effort);
 177
 178         vsi = pf->fdir.fdir_vsi;
 179         if (vsi) {
 180                 PMD_DRV_LOG(INFO, "FDIR initialization has been done.");
 181                 return I40E_SUCCESS;
 182         }
 183
 184         /* make new FDIR VSI */
 185         vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->main_vsi, 0);
 186         if (!vsi) {
 187                 PMD_DRV_LOG(ERR, "Couldn't create FDIR VSI.");
 188                 return I40E_ERR_NO_AVAILABLE_VSI;
 189         }
 190         pf->fdir.fdir_vsi = vsi;
 191
 192         /*Fdir tx queue setup*/
 193         err = i40e_fdir_setup_tx_resources(pf);
 194         if (err) {
 195                 PMD_DRV_LOG(ERR, "Failed to setup FDIR TX resources.");
 196                 goto fail_setup_tx;
 197         }
 198
 199         /*Fdir rx queue setup*/
 200         err = i40e_fdir_setup_rx_resources(pf);
 201         if (err) {
 202                 PMD_DRV_LOG(ERR, "Failed to setup FDIR RX resources.");
 203                 goto fail_setup_rx;
 204         }
 205
 206         err = i40e_tx_queue_init(pf->fdir.txq);
 207         if (err) {
 208                 PMD_DRV_LOG(ERR, "Failed to do FDIR TX initialization.");
 209                 goto fail_mem;
 210         }
 211
 212         /* need switch on before dev start*/
 213         err = i40e_switch_tx_queue(hw, vsi->base_queue, TRUE);
 214         if (err) {
 215                 PMD_DRV_LOG(ERR, "Failed to do fdir TX switch on.");
 216                 goto fail_mem;
 217         }
 218
 219         /* Init the rx queue in hardware */
 220         err = i40e_fdir_rx_queue_init(pf->fdir.rxq);
 221         if (err) {
 222                 PMD_DRV_LOG(ERR, "Failed to do FDIR RX initialization.");
 223                 goto fail_mem;
 224         }
 225
 226         /* switch on rx queue */
 227         err = i40e_switch_rx_queue(hw, vsi->base_queue, TRUE);
 228         if (err) {
 229                 PMD_DRV_LOG(ERR, "Failed to do FDIR RX switch on.");
 230                 goto fail_mem;
 231         }
 232
 233         /* reserve memory for the fdir programming packet */
 234         snprintf(z_name, sizeof(z_name), "%s_%s_%d",
 235                         eth_dev->device->driver->name,
 236                         I40E_FDIR_MZ_NAME,
 237                         eth_dev->data->port_id);
 238         mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN, SOCKET_ID_ANY);
 239         if (!mz) {
 240                 PMD_DRV_LOG(ERR, "Cannot init memzone for "
 241                                  "flow director program packet.");
 242                 err = I40E_ERR_NO_MEMORY;
 243                 goto fail_mem;
 244         }
 245         pf->fdir.prg_pkt = mz->addr;
 246         pf->fdir.dma_addr = mz->iova;
 247
 248         pf->fdir.match_counter_index = I40E_COUNTER_INDEX_FDIR(hw->pf_id);
 249         pf->fdir.fdir_actual_cnt = 0;
 250         pf->fdir.fdir_guarantee_free_space =
 251                 pf->fdir.fdir_guarantee_total_space;
 252
 253         PMD_DRV_LOG(INFO, "FDIR setup successfully, with programming queue %u.",
 254                     vsi->base_queue);
 255         return I40E_SUCCESS;
 256
 257 fail_mem:
 258         i40e_dev_rx_queue_release(pf->fdir.rxq);
 259         pf->fdir.rxq = NULL;
 260 fail_setup_rx:
 261         i40e_dev_tx_queue_release(pf->fdir.txq);
 262         pf->fdir.txq = NULL;
 263 fail_setup_tx:
 264         i40e_vsi_release(vsi);
 265         pf->fdir.fdir_vsi = NULL;
 266         return err;
 267 }
 268
 269 /*
 270  * i40e_fdir_teardown - release the Flow Director resources
 271  * @pf: board private structure
 272  */
 273 void
 274 i40e_fdir_teardown(struct i40e_pf *pf)
 275 {
 276         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 277         struct i40e_vsi *vsi;
 278         struct rte_eth_dev *dev = pf->adapter->eth_dev;
 279
 280         vsi = pf->fdir.fdir_vsi;
 281         if (!vsi)
 282                 return;
 283         int err = i40e_switch_tx_queue(hw, vsi->base_queue, FALSE);
 284         if (err)
 285                 PMD_DRV_LOG(DEBUG, "Failed to do FDIR TX switch off");
 286         err = i40e_switch_rx_queue(hw, vsi->base_queue, FALSE);
 287         if (err)
 288                 PMD_DRV_LOG(DEBUG, "Failed to do FDIR RX switch off");
 289         i40e_dev_rx_queue_release(pf->fdir.rxq);
 290         rte_eth_dma_zone_free(dev, "fdir_rx_ring", pf->fdir.rxq->queue_id);
 291         pf->fdir.rxq = NULL;
 292         i40e_dev_tx_queue_release(pf->fdir.txq);
 293         rte_eth_dma_zone_free(dev, "fdir_tx_ring", pf->fdir.txq->queue_id);
 294         pf->fdir.txq = NULL;
 295         i40e_vsi_release(vsi);
 296         pf->fdir.fdir_vsi = NULL;
 297 }
 298
 299 /* check whether the flow director table in empty */
 300 static inline int
 301 i40e_fdir_empty(struct i40e_hw *hw)
 302 {
 303         uint32_t guarant_cnt, best_cnt;
 304
 305         guarant_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
 306                                  I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
 307                                  I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
 308         best_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
 309                               I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
 310                               I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
 311         if (best_cnt + guarant_cnt > 0)
 312                 return -1;
 313
 314         return 0;
 315 }
 316
 317 /*
 318  * Initialize the configuration about bytes stream extracted as flexible payload
 319  * and mask setting
 320  */
 321 static inline void
 322 i40e_init_flx_pld(struct i40e_pf *pf)
 323 {
 324         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 325         uint8_t pctype;
 326         int i, index;
 327         uint16_t flow_type;
 328
 329         /*
 330          * Define the bytes stream extracted as flexible payload in
 331          * field vector. By default, select 8 words from the beginning
 332          * of payload as flexible payload.
 333          */
 334         for (i = I40E_FLXPLD_L2_IDX; i < I40E_MAX_FLXPLD_LAYER; i++) {
 335                 index = i * I40E_MAX_FLXPLD_FIED;
 336                 pf->fdir.flex_set[index].src_offset = 0;
 337                 pf->fdir.flex_set[index].size = I40E_FDIR_MAX_FLEXWORD_NUM;
 338                 pf->fdir.flex_set[index].dst_offset = 0;
 339                 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(index), 0x0000C900);
 340                 I40E_WRITE_REG(hw,
 341                         I40E_PRTQF_FLX_PIT(index + 1), 0x0000FC29);/*non-used*/
 342                 I40E_WRITE_REG(hw,
 343                         I40E_PRTQF_FLX_PIT(index + 2), 0x0000FC2A);/*non-used*/
 344         }
 345
 346         /* initialize the masks */
 347         for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
 348              pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) {
 349                 flow_type = i40e_pctype_to_flowtype(pf->adapter, pctype);
 350
 351                 if (flow_type == RTE_ETH_FLOW_UNKNOWN)
 352                         continue;
 353                 pf->fdir.flex_mask[pctype].word_mask = 0;
 354                 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), 0);
 355                 for (i = 0; i < I40E_FDIR_BITMASK_NUM_WORD; i++) {
 356                         pf->fdir.flex_mask[pctype].bitmask[i].offset = 0;
 357                         pf->fdir.flex_mask[pctype].bitmask[i].mask = 0;
 358                         i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), 0);
 359                 }
 360         }
 361 }
 362
 363 #define I40E_VALIDATE_FLEX_PIT(flex_pit1, flex_pit2) do { \
 364         if ((flex_pit2).src_offset < \
 365                 (flex_pit1).src_offset + (flex_pit1).size) { \
 366                 PMD_DRV_LOG(ERR, "src_offset should be not" \
 367                         " less than than previous offset" \
 368                         " + previous FSIZE."); \
 369                 return -EINVAL; \
 370         } \
 371 } while (0)
 372
 373 /*
 374  * i40e_srcoff_to_flx_pit - transform the src_offset into flex_pit structure,
 375  * and the flex_pit will be sorted by it's src_offset value
 376  */
 377 static inline uint16_t
 378 i40e_srcoff_to_flx_pit(const uint16_t *src_offset,
 379                         struct i40e_fdir_flex_pit *flex_pit)
 380 {
 381         uint16_t src_tmp, size, num = 0;
 382         uint16_t i, k, j = 0;
 383
 384         while (j < I40E_FDIR_MAX_FLEX_LEN) {
 385                 size = 1;
 386                 for (; j < I40E_FDIR_MAX_FLEX_LEN - 1; j++) {
 387                         if (src_offset[j + 1] == src_offset[j] + 1)
 388                                 size++;
 389                         else
 390                                 break;
 391                 }
 392                 src_tmp = src_offset[j] + 1 - size;
 393                 /* the flex_pit need to be sort by src_offset */
 394                 for (i = 0; i < num; i++) {
 395                         if (src_tmp < flex_pit[i].src_offset)
 396                                 break;
 397                 }
 398                 /* if insert required, move backward */
 399                 for (k = num; k > i; k--)
 400                         flex_pit[k] = flex_pit[k - 1];
 401                 /* insert */
 402                 flex_pit[i].dst_offset = j + 1 - size;
 403                 flex_pit[i].src_offset = src_tmp;
 404                 flex_pit[i].size = size;
 405                 j++;
 406                 num++;
 407         }
 408         return num;
 409 }
 410
 411 /* i40e_check_fdir_flex_payload -check flex payload configuration arguments */
 412 static inline int
 413 i40e_check_fdir_flex_payload(const struct rte_eth_flex_payload_cfg *flex_cfg)
 414 {
 415         struct i40e_fdir_flex_pit flex_pit[I40E_FDIR_MAX_FLEX_LEN];
 416         uint16_t num, i;
 417
 418         for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i++) {
 419                 if (flex_cfg->src_offset[i] >= I40E_MAX_FLX_SOURCE_OFF) {
 420                         PMD_DRV_LOG(ERR, "exceeds maxmial payload limit.");
 421                         return -EINVAL;
 422                 }
 423         }
 424
 425         memset(flex_pit, 0, sizeof(flex_pit));
 426         num = i40e_srcoff_to_flx_pit(flex_cfg->src_offset, flex_pit);
 427         if (num > I40E_MAX_FLXPLD_FIED) {
 428                 PMD_DRV_LOG(ERR, "exceeds maxmial number of flex fields.");
 429                 return -EINVAL;
 430         }
 431         for (i = 0; i < num; i++) {
 432                 if (flex_pit[i].size & 0x01 || flex_pit[i].dst_offset & 0x01 ||
 433                         flex_pit[i].src_offset & 0x01) {
 434                         PMD_DRV_LOG(ERR, "flexpayload should be measured"
 435                                 " in word");
 436                         return -EINVAL;
 437                 }
 438                 if (i != num - 1)
 439                         I40E_VALIDATE_FLEX_PIT(flex_pit[i], flex_pit[i + 1]);
 440         }
 441         return 0;
 442 }
 443
 444 /*
 445  * i40e_check_fdir_flex_conf -check if the flex payload and mask configuration
 446  * arguments are valid
 447  */
 448 static int
 449 i40e_check_fdir_flex_conf(const struct i40e_adapter *adapter,
 450                           const struct rte_eth_fdir_flex_conf *conf)
 451 {
 452         const struct rte_eth_flex_payload_cfg *flex_cfg;
 453         const struct rte_eth_fdir_flex_mask *flex_mask;
 454         uint16_t mask_tmp;
 455         uint8_t nb_bitmask;
 456         uint16_t i, j;
 457         int ret = 0;
 458         enum i40e_filter_pctype pctype;
 459
 460         if (conf == NULL) {
 461                 PMD_DRV_LOG(INFO, "NULL pointer.");
 462                 return -EINVAL;
 463         }
 464         /* check flexible payload setting configuration */
 465         if (conf->nb_payloads > RTE_ETH_L4_PAYLOAD) {
 466                 PMD_DRV_LOG(ERR, "invalid number of payload setting.");
 467                 return -EINVAL;
 468         }
 469         for (i = 0; i < conf->nb_payloads; i++) {
 470                 flex_cfg = &conf->flex_set[i];
 471                 if (flex_cfg->type > RTE_ETH_L4_PAYLOAD) {
 472                         PMD_DRV_LOG(ERR, "invalid payload type.");
 473                         return -EINVAL;
 474                 }
 475                 ret = i40e_check_fdir_flex_payload(flex_cfg);
 476                 if (ret < 0) {
 477                         PMD_DRV_LOG(ERR, "invalid flex payload arguments.");
 478                         return -EINVAL;
 479                 }
 480         }
 481
 482         /* check flex mask setting configuration */
 483         if (conf->nb_flexmasks >= RTE_ETH_FLOW_MAX) {
 484                 PMD_DRV_LOG(ERR, "invalid number of flex masks.");
 485                 return -EINVAL;
 486         }
 487         for (i = 0; i < conf->nb_flexmasks; i++) {
 488                 flex_mask = &conf->flex_mask[i];
 489                 pctype = i40e_flowtype_to_pctype(adapter, flex_mask->flow_type);
 490                 if (pctype == I40E_FILTER_PCTYPE_INVALID) {
 491                         PMD_DRV_LOG(WARNING, "invalid flow type.");
 492                         return -EINVAL;
 493                 }
 494                 nb_bitmask = 0;
 495                 for (j = 0; j < I40E_FDIR_MAX_FLEX_LEN; j += sizeof(uint16_t)) {
 496                         mask_tmp = I40E_WORD(flex_mask->mask[j],
 497                                              flex_mask->mask[j + 1]);
 498                         if (mask_tmp != 0x0 && mask_tmp != UINT16_MAX) {
 499                                 nb_bitmask++;
 500                                 if (nb_bitmask > I40E_FDIR_BITMASK_NUM_WORD) {
 501                                         PMD_DRV_LOG(ERR, " exceed maximal"
 502                                                 " number of bitmasks.");
 503                                         return -EINVAL;
 504                                 }
 505                         }
 506                 }
 507         }
 508         return 0;
 509 }
 510
 511 /*
 512  * i40e_set_flx_pld_cfg -configure the rule how bytes stream is extracted as flexible payload
 513  * @pf: board private structure
 514  * @cfg: the rule how bytes stream is extracted as flexible payload
 515  */
 516 static void
 517 i40e_set_flx_pld_cfg(struct i40e_pf *pf,
 518                          const struct rte_eth_flex_payload_cfg *cfg)
 519 {
 520         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 521         struct i40e_fdir_flex_pit flex_pit[I40E_MAX_FLXPLD_FIED];
 522         uint32_t flx_pit, flx_ort;
 523         uint16_t num, min_next_off;  /* in words */
 524         uint8_t field_idx = 0;
 525         uint8_t layer_idx = 0;
 526         uint16_t i;
 527
 528         if (cfg->type == RTE_ETH_L2_PAYLOAD)
 529                 layer_idx = I40E_FLXPLD_L2_IDX;
 530         else if (cfg->type == RTE_ETH_L3_PAYLOAD)
 531                 layer_idx = I40E_FLXPLD_L3_IDX;
 532         else if (cfg->type == RTE_ETH_L4_PAYLOAD)
 533                 layer_idx = I40E_FLXPLD_L4_IDX;
 534
 535         memset(flex_pit, 0, sizeof(flex_pit));
 536         num = RTE_MIN(i40e_srcoff_to_flx_pit(cfg->src_offset, flex_pit),
 537                       RTE_DIM(flex_pit));
 538
 539         if (num) {
 540                 flx_ort = (1 << I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT) |
 541                           (num << I40E_GLQF_ORT_FIELD_CNT_SHIFT) |
 542                           (layer_idx * I40E_MAX_FLXPLD_FIED);
 543                 I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33 + layer_idx), flx_ort);
 544         }
 545
 546         for (i = 0; i < num; i++) {
 547                 field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i;
 548                 /* record the info in fdir structure */
 549                 pf->fdir.flex_set[field_idx].src_offset =
 550                         flex_pit[i].src_offset / sizeof(uint16_t);
 551                 pf->fdir.flex_set[field_idx].size =
 552                         flex_pit[i].size / sizeof(uint16_t);
 553                 pf->fdir.flex_set[field_idx].dst_offset =
 554                         flex_pit[i].dst_offset / sizeof(uint16_t);
 555                 flx_pit = MK_FLX_PIT(pf->fdir.flex_set[field_idx].src_offset,
 556                                 pf->fdir.flex_set[field_idx].size,
 557                                 pf->fdir.flex_set[field_idx].dst_offset);
 558
 559                 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit);
 560         }
 561         min_next_off = pf->fdir.flex_set[field_idx].src_offset +
 562                                 pf->fdir.flex_set[field_idx].size;
 563
 564         for (; i < I40E_MAX_FLXPLD_FIED; i++) {
 565                 /* set the non-used register obeying register's constrain */
 566                 flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE,
 567                            NONUSE_FLX_PIT_DEST_OFF);
 568                 I40E_WRITE_REG(hw,
 569                         I40E_PRTQF_FLX_PIT(layer_idx * I40E_MAX_FLXPLD_FIED + i),
 570                         flx_pit);
 571                 min_next_off++;
 572         }
 573 }
 574
 575 /*
 576  * i40e_set_flex_mask_on_pctype - configure the mask on flexible payload
 577  * @pf: board private structure
 578  * @pctype: packet classify type
 579  * @flex_masks: mask for flexible payload
 580  */
 581 static void
 582 i40e_set_flex_mask_on_pctype(struct i40e_pf *pf,
 583                 enum i40e_filter_pctype pctype,
 584                 const struct rte_eth_fdir_flex_mask *mask_cfg)
 585 {
 586         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 587         struct i40e_fdir_flex_mask *flex_mask;
 588         uint32_t flxinset, fd_mask;
 589         uint16_t mask_tmp;
 590         uint8_t i, nb_bitmask = 0;
 591
 592         flex_mask = &pf->fdir.flex_mask[pctype];
 593         memset(flex_mask, 0, sizeof(struct i40e_fdir_flex_mask));
 594         for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i += sizeof(uint16_t)) {
 595                 mask_tmp = I40E_WORD(mask_cfg->mask[i], mask_cfg->mask[i + 1]);
 596                 if (mask_tmp != 0x0) {
 597                         flex_mask->word_mask |=
 598                                 I40E_FLEX_WORD_MASK(i / sizeof(uint16_t));
 599                         if (mask_tmp != UINT16_MAX) {
 600                                 /* set bit mask */
 601                                 flex_mask->bitmask[nb_bitmask].mask = ~mask_tmp;
 602                                 flex_mask->bitmask[nb_bitmask].offset =
 603                                         i / sizeof(uint16_t);
 604                                 nb_bitmask++;
 605                         }
 606                 }
 607         }
 608         /* write mask to hw */
 609         flxinset = (flex_mask->word_mask <<
 610                 I40E_PRTQF_FD_FLXINSET_INSET_SHIFT) &
 611                 I40E_PRTQF_FD_FLXINSET_INSET_MASK;
 612         i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), flxinset);
 613
 614         for (i = 0; i < nb_bitmask; i++) {
 615                 fd_mask = (flex_mask->bitmask[i].mask <<
 616                         I40E_PRTQF_FD_MSK_MASK_SHIFT) &
 617                         I40E_PRTQF_FD_MSK_MASK_MASK;
 618                 fd_mask |= ((flex_mask->bitmask[i].offset +
 619                         I40E_FLX_OFFSET_IN_FIELD_VECTOR) <<
 620                         I40E_PRTQF_FD_MSK_OFFSET_SHIFT) &
 621                         I40E_PRTQF_FD_MSK_OFFSET_MASK;
 622                 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), fd_mask);
 623         }
 624 }
 625
 626 /*
 627  * Enable/disable flow director RX processing in vector routines.
 628  */
 629 void
 630 i40e_fdir_rx_proc_enable(struct rte_eth_dev *dev, bool on)
 631 {
 632         int32_t i;
 633
 634         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 635                 struct i40e_rx_queue *rxq = dev->data->rx_queues[i];
 636                 if (!rxq)
 637                         continue;
 638                 rxq->fdir_enabled = on;
 639         }
 640         PMD_DRV_LOG(DEBUG, "Flow Director processing on RX set to %d", on);
 641 }
 642
 643 /*
 644  * Configure flow director related setting
 645  */
 646 int
 647 i40e_fdir_configure(struct rte_eth_dev *dev)
 648 {
 649         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 650         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 651         struct rte_eth_fdir_flex_conf *conf;
 652         enum i40e_filter_pctype pctype;
 653         uint32_t val;
 654         uint8_t i;
 655         int ret = 0;
 656
 657         /*
 658         * configuration need to be done before
 659         * flow director filters are added
 660         * If filters exist, flush them.
 661         */
 662         if (i40e_fdir_empty(hw) < 0) {
 663                 ret = i40e_fdir_flush(dev);
 664                 if (ret) {
 665                         PMD_DRV_LOG(ERR, "failed to flush fdir table.");
 666                         return ret;
 667                 }
 668         }
 669
 670         /* enable FDIR filter */
 671         val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
 672         val |= I40E_PFQF_CTL_0_FD_ENA_MASK;
 673         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, val);
 674
 675         i40e_init_flx_pld(pf); /* set flex config to default value */
 676
 677         conf = &dev->data->dev_conf.fdir_conf.flex_conf;
 678         ret = i40e_check_fdir_flex_conf(pf->adapter, conf);
 679         if (ret < 0) {
 680                 PMD_DRV_LOG(ERR, " invalid configuration arguments.");
 681                 return -EINVAL;
 682         }
 683
 684         if (!pf->support_multi_driver) {
 685                 /* configure flex payload */
 686                 for (i = 0; i < conf->nb_payloads; i++)
 687                         i40e_set_flx_pld_cfg(pf, &conf->flex_set[i]);
 688                 /* configure flex mask*/
 689                 for (i = 0; i < conf->nb_flexmasks; i++) {
 690                         if (hw->mac.type == I40E_MAC_X722) {
 691                                 /* get pctype value in fd pctype register */
 692                                 pctype = (enum i40e_filter_pctype)
 693                                           i40e_read_rx_ctl(hw,
 694                                                 I40E_GLQF_FD_PCTYPES(
 695                                                 (int)i40e_flowtype_to_pctype(
 696                                                 pf->adapter,
 697                                                 conf->flex_mask[i].flow_type)));
 698                         } else {
 699                                 pctype = i40e_flowtype_to_pctype(pf->adapter,
 700                                                   conf->flex_mask[i].flow_type);
 701                         }
 702
 703                         i40e_set_flex_mask_on_pctype(pf, pctype,
 704                                                      &conf->flex_mask[i]);
 705                 }
 706         } else {
 707                 PMD_DRV_LOG(ERR, "Not support flexible payload.");
 708         }
 709
 710         /* Enable FDIR processing in RX routines */
 711         i40e_fdir_rx_proc_enable(dev, 1);
 712
 713         return ret;
 714 }
 715
 716 static inline int
 717 i40e_fdir_fill_eth_ip_head(const struct rte_eth_fdir_input *fdir_input,
 718                            unsigned char *raw_pkt,
 719                            bool vlan)
 720 {
 721         static uint8_t vlan_frame[] = {0x81, 0, 0, 0};
 722         uint16_t *ether_type;
 723         uint8_t len = 2 * sizeof(struct rte_ether_addr);
 724         struct rte_ipv4_hdr *ip;
 725         struct rte_ipv6_hdr *ip6;
 726         static const uint8_t next_proto[] = {
 727                 [RTE_ETH_FLOW_FRAG_IPV4] = IPPROTO_IP,
 728                 [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = IPPROTO_TCP,
 729                 [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = IPPROTO_UDP,
 730                 [RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] = IPPROTO_SCTP,
 731                 [RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] = IPPROTO_IP,
 732                 [RTE_ETH_FLOW_FRAG_IPV6] = IPPROTO_NONE,
 733                 [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = IPPROTO_TCP,
 734                 [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = IPPROTO_UDP,
 735                 [RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] = IPPROTO_SCTP,
 736                 [RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] = IPPROTO_NONE,
 737         };
 738
 739         raw_pkt += 2 * sizeof(struct rte_ether_addr);
 740         if (vlan && fdir_input->flow_ext.vlan_tci) {
 741                 rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame));
 742                 rte_memcpy(raw_pkt + sizeof(uint16_t),
 743                            &fdir_input->flow_ext.vlan_tci,
 744                            sizeof(uint16_t));
 745                 raw_pkt += sizeof(vlan_frame);
 746                 len += sizeof(vlan_frame);
 747         }
 748         ether_type = (uint16_t *)raw_pkt;
 749         raw_pkt += sizeof(uint16_t);
 750         len += sizeof(uint16_t);
 751
 752         switch (fdir_input->flow_type) {
 753         case RTE_ETH_FLOW_L2_PAYLOAD:
 754                 *ether_type = fdir_input->flow.l2_flow.ether_type;
 755                 break;
 756         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 757         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 758         case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP:
 759         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 760         case RTE_ETH_FLOW_FRAG_IPV4:
 761                 ip = (struct rte_ipv4_hdr *)raw_pkt;
 762
 763                 *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
 764                 ip->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL;
 765                 /* set len to by default */
 766                 ip->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN);
 767                 ip->next_proto_id = fdir_input->flow.ip4_flow.proto ?
 768                                         fdir_input->flow.ip4_flow.proto :
 769                                         next_proto[fdir_input->flow_type];
 770                 ip->time_to_live = fdir_input->flow.ip4_flow.ttl ?
 771                                         fdir_input->flow.ip4_flow.ttl :
 772                                         I40E_FDIR_IP_DEFAULT_TTL;
 773                 ip->type_of_service = fdir_input->flow.ip4_flow.tos;
 774                 /*
 775                  * The source and destination fields in the transmitted packet
 776                  * need to be presented in a reversed order with respect
 777                  * to the expected received packets.
 778                  */
 779                 ip->src_addr = fdir_input->flow.ip4_flow.dst_ip;
 780                 ip->dst_addr = fdir_input->flow.ip4_flow.src_ip;
 781                 len += sizeof(struct rte_ipv4_hdr);
 782                 break;
 783         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 784         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 785         case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP:
 786         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 787         case RTE_ETH_FLOW_FRAG_IPV6:
 788                 ip6 = (struct rte_ipv6_hdr *)raw_pkt;
 789
 790                 *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
 791                 ip6->vtc_flow =
 792                         rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
 793                                          (fdir_input->flow.ipv6_flow.tc <<
 794                                           I40E_FDIR_IPv6_TC_OFFSET));
 795                 ip6->payload_len =
 796                         rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
 797                 ip6->proto = fdir_input->flow.ipv6_flow.proto ?
 798                                         fdir_input->flow.ipv6_flow.proto :
 799                                         next_proto[fdir_input->flow_type];
 800                 ip6->hop_limits = fdir_input->flow.ipv6_flow.hop_limits ?
 801                                         fdir_input->flow.ipv6_flow.hop_limits :
 802                                         I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
 803                 /*
 804                  * The source and destination fields in the transmitted packet
 805                  * need to be presented in a reversed order with respect
 806                  * to the expected received packets.
 807                  */
 808                 rte_memcpy(&(ip6->src_addr),
 809                            &(fdir_input->flow.ipv6_flow.dst_ip),
 810                            IPV6_ADDR_LEN);
 811                 rte_memcpy(&(ip6->dst_addr),
 812                            &(fdir_input->flow.ipv6_flow.src_ip),
 813                            IPV6_ADDR_LEN);
 814                 len += sizeof(struct rte_ipv6_hdr);
 815                 break;
 816         default:
 817                 PMD_DRV_LOG(ERR, "unknown flow type %u.",
 818                             fdir_input->flow_type);
 819                 return -1;
 820         }
 821         return len;
 822 }
 823
 824
 825 /*
 826  * i40e_fdir_construct_pkt - construct packet based on fields in input
 827  * @pf: board private structure
 828  * @fdir_input: input set of the flow director entry
 829  * @raw_pkt: a packet to be constructed
 830  */
 831 static int
 832 i40e_fdir_construct_pkt(struct i40e_pf *pf,
 833                              const struct rte_eth_fdir_input *fdir_input,
 834                              unsigned char *raw_pkt)
 835 {
 836         unsigned char *payload, *ptr;
 837         struct rte_udp_hdr *udp;
 838         struct rte_tcp_hdr *tcp;
 839         struct rte_sctp_hdr *sctp;
 840         uint8_t size, dst = 0;
 841         uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/
 842         int len;
 843
 844         /* fill the ethernet and IP head */
 845         len = i40e_fdir_fill_eth_ip_head(fdir_input, raw_pkt,
 846                                          !!fdir_input->flow_ext.vlan_tci);
 847         if (len < 0)
 848                 return -EINVAL;
 849
 850         /* fill the L4 head */
 851         switch (fdir_input->flow_type) {
 852         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 853                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
 854                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
 855                 /*
 856                  * The source and destination fields in the transmitted packet
 857                  * need to be presented in a reversed order with respect
 858                  * to the expected received packets.
 859                  */
 860                 udp->src_port = fdir_input->flow.udp4_flow.dst_port;
 861                 udp->dst_port = fdir_input->flow.udp4_flow.src_port;
 862                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
 863                 break;
 864
 865         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 866                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
 867                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
 868                 /*
 869                  * The source and destination fields in the transmitted packet
 870                  * need to be presented in a reversed order with respect
 871                  * to the expected received packets.
 872                  */
 873                 tcp->src_port = fdir_input->flow.tcp4_flow.dst_port;
 874                 tcp->dst_port = fdir_input->flow.tcp4_flow.src_port;
 875                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
 876                 break;
 877
 878         case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP:
 879                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
 880                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
 881                 /*
 882                  * The source and destination fields in the transmitted packet
 883                  * need to be presented in a reversed order with respect
 884                  * to the expected received packets.
 885                  */
 886                 sctp->src_port = fdir_input->flow.sctp4_flow.dst_port;
 887                 sctp->dst_port = fdir_input->flow.sctp4_flow.src_port;
 888                 sctp->tag = fdir_input->flow.sctp4_flow.verify_tag;
 889                 break;
 890
 891         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 892         case RTE_ETH_FLOW_FRAG_IPV4:
 893                 payload = raw_pkt + len;
 894                 set_idx = I40E_FLXPLD_L3_IDX;
 895                 break;
 896
 897         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 898                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
 899                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
 900                 /*
 901                  * The source and destination fields in the transmitted packet
 902                  * need to be presented in a reversed order with respect
 903                  * to the expected received packets.
 904                  */
 905                 udp->src_port = fdir_input->flow.udp6_flow.dst_port;
 906                 udp->dst_port = fdir_input->flow.udp6_flow.src_port;
 907                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
 908                 break;
 909
 910         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 911                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
 912                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
 913                 /*
 914                  * The source and destination fields in the transmitted packet
 915                  * need to be presented in a reversed order with respect
 916                  * to the expected received packets.
 917                  */
 918                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
 919                 tcp->src_port = fdir_input->flow.udp6_flow.dst_port;
 920                 tcp->dst_port = fdir_input->flow.udp6_flow.src_port;
 921                 break;
 922
 923         case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP:
 924                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
 925                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
 926                 /*
 927                  * The source and destination fields in the transmitted packet
 928                  * need to be presented in a reversed order with respect
 929                  * to the expected received packets.
 930                  */
 931                 sctp->src_port = fdir_input->flow.sctp6_flow.dst_port;
 932                 sctp->dst_port = fdir_input->flow.sctp6_flow.src_port;
 933                 sctp->tag = fdir_input->flow.sctp6_flow.verify_tag;
 934                 break;
 935
 936         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 937         case RTE_ETH_FLOW_FRAG_IPV6:
 938                 payload = raw_pkt + len;
 939                 set_idx = I40E_FLXPLD_L3_IDX;
 940                 break;
 941         case RTE_ETH_FLOW_L2_PAYLOAD:
 942                 payload = raw_pkt + len;
 943                 /*
 944                  * ARP packet is a special case on which the payload
 945                  * starts after the whole ARP header
 946                  */
 947                 if (fdir_input->flow.l2_flow.ether_type ==
 948                                 rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
 949                         payload += sizeof(struct rte_arp_hdr);
 950                 set_idx = I40E_FLXPLD_L2_IDX;
 951                 break;
 952         default:
 953                 PMD_DRV_LOG(ERR, "unknown flow type %u.", fdir_input->flow_type);
 954                 return -EINVAL;
 955         }
 956
 957         /* fill the flexbytes to payload */
 958         for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
 959                 pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i;
 960                 size = pf->fdir.flex_set[pit_idx].size;
 961                 if (size == 0)
 962                         continue;
 963                 dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t);
 964                 ptr = payload +
 965                         pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t);
 966                 rte_memcpy(ptr,
 967                                  &fdir_input->flow_ext.flexbytes[dst],
 968                                  size * sizeof(uint16_t));
 969         }
 970
 971         return 0;
 972 }
 973
 974 static struct i40e_customized_pctype *
 975 i40e_flow_fdir_find_customized_pctype(struct i40e_pf *pf, uint8_t pctype)
 976 {
 977         struct i40e_customized_pctype *cus_pctype;
 978         enum i40e_new_pctype i = I40E_CUSTOMIZED_GTPC;
 979
 980         for (; i < I40E_CUSTOMIZED_MAX; i++) {
 981                 cus_pctype = &pf->customized_pctype[i];
 982                 if (pctype == cus_pctype->pctype)
 983                         return cus_pctype;
 984         }
 985         return NULL;
 986 }
 987
 988 static inline int
 989 fill_ip6_head(const struct i40e_fdir_input *fdir_input, unsigned char *raw_pkt,
 990                 uint8_t next_proto, uint8_t len, uint16_t *ether_type)
 991 {
 992         struct rte_ipv6_hdr *ip6;
 993
 994         ip6 = (struct rte_ipv6_hdr *)raw_pkt;
 995
 996         *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
 997         ip6->vtc_flow = rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
 998                 (fdir_input->flow.ipv6_flow.tc << I40E_FDIR_IPv6_TC_OFFSET));
 999         ip6->payload_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
1000         ip6->proto = fdir_input->flow.ipv6_flow.proto ?
1001                 fdir_input->flow.ipv6_flow.proto : next_proto;
1002         ip6->hop_limits = fdir_input->flow.ipv6_flow.hop_limits ?
1003                 fdir_input->flow.ipv6_flow.hop_limits :
1004                 I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
1005         /**
1006          * The source and destination fields in the transmitted packet
1007          * need to be presented in a reversed order with respect
1008          * to the expected received packets.
1009          */
1010         rte_memcpy(&ip6->src_addr, &fdir_input->flow.ipv6_flow.dst_ip,
1011                 IPV6_ADDR_LEN);
1012         rte_memcpy(&ip6->dst_addr, &fdir_input->flow.ipv6_flow.src_ip,
1013                 IPV6_ADDR_LEN);
1014         len += sizeof(struct rte_ipv6_hdr);
1015
1016         return len;
1017 }
1018
1019 static inline int
1020 fill_ip4_head(const struct i40e_fdir_input *fdir_input, unsigned char *raw_pkt,
1021                 uint8_t next_proto, uint8_t len, uint16_t *ether_type)
1022 {
1023         struct rte_ipv4_hdr *ip4;
1024
1025         ip4 = (struct rte_ipv4_hdr *)raw_pkt;
1026
1027         *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1028         ip4->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL;
1029         /* set len to by default */
1030         ip4->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN);
1031         ip4->time_to_live = fdir_input->flow.ip4_flow.ttl ?
1032                 fdir_input->flow.ip4_flow.ttl :
1033                 I40E_FDIR_IP_DEFAULT_TTL;
1034         ip4->type_of_service = fdir_input->flow.ip4_flow.tos;
1035         ip4->next_proto_id = fdir_input->flow.ip4_flow.proto ?
1036                 fdir_input->flow.ip4_flow.proto : next_proto;
1037         /**
1038          * The source and destination fields in the transmitted packet
1039          * need to be presented in a reversed order with respect
1040          * to the expected received packets.
1041          */
1042         ip4->src_addr = fdir_input->flow.ip4_flow.dst_ip;
1043         ip4->dst_addr = fdir_input->flow.ip4_flow.src_ip;
1044         len += sizeof(struct rte_ipv4_hdr);
1045
1046         return len;
1047 }
1048
1049 static inline int
1050 i40e_flow_fdir_fill_eth_ip_head(struct i40e_pf *pf,
1051                                 const struct i40e_fdir_input *fdir_input,
1052                                 unsigned char *raw_pkt,
1053                                 bool vlan)
1054 {
1055         struct i40e_customized_pctype *cus_pctype = NULL;
1056         static uint8_t vlan_frame[] = {0x81, 0, 0, 0};
1057         uint16_t *ether_type;
1058         uint8_t len = 2 * sizeof(struct rte_ether_addr);
1059         uint8_t pctype = fdir_input->pctype;
1060         bool is_customized_pctype = fdir_input->flow_ext.customized_pctype;
1061         static const uint8_t next_proto[] = {
1062                 [I40E_FILTER_PCTYPE_FRAG_IPV4] = IPPROTO_IP,
1063                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = IPPROTO_TCP,
1064                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = IPPROTO_UDP,
1065                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = IPPROTO_SCTP,
1066                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = IPPROTO_IP,
1067                 [I40E_FILTER_PCTYPE_FRAG_IPV6] = IPPROTO_NONE,
1068                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = IPPROTO_TCP,
1069                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = IPPROTO_UDP,
1070                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = IPPROTO_SCTP,
1071                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = IPPROTO_NONE,
1072         };
1073
1074         rte_memcpy(raw_pkt, &fdir_input->flow.l2_flow.dst,
1075                 sizeof(struct rte_ether_addr));
1076         rte_memcpy(raw_pkt + sizeof(struct rte_ether_addr),
1077                 &fdir_input->flow.l2_flow.src,
1078                 sizeof(struct rte_ether_addr));
1079         raw_pkt += 2 * sizeof(struct rte_ether_addr);
1080
1081         if (vlan && fdir_input->flow_ext.vlan_tci) {
1082                 rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame));
1083                 rte_memcpy(raw_pkt + sizeof(uint16_t),
1084                            &fdir_input->flow_ext.vlan_tci,
1085                            sizeof(uint16_t));
1086                 raw_pkt += sizeof(vlan_frame);
1087                 len += sizeof(vlan_frame);
1088         }
1089         ether_type = (uint16_t *)raw_pkt;
1090         raw_pkt += sizeof(uint16_t);
1091         len += sizeof(uint16_t);
1092
1093         if (is_customized_pctype) {
1094                 cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype);
1095                 if (!cus_pctype) {
1096                         PMD_DRV_LOG(ERR, "unknown pctype %u.",
1097                                     fdir_input->pctype);
1098                         return -1;
1099                 }
1100         }
1101
1102         if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD)
1103                 *ether_type = fdir_input->flow.l2_flow.ether_type;
1104         else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP ||
1105                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP ||
1106                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP ||
1107                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1108                  pctype == I40E_FILTER_PCTYPE_FRAG_IPV4 ||
1109                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP ||
1110                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP ||
1111                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP ||
1112                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1113                  pctype == I40E_FILTER_PCTYPE_FRAG_IPV6 ||
1114                  is_customized_pctype) {
1115                 if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP ||
1116                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP ||
1117                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP ||
1118                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1119                         pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) {
1120                         len = fill_ip4_head(fdir_input, raw_pkt,
1121                                         next_proto[pctype], len, ether_type);
1122                 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP ||
1123                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP ||
1124                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP ||
1125                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1126                         pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) {
1127                         len = fill_ip6_head(fdir_input, raw_pkt,
1128                                         next_proto[pctype], len,
1129                                         ether_type);
1130                 } else if (cus_pctype->index == I40E_CUSTOMIZED_GTPC ||
1131                          cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 ||
1132                          cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 ||
1133                          cus_pctype->index == I40E_CUSTOMIZED_GTPU) {
1134                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1135                                         len, ether_type);
1136                 } else if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3) {
1137                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_L2TP,
1138                                         len, ether_type);
1139                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4) {
1140                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_ESP,
1141                                         len, ether_type);
1142                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1143                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1144                                         len, ether_type);
1145                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1146                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1147                                         len, ether_type);
1148                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6)
1149                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_ESP,
1150                                         len, ether_type);
1151                 else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6_UDP)
1152                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_UDP,
1153                                         len, ether_type);
1154                 else if (cus_pctype->index == I40E_CUSTOMIZED_IPV6_L2TPV3)
1155                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_L2TP,
1156                                         len, ether_type);
1157         } else {
1158                 PMD_DRV_LOG(ERR, "unknown pctype %u.", fdir_input->pctype);
1159                 return -1;
1160         }
1161
1162         return len;
1163 }
1164
1165 /**
1166  * i40e_flow_fdir_construct_pkt - construct packet based on fields in input
1167  * @pf: board private structure
1168  * @fdir_input: input set of the flow director entry
1169  * @raw_pkt: a packet to be constructed
1170  */
1171 static int
1172 i40e_flow_fdir_construct_pkt(struct i40e_pf *pf,
1173                              const struct i40e_fdir_input *fdir_input,
1174                              unsigned char *raw_pkt)
1175 {
1176         unsigned char *payload = NULL;
1177         unsigned char *ptr;
1178         struct rte_udp_hdr *udp;
1179         struct rte_tcp_hdr *tcp;
1180         struct rte_sctp_hdr *sctp;
1181         struct rte_flow_item_gtp *gtp;
1182         struct rte_ipv4_hdr *gtp_ipv4;
1183         struct rte_ipv6_hdr *gtp_ipv6;
1184         struct rte_flow_item_l2tpv3oip *l2tpv3oip;
1185         struct rte_flow_item_esp *esp;
1186         struct rte_ipv4_hdr *esp_ipv4;
1187         struct rte_ipv6_hdr *esp_ipv6;
1188
1189         uint8_t size, dst = 0;
1190         uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/
1191         int len;
1192         uint8_t pctype = fdir_input->pctype;
1193         struct i40e_customized_pctype *cus_pctype;
1194
1195         /* raw pcket template - just copy contents of the raw packet */
1196         if (fdir_input->flow_ext.pkt_template) {
1197                 memcpy(raw_pkt, fdir_input->flow.raw_flow.packet,
1198                        fdir_input->flow.raw_flow.length);
1199                 return 0;
1200         }
1201
1202         /* fill the ethernet and IP head */
1203         len = i40e_flow_fdir_fill_eth_ip_head(pf, fdir_input, raw_pkt,
1204                                               !!fdir_input->flow_ext.vlan_tci);
1205         if (len < 0)
1206                 return -EINVAL;
1207
1208         /* fill the L4 head */
1209         if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP) {
1210                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
1211                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
1212                 /**
1213                  * The source and destination fields in the transmitted packet
1214                  * need to be presented in a reversed order with respect
1215                  * to the expected received packets.
1216                  */
1217                 udp->src_port = fdir_input->flow.udp4_flow.dst_port;
1218                 udp->dst_port = fdir_input->flow.udp4_flow.src_port;
1219                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
1220         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP) {
1221                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
1222                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
1223                 /**
1224                  * The source and destination fields in the transmitted packet
1225                  * need to be presented in a reversed order with respect
1226                  * to the expected received packets.
1227                  */
1228                 tcp->src_port = fdir_input->flow.tcp4_flow.dst_port;
1229                 tcp->dst_port = fdir_input->flow.tcp4_flow.src_port;
1230                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
1231         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) {
1232                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
1233                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
1234                 /**
1235                  * The source and destination fields in the transmitted packet
1236                  * need to be presented in a reversed order with respect
1237                  * to the expected received packets.
1238                  */
1239                 sctp->src_port = fdir_input->flow.sctp4_flow.dst_port;
1240                 sctp->dst_port = fdir_input->flow.sctp4_flow.src_port;
1241                 sctp->tag = fdir_input->flow.sctp4_flow.verify_tag;
1242         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1243                    pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) {
1244                 payload = raw_pkt + len;
1245                 set_idx = I40E_FLXPLD_L3_IDX;
1246         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP) {
1247                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
1248                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
1249                 /**
1250                  * The source and destination fields in the transmitted packet
1251                  * need to be presented in a reversed order with respect
1252                  * to the expected received packets.
1253                  */
1254                 udp->src_port = fdir_input->flow.udp6_flow.dst_port;
1255                 udp->dst_port = fdir_input->flow.udp6_flow.src_port;
1256                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
1257         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP) {
1258                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
1259                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
1260                 /**
1261                  * The source and destination fields in the transmitted packet
1262                  * need to be presented in a reversed order with respect
1263                  * to the expected received packets.
1264                  */
1265                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
1266                 tcp->src_port = fdir_input->flow.udp6_flow.dst_port;
1267                 tcp->dst_port = fdir_input->flow.udp6_flow.src_port;
1268         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) {
1269                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
1270                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
1271                 /**
1272                  * The source and destination fields in the transmitted packet
1273                  * need to be presented in a reversed order with respect
1274                  * to the expected received packets.
1275                  */
1276                 sctp->src_port = fdir_input->flow.sctp6_flow.dst_port;
1277                 sctp->dst_port = fdir_input->flow.sctp6_flow.src_port;
1278                 sctp->tag = fdir_input->flow.sctp6_flow.verify_tag;
1279         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1280                    pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) {
1281                 payload = raw_pkt + len;
1282                 set_idx = I40E_FLXPLD_L3_IDX;
1283         } else if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD) {
1284                 payload = raw_pkt + len;
1285                 /**
1286                  * ARP packet is a special case on which the payload
1287                  * starts after the whole ARP header
1288                  */
1289                 if (fdir_input->flow.l2_flow.ether_type ==
1290                                 rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
1291                         payload += sizeof(struct rte_arp_hdr);
1292                 set_idx = I40E_FLXPLD_L2_IDX;
1293         } else if (fdir_input->flow_ext.customized_pctype) {
1294                 /* If customized pctype is used */
1295                 cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype);
1296                 if (cus_pctype->index == I40E_CUSTOMIZED_GTPC ||
1297                     cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 ||
1298                     cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 ||
1299                     cus_pctype->index == I40E_CUSTOMIZED_GTPU) {
1300                         udp = (struct rte_udp_hdr *)(raw_pkt + len);
1301                         udp->dgram_len =
1302                                 rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
1303
1304                         gtp = (struct rte_flow_item_gtp *)
1305                                 ((unsigned char *)udp +
1306                                         sizeof(struct rte_udp_hdr));
1307                         gtp->msg_len =
1308                                 rte_cpu_to_be_16(I40E_FDIR_GTP_DEFAULT_LEN);
1309                         gtp->teid = fdir_input->flow.gtp_flow.teid;
1310                         gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0X01;
1311
1312                         /* GTP-C message type is not supported. */
1313                         if (cus_pctype->index == I40E_CUSTOMIZED_GTPC) {
1314                                 udp->dst_port =
1315                                       rte_cpu_to_be_16(I40E_FDIR_GTPC_DST_PORT);
1316                                 gtp->v_pt_rsv_flags =
1317                                         I40E_FDIR_GTP_VER_FLAG_0X32;
1318                         } else {
1319                                 udp->dst_port =
1320                                       rte_cpu_to_be_16(I40E_FDIR_GTPU_DST_PORT);
1321                                 gtp->v_pt_rsv_flags =
1322                                         I40E_FDIR_GTP_VER_FLAG_0X30;
1323                         }
1324
1325                         if (cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4) {
1326                                 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF;
1327                                 gtp_ipv4 = (struct rte_ipv4_hdr *)
1328                                         ((unsigned char *)gtp +
1329                                          sizeof(struct rte_flow_item_gtp));
1330                                 gtp_ipv4->version_ihl =
1331                                         I40E_FDIR_IP_DEFAULT_VERSION_IHL;
1332                                 gtp_ipv4->next_proto_id = IPPROTO_IP;
1333                                 gtp_ipv4->total_length =
1334                                         rte_cpu_to_be_16(
1335                                                 I40E_FDIR_INNER_IP_DEFAULT_LEN);
1336                                 payload = (unsigned char *)gtp_ipv4 +
1337                                         sizeof(struct rte_ipv4_hdr);
1338                         } else if (cus_pctype->index ==
1339                                    I40E_CUSTOMIZED_GTPU_IPV6) {
1340                                 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF;
1341                                 gtp_ipv6 = (struct rte_ipv6_hdr *)
1342                                         ((unsigned char *)gtp +
1343                                          sizeof(struct rte_flow_item_gtp));
1344                                 gtp_ipv6->vtc_flow =
1345                                         rte_cpu_to_be_32(
1346                                                I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
1347                                                (0 << I40E_FDIR_IPv6_TC_OFFSET));
1348                                 gtp_ipv6->proto = IPPROTO_NONE;
1349                                 gtp_ipv6->payload_len =
1350                                         rte_cpu_to_be_16(
1351                                               I40E_FDIR_INNER_IPV6_DEFAULT_LEN);
1352                                 gtp_ipv6->hop_limits =
1353                                         I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
1354                                 payload = (unsigned char *)gtp_ipv6 +
1355                                         sizeof(struct rte_ipv6_hdr);
1356                         } else
1357                                 payload = (unsigned char *)gtp +
1358                                         sizeof(struct rte_flow_item_gtp);
1359                 } else if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3 ||
1360                            cus_pctype->index == I40E_CUSTOMIZED_IPV6_L2TPV3) {
1361                         l2tpv3oip = (struct rte_flow_item_l2tpv3oip *)(raw_pkt
1362                                                                        + len);
1363
1364                         if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3)
1365                                 l2tpv3oip->session_id =
1366                                  fdir_input->flow.ip4_l2tpv3oip_flow.session_id;
1367                         else
1368                                 l2tpv3oip->session_id =
1369                                  fdir_input->flow.ip6_l2tpv3oip_flow.session_id;
1370                         payload = (unsigned char *)l2tpv3oip +
1371                                 sizeof(struct rte_flow_item_l2tpv3oip);
1372                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4 ||
1373                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6 ||
1374                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP ||
1375                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6_UDP) {
1376                         if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4) {
1377                                 esp_ipv4 = (struct rte_ipv4_hdr *)
1378                                         (raw_pkt + len);
1379                                 esp = (struct rte_flow_item_esp *)esp_ipv4;
1380                                 esp->hdr.spi =
1381                                         fdir_input->flow.esp_ipv4_flow.spi;
1382                                 payload = (unsigned char *)esp +
1383                                         sizeof(struct rte_esp_hdr);
1384                                 len += sizeof(struct rte_esp_hdr);
1385                         } else if (cus_pctype->index ==
1386                                         I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1387                                 esp_ipv4 = (struct rte_ipv4_hdr *)
1388                                         (raw_pkt + len);
1389                                 udp = (struct rte_udp_hdr *)esp_ipv4;
1390                                 udp->dst_port = rte_cpu_to_be_16
1391                                         (I40E_FDIR_ESP_DST_PORT);
1392
1393                                 udp->dgram_len = rte_cpu_to_be_16
1394                                                 (I40E_FDIR_UDP_DEFAULT_LEN);
1395                                 esp = (struct rte_flow_item_esp *)
1396                                         ((unsigned char *)esp_ipv4 +
1397                                                 sizeof(struct rte_udp_hdr));
1398                                 esp->hdr.spi =
1399                                         fdir_input->flow.esp_ipv4_udp_flow.spi;
1400                                 payload = (unsigned char *)esp +
1401                                         sizeof(struct rte_esp_hdr);
1402                                 len += sizeof(struct rte_udp_hdr) +
1403                                                 sizeof(struct rte_esp_hdr);
1404                         } else if (cus_pctype->index ==
1405                                         I40E_CUSTOMIZED_ESP_IPV6) {
1406                                 esp_ipv6 = (struct rte_ipv6_hdr *)
1407                                         (raw_pkt + len);
1408                                 esp = (struct rte_flow_item_esp *)esp_ipv6;
1409                                 esp->hdr.spi =
1410                                         fdir_input->flow.esp_ipv6_flow.spi;
1411                                 payload = (unsigned char *)esp +
1412                                         sizeof(struct rte_esp_hdr);
1413                                 len += sizeof(struct rte_esp_hdr);
1414                         } else if (cus_pctype->index ==
1415                                         I40E_CUSTOMIZED_ESP_IPV6_UDP) {
1416                                 esp_ipv6 = (struct rte_ipv6_hdr *)
1417                                         (raw_pkt + len);
1418                                 udp = (struct rte_udp_hdr *)esp_ipv6;
1419                                 udp->dst_port = rte_cpu_to_be_16
1420                                         (I40E_FDIR_ESP_DST_PORT);
1421
1422                                 udp->dgram_len = rte_cpu_to_be_16
1423                                         (I40E_FDIR_UDP_DEFAULT_LEN);
1424                                 esp = (struct rte_flow_item_esp *)
1425                                         ((unsigned char *)esp_ipv6 +
1426                                                 sizeof(struct rte_udp_hdr));
1427                                 esp->hdr.spi =
1428                                         fdir_input->flow.esp_ipv6_udp_flow.spi;
1429                                 payload = (unsigned char *)esp +
1430                                         sizeof(struct rte_esp_hdr);
1431                                 len += sizeof(struct rte_udp_hdr) +
1432                                                 sizeof(struct rte_esp_hdr);
1433                         }
1434                 }
1435         } else {
1436                 PMD_DRV_LOG(ERR, "unknown pctype %u.", fdir_input->pctype);
1437                 return -1;
1438         }
1439
1440         /* fill the flexbytes to payload */
1441         for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
1442                 pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i;
1443                 size = pf->fdir.flex_set[pit_idx].size;
1444                 if (size == 0)
1445                         continue;
1446                 dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t);
1447                 ptr = payload +
1448                       pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t);
1449                 (void)rte_memcpy(ptr,
1450                                  &fdir_input->flow_ext.flexbytes[dst],
1451                                  size * sizeof(uint16_t));
1452         }
1453
1454         return 0;
1455 }
1456
1457 /* Construct the tx flags */
1458 static inline uint64_t
1459 i40e_build_ctob(uint32_t td_cmd,
1460                 uint32_t td_offset,
1461                 unsigned int size,
1462                 uint32_t td_tag)
1463 {
1464         return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA |
1465                         ((uint64_t)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
1466                         ((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
1467                         ((uint64_t)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
1468                         ((uint64_t)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
1469 }
1470
1471 /*
1472  * check the programming status descriptor in rx queue.
1473  * done after Programming Flow Director is programmed on
1474  * tx queue
1475  */
1476 static inline int
1477 i40e_check_fdir_programming_status(struct i40e_rx_queue *rxq)
1478 {
1479         volatile union i40e_rx_desc *rxdp;
1480         uint64_t qword1;
1481         uint32_t rx_status;
1482         uint32_t len, id;
1483         uint32_t error;
1484         int ret = 0;
1485
1486         rxdp = &rxq->rx_ring[rxq->rx_tail];
1487         qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
1488         rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK)
1489                         >> I40E_RXD_QW1_STATUS_SHIFT;
1490
1491         if (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1492                 len = qword1 >> I40E_RX_PROG_STATUS_DESC_LENGTH_SHIFT;
1493                 id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
1494                             I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
1495
1496                 if (len  == I40E_RX_PROG_STATUS_DESC_LENGTH &&
1497                     id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) {
1498                         error = (qword1 &
1499                                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
1500                                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
1501                         if (error == (0x1 <<
1502                                 I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
1503                                 PMD_DRV_LOG(ERR, "Failed to add FDIR filter"
1504                                             " (FD_ID %u): programming status"
1505                                             " reported.",
1506                                             rxdp->wb.qword0.hi_dword.fd_id);
1507                                 ret = -1;
1508                         } else if (error == (0x1 <<
1509                                 I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
1510                                 PMD_DRV_LOG(ERR, "Failed to delete FDIR filter"
1511                                             " (FD_ID %u): programming status"
1512                                             " reported.",
1513                                             rxdp->wb.qword0.hi_dword.fd_id);
1514                                 ret = -1;
1515                         } else
1516                                 PMD_DRV_LOG(ERR, "invalid programming status"
1517                                             " reported, error = %u.", error);
1518                 } else
1519                         PMD_DRV_LOG(INFO, "unknown programming status"
1520                                     " reported, len = %d, id = %u.", len, id);
1521                 rxdp->wb.qword1.status_error_len = 0;
1522                 rxq->rx_tail++;
1523                 if (unlikely(rxq->rx_tail == rxq->nb_rx_desc))
1524                         rxq->rx_tail = 0;
1525                 if (rxq->rx_tail == 0)
1526                         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1527                 else
1528                         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_tail - 1);
1529         }
1530
1531         return ret;
1532 }
1533
1534 static int
1535 i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input,
1536                          struct i40e_fdir_filter *filter)
1537 {
1538         rte_memcpy(&filter->fdir, input, sizeof(struct i40e_fdir_filter_conf));
1539         if (input->input.flow_ext.pkt_template) {
1540                 filter->fdir.input.flow.raw_flow.packet = NULL;
1541                 filter->fdir.input.flow.raw_flow.length =
1542                         rte_hash_crc(input->input.flow.raw_flow.packet,
1543                                      input->input.flow.raw_flow.length,
1544                                      input->input.flow.raw_flow.pctype);
1545         }
1546         return 0;
1547 }
1548
1549 /* Check if there exists the flow director filter */
1550 static struct i40e_fdir_filter *
1551 i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info,
1552                         const struct i40e_fdir_input *input)
1553 {
1554         int ret;
1555
1556         if (input->flow_ext.pkt_template)
1557                 ret = rte_hash_lookup_with_hash(fdir_info->hash_table,
1558                                                 (const void *)input,
1559                                                 input->flow.raw_flow.length);
1560         else
1561                 ret = rte_hash_lookup(fdir_info->hash_table,
1562                                       (const void *)input);
1563         if (ret < 0)
1564                 return NULL;
1565
1566         return fdir_info->hash_map[ret];
1567 }
1568
1569 /* Add a flow director filter into the SW list */
1570 static int
1571 i40e_sw_fdir_filter_insert(struct i40e_pf *pf, struct i40e_fdir_filter *filter)
1572 {
1573         struct i40e_fdir_info *fdir_info = &pf->fdir;
1574         struct i40e_fdir_filter *hash_filter;
1575         int ret;
1576
1577         if (filter->fdir.input.flow_ext.pkt_template)
1578                 ret = rte_hash_add_key_with_hash(fdir_info->hash_table,
1579                                  &filter->fdir.input,
1580                                  filter->fdir.input.flow.raw_flow.length);
1581         else
1582                 ret = rte_hash_add_key(fdir_info->hash_table,
1583                                        &filter->fdir.input);
1584         if (ret < 0) {
1585                 PMD_DRV_LOG(ERR,
1586                             "Failed to insert fdir filter to hash table %d!",
1587                             ret);
1588                 return ret;
1589         }
1590
1591         if (fdir_info->hash_map[ret])
1592                 return -1;
1593
1594         hash_filter = &fdir_info->fdir_filter_array[ret];
1595         rte_memcpy(hash_filter, filter, sizeof(*filter));
1596         fdir_info->hash_map[ret] = hash_filter;
1597         TAILQ_INSERT_TAIL(&fdir_info->fdir_list, hash_filter, rules);
1598
1599         return 0;
1600 }
1601
1602 /* Delete a flow director filter from the SW list */
1603 int
1604 i40e_sw_fdir_filter_del(struct i40e_pf *pf, struct i40e_fdir_input *input)
1605 {
1606         struct i40e_fdir_info *fdir_info = &pf->fdir;
1607         struct i40e_fdir_filter *filter;
1608         int ret;
1609
1610         if (input->flow_ext.pkt_template)
1611                 ret = rte_hash_del_key_with_hash(fdir_info->hash_table,
1612                                                  input,
1613                                                  input->flow.raw_flow.length);
1614         else
1615                 ret = rte_hash_del_key(fdir_info->hash_table, input);
1616         if (ret < 0) {
1617                 PMD_DRV_LOG(ERR,
1618                             "Failed to delete fdir filter to hash table %d!",
1619                             ret);
1620                 return ret;
1621         }
1622         filter = fdir_info->hash_map[ret];
1623         fdir_info->hash_map[ret] = NULL;
1624
1625         TAILQ_REMOVE(&fdir_info->fdir_list, filter, rules);
1626
1627         return 0;
1628 }
1629
1630 struct rte_flow *
1631 i40e_fdir_entry_pool_get(struct i40e_fdir_info *fdir_info)
1632 {
1633         struct rte_flow *flow = NULL;
1634         uint64_t slab = 0;
1635         uint32_t pos = 0;
1636         uint32_t i = 0;
1637         int ret;
1638
1639         if (fdir_info->fdir_actual_cnt >=
1640                         fdir_info->fdir_space_size) {
1641                 PMD_DRV_LOG(ERR, "Fdir space full");
1642                 return NULL;
1643         }
1644
1645         ret = rte_bitmap_scan(fdir_info->fdir_flow_pool.bitmap, &pos,
1646                         &slab);
1647
1648         /* normally this won't happen as the fdir_actual_cnt should be
1649          * same with the number of the set bits in fdir_flow_pool,
1650          * but anyway handle this error condition here for safe
1651          */
1652         if (ret == 0) {
1653                 PMD_DRV_LOG(ERR, "fdir_actual_cnt out of sync");
1654                 return NULL;
1655         }
1656
1657         i = rte_bsf64(slab);
1658         pos += i;
1659         rte_bitmap_clear(fdir_info->fdir_flow_pool.bitmap, pos);
1660         flow = &fdir_info->fdir_flow_pool.pool[pos].flow;
1661
1662         memset(flow, 0, sizeof(struct rte_flow));
1663
1664         return flow;
1665 }
1666
1667 void
1668 i40e_fdir_entry_pool_put(struct i40e_fdir_info *fdir_info,
1669                 struct rte_flow *flow)
1670 {
1671         struct i40e_fdir_entry *f;
1672
1673         f = FLOW_TO_FLOW_BITMAP(flow);
1674         rte_bitmap_set(fdir_info->fdir_flow_pool.bitmap, f->idx);
1675 }
1676
1677 /*
1678  * i40e_add_del_fdir_filter - add or remove a flow director filter.
1679  * @pf: board private structure
1680  * @filter: fdir filter entry
1681  * @add: 0 - delete, 1 - add
1682  */
1683 int
1684 i40e_add_del_fdir_filter(struct rte_eth_dev *dev,
1685                          const struct rte_eth_fdir_filter *filter,
1686                          bool add)
1687 {
1688         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1689         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1690         unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
1691         enum i40e_filter_pctype pctype;
1692         int ret = 0;
1693
1694         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_PERFECT) {
1695                 PMD_DRV_LOG(ERR, "FDIR is not enabled, please"
1696                         " check the mode in fdir_conf.");
1697                 return -ENOTSUP;
1698         }
1699
1700         pctype = i40e_flowtype_to_pctype(pf->adapter, filter->input.flow_type);
1701         if (pctype == I40E_FILTER_PCTYPE_INVALID) {
1702                 PMD_DRV_LOG(ERR, "invalid flow_type input.");
1703                 return -EINVAL;
1704         }
1705         if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) {
1706                 PMD_DRV_LOG(ERR, "Invalid queue ID");
1707                 return -EINVAL;
1708         }
1709         if (filter->input.flow_ext.is_vf &&
1710                 filter->input.flow_ext.dst_id >= pf->vf_num) {
1711                 PMD_DRV_LOG(ERR, "Invalid VF ID");
1712                 return -EINVAL;
1713         }
1714
1715         memset(pkt, 0, I40E_FDIR_PKT_LEN);
1716
1717         ret = i40e_fdir_construct_pkt(pf, &filter->input, pkt);
1718         if (ret < 0) {
1719                 PMD_DRV_LOG(ERR, "construct packet for fdir fails.");
1720                 return ret;
1721         }
1722
1723         if (hw->mac.type == I40E_MAC_X722) {
1724                 /* get translated pctype value in fd pctype register */
1725                 pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(
1726                         hw, I40E_GLQF_FD_PCTYPES((int)pctype));
1727         }
1728
1729         ret = i40e_fdir_filter_programming(pf, pctype, filter, add);
1730         if (ret < 0) {
1731                 PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).",
1732                             pctype);
1733                 return ret;
1734         }
1735
1736         return ret;
1737 }
1738
1739 /**
1740  * i40e_flow_add_del_fdir_filter - add or remove a flow director filter.
1741  * @pf: board private structure
1742  * @filter: fdir filter entry
1743  * @add: 0 - delete, 1 - add
1744  */
1745 int
1746 i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev,
1747                               const struct i40e_fdir_filter_conf *filter,
1748                               bool add)
1749 {
1750         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1751         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1752         unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
1753         enum i40e_filter_pctype pctype;
1754         struct i40e_fdir_info *fdir_info = &pf->fdir;
1755         struct i40e_fdir_filter *node;
1756         struct i40e_fdir_filter check_filter; /* Check if the filter exists */
1757         int ret = 0;
1758
1759         if (pf->fdir.fdir_vsi == NULL) {
1760                 PMD_DRV_LOG(ERR, "FDIR is not enabled");
1761                 return -ENOTSUP;
1762         }
1763
1764         if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) {
1765                 PMD_DRV_LOG(ERR, "Invalid queue ID");
1766                 return -EINVAL;
1767         }
1768         if (filter->input.flow_ext.is_vf &&
1769             filter->input.flow_ext.dst_id >= pf->vf_num) {
1770                 PMD_DRV_LOG(ERR, "Invalid VF ID");
1771                 return -EINVAL;
1772         }
1773         if (filter->input.flow_ext.pkt_template) {
1774                 if (filter->input.flow.raw_flow.length > I40E_FDIR_PKT_LEN ||
1775                     !filter->input.flow.raw_flow.packet) {
1776                         PMD_DRV_LOG(ERR, "Invalid raw packet template"
1777                                 " flow filter parameters!");
1778                         return -EINVAL;
1779                 }
1780                 pctype = filter->input.flow.raw_flow.pctype;
1781         } else {
1782                 pctype = filter->input.pctype;
1783         }
1784
1785         /* Check if there is the filter in SW list */
1786         memset(&check_filter, 0, sizeof(check_filter));
1787         i40e_fdir_filter_convert(filter, &check_filter);
1788
1789         if (add) {
1790                 ret = i40e_sw_fdir_filter_insert(pf, &check_filter);
1791                 if (ret < 0) {
1792                         PMD_DRV_LOG(ERR,
1793                                     "Conflict with existing flow director rules!");
1794                         return -EINVAL;
1795                 }
1796         } else {
1797                 node = i40e_sw_fdir_filter_lookup(fdir_info,
1798                                 &check_filter.fdir.input);
1799                 if (!node) {
1800                         PMD_DRV_LOG(ERR,
1801                                     "There's no corresponding flow firector filter!");
1802                         return -EINVAL;
1803                 }
1804
1805                 ret = i40e_sw_fdir_filter_del(pf, &node->fdir.input);
1806                 if (ret < 0) {
1807                         PMD_DRV_LOG(ERR,
1808                                         "Error deleting fdir rule from hash table!");
1809                         return -EINVAL;
1810                 }
1811         }
1812
1813         memset(pkt, 0, I40E_FDIR_PKT_LEN);
1814         ret = i40e_flow_fdir_construct_pkt(pf, &filter->input, pkt);
1815         if (ret < 0) {
1816                 PMD_DRV_LOG(ERR, "construct packet for fdir fails.");
1817                 goto error_op;
1818         }
1819
1820         if (hw->mac.type == I40E_MAC_X722) {
1821                 /* get translated pctype value in fd pctype register */
1822                 pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(
1823                         hw, I40E_GLQF_FD_PCTYPES((int)pctype));
1824         }
1825
1826         ret = i40e_flow_fdir_filter_programming(pf, pctype, filter, add);
1827         if (ret < 0) {
1828                 PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).",
1829                             pctype);
1830                 goto error_op;
1831         }
1832
1833         if (add) {
1834                 fdir_info->fdir_actual_cnt++;
1835                 if (fdir_info->fdir_invalprio == 1 &&
1836                                 fdir_info->fdir_guarantee_free_space > 0)
1837                         fdir_info->fdir_guarantee_free_space--;
1838         } else {
1839                 fdir_info->fdir_actual_cnt--;
1840                 if (fdir_info->fdir_invalprio == 1 &&
1841                                 fdir_info->fdir_guarantee_free_space <
1842                                 fdir_info->fdir_guarantee_total_space)
1843                         fdir_info->fdir_guarantee_free_space++;
1844         }
1845
1846         return ret;
1847
1848 error_op:
1849         /* roll back */
1850         if (add)
1851                 i40e_sw_fdir_filter_del(pf, &check_filter.fdir.input);
1852         else
1853                 i40e_sw_fdir_filter_insert(pf, &check_filter);
1854
1855         return ret;
1856 }
1857
1858 /*
1859  * i40e_fdir_filter_programming - Program a flow director filter rule.
1860  * Is done by Flow Director Programming Descriptor followed by packet
1861  * structure that contains the filter fields need to match.
1862  * @pf: board private structure
1863  * @pctype: pctype
1864  * @filter: fdir filter entry
1865  * @add: 0 - delete, 1 - add
1866  */
1867 static int
1868 i40e_fdir_filter_programming(struct i40e_pf *pf,
1869                         enum i40e_filter_pctype pctype,
1870                         const struct rte_eth_fdir_filter *filter,
1871                         bool add)
1872 {
1873         struct i40e_tx_queue *txq = pf->fdir.txq;
1874         struct i40e_rx_queue *rxq = pf->fdir.rxq;
1875         const struct rte_eth_fdir_action *fdir_action = &filter->action;
1876         volatile struct i40e_tx_desc *txdp;
1877         volatile struct i40e_filter_program_desc *fdirdp;
1878         uint32_t td_cmd;
1879         uint16_t vsi_id, i;
1880         uint8_t dest;
1881
1882         PMD_DRV_LOG(INFO, "filling filter programming descriptor.");
1883         fdirdp = (volatile struct i40e_filter_program_desc *)
1884                         (&(txq->tx_ring[txq->tx_tail]));
1885
1886         fdirdp->qindex_flex_ptype_vsi =
1887                         rte_cpu_to_le_32((fdir_action->rx_queue <<
1888                                           I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1889                                           I40E_TXD_FLTR_QW0_QINDEX_MASK);
1890
1891         fdirdp->qindex_flex_ptype_vsi |=
1892                         rte_cpu_to_le_32((fdir_action->flex_off <<
1893                                           I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
1894                                           I40E_TXD_FLTR_QW0_FLEXOFF_MASK);
1895
1896         fdirdp->qindex_flex_ptype_vsi |=
1897                         rte_cpu_to_le_32((pctype <<
1898                                           I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
1899                                           I40E_TXD_FLTR_QW0_PCTYPE_MASK);
1900
1901         if (filter->input.flow_ext.is_vf)
1902                 vsi_id = pf->vfs[filter->input.flow_ext.dst_id].vsi->vsi_id;
1903         else
1904                 /* Use LAN VSI Id by default */
1905                 vsi_id = pf->main_vsi->vsi_id;
1906         fdirdp->qindex_flex_ptype_vsi |=
1907                 rte_cpu_to_le_32(((uint32_t)vsi_id <<
1908                                   I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
1909                                   I40E_TXD_FLTR_QW0_DEST_VSI_MASK);
1910
1911         fdirdp->dtype_cmd_cntindex =
1912                         rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG);
1913
1914         if (add)
1915                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1916                                 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1917                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1918         else
1919                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1920                                 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1921                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1922
1923         if (fdir_action->behavior == RTE_ETH_FDIR_REJECT)
1924                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
1925         else if (fdir_action->behavior == RTE_ETH_FDIR_ACCEPT)
1926                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX;
1927         else if (fdir_action->behavior == RTE_ETH_FDIR_PASSTHRU)
1928                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_OTHER;
1929         else {
1930                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter:"
1931                             " unsupported fdir behavior.");
1932                 return -EINVAL;
1933         }
1934
1935         fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest <<
1936                                 I40E_TXD_FLTR_QW1_DEST_SHIFT) &
1937                                 I40E_TXD_FLTR_QW1_DEST_MASK);
1938
1939         fdirdp->dtype_cmd_cntindex |=
1940                 rte_cpu_to_le_32((fdir_action->report_status<<
1941                                 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
1942                                 I40E_TXD_FLTR_QW1_FD_STATUS_MASK);
1943
1944         fdirdp->dtype_cmd_cntindex |=
1945                         rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK);
1946         fdirdp->dtype_cmd_cntindex |=
1947                         rte_cpu_to_le_32(
1948                         ((uint32_t)pf->fdir.match_counter_index <<
1949                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
1950                         I40E_TXD_FLTR_QW1_CNTINDEX_MASK);
1951
1952         fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id);
1953
1954         PMD_DRV_LOG(INFO, "filling transmit descriptor.");
1955         txdp = &(txq->tx_ring[txq->tx_tail + 1]);
1956         txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
1957         td_cmd = I40E_TX_DESC_CMD_EOP |
1958                  I40E_TX_DESC_CMD_RS  |
1959                  I40E_TX_DESC_CMD_DUMMY;
1960
1961         txdp->cmd_type_offset_bsz =
1962                 i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0);
1963
1964         txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */
1965         if (txq->tx_tail >= txq->nb_tx_desc)
1966                 txq->tx_tail = 0;
1967         /* Update the tx tail register */
1968         rte_wmb();
1969         I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
1970         for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
1971                 if ((txdp->cmd_type_offset_bsz &
1972                                 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
1973                                 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
1974                         break;
1975                 rte_delay_us(1);
1976         }
1977         if (i >= I40E_FDIR_MAX_WAIT_US) {
1978                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter:"
1979                             " time out to get DD on tx queue.");
1980                 return -ETIMEDOUT;
1981         }
1982         /* totally delay 10 ms to check programming status*/
1983         for (; i < I40E_FDIR_MAX_WAIT_US; i++) {
1984                 if (i40e_check_fdir_programming_status(rxq) >= 0)
1985                         return 0;
1986                 rte_delay_us(1);
1987         }
1988         PMD_DRV_LOG(ERR,
1989                 "Failed to program FDIR filter: programming status reported.");
1990         return -ETIMEDOUT;
1991 }
1992
1993 /*
1994  * i40e_flow_fdir_filter_programming - Program a flow director filter rule.
1995  * Is done by Flow Director Programming Descriptor followed by packet
1996  * structure that contains the filter fields need to match.
1997  * @pf: board private structure
1998  * @pctype: pctype
1999  * @filter: fdir filter entry
2000  * @add: 0 - delete, 1 - add
2001  */
2002 static int
2003 i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
2004                                   enum i40e_filter_pctype pctype,
2005                                   const struct i40e_fdir_filter_conf *filter,
2006                                   bool add)
2007 {
2008         struct i40e_tx_queue *txq = pf->fdir.txq;
2009         struct i40e_rx_queue *rxq = pf->fdir.rxq;
2010         const struct i40e_fdir_action *fdir_action = &filter->action;
2011         volatile struct i40e_tx_desc *txdp;
2012         volatile struct i40e_filter_program_desc *fdirdp;
2013         uint32_t td_cmd;
2014         uint16_t vsi_id, i;
2015         uint8_t dest;
2016
2017         PMD_DRV_LOG(INFO, "filling filter programming descriptor.");
2018         fdirdp = (volatile struct i40e_filter_program_desc *)
2019                                 (&txq->tx_ring[txq->tx_tail]);
2020
2021         fdirdp->qindex_flex_ptype_vsi =
2022                         rte_cpu_to_le_32((fdir_action->rx_queue <<
2023                                           I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
2024                                           I40E_TXD_FLTR_QW0_QINDEX_MASK);
2025
2026         fdirdp->qindex_flex_ptype_vsi |=
2027                         rte_cpu_to_le_32((fdir_action->flex_off <<
2028                                           I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
2029                                           I40E_TXD_FLTR_QW0_FLEXOFF_MASK);
2030
2031         fdirdp->qindex_flex_ptype_vsi |=
2032                         rte_cpu_to_le_32((pctype <<
2033                                           I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
2034                                           I40E_TXD_FLTR_QW0_PCTYPE_MASK);
2035
2036         if (filter->input.flow_ext.is_vf)
2037                 vsi_id = pf->vfs[filter->input.flow_ext.dst_id].vsi->vsi_id;
2038         else
2039                 /* Use LAN VSI Id by default */
2040                 vsi_id = pf->main_vsi->vsi_id;
2041         fdirdp->qindex_flex_ptype_vsi |=
2042                 rte_cpu_to_le_32(((uint32_t)vsi_id <<
2043                                   I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
2044                                   I40E_TXD_FLTR_QW0_DEST_VSI_MASK);
2045
2046         fdirdp->dtype_cmd_cntindex =
2047                         rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG);
2048
2049         if (add)
2050                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
2051                                 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
2052                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
2053         else
2054                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
2055                                 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
2056                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
2057
2058         if (fdir_action->behavior == I40E_FDIR_REJECT)
2059                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
2060         else if (fdir_action->behavior == I40E_FDIR_ACCEPT)
2061                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX;
2062         else if (fdir_action->behavior == I40E_FDIR_PASSTHRU)
2063                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_OTHER;
2064         else {
2065                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter: unsupported fdir behavior.");
2066                 return -EINVAL;
2067         }
2068
2069         fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest <<
2070                                 I40E_TXD_FLTR_QW1_DEST_SHIFT) &
2071                                 I40E_TXD_FLTR_QW1_DEST_MASK);
2072
2073         fdirdp->dtype_cmd_cntindex |=
2074                 rte_cpu_to_le_32((fdir_action->report_status <<
2075                                 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
2076                                 I40E_TXD_FLTR_QW1_FD_STATUS_MASK);
2077
2078         fdirdp->dtype_cmd_cntindex |=
2079                         rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK);
2080         fdirdp->dtype_cmd_cntindex |=
2081                         rte_cpu_to_le_32(
2082                         ((uint32_t)pf->fdir.match_counter_index <<
2083                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
2084                         I40E_TXD_FLTR_QW1_CNTINDEX_MASK);
2085
2086         fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id);
2087
2088         PMD_DRV_LOG(INFO, "filling transmit descriptor.");
2089         txdp = &txq->tx_ring[txq->tx_tail + 1];
2090         txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
2091         td_cmd = I40E_TX_DESC_CMD_EOP |
2092                  I40E_TX_DESC_CMD_RS  |
2093                  I40E_TX_DESC_CMD_DUMMY;
2094
2095         txdp->cmd_type_offset_bsz =
2096                 i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0);
2097
2098         txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */
2099         if (txq->tx_tail >= txq->nb_tx_desc)
2100                 txq->tx_tail = 0;
2101         /* Update the tx tail register */
2102         rte_wmb();
2103         I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
2104         for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
2105                 if ((txdp->cmd_type_offset_bsz &
2106                                 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
2107                                 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
2108                         break;
2109                 rte_delay_us(1);
2110         }
2111         if (i >= I40E_FDIR_MAX_WAIT_US) {
2112                 PMD_DRV_LOG(ERR,
2113                     "Failed to program FDIR filter: time out to get DD on tx queue.");
2114                 return -ETIMEDOUT;
2115         }
2116         /* totally delay 10 ms to check programming status*/
2117         rte_delay_us(I40E_FDIR_MAX_WAIT_US);
2118         if (i40e_check_fdir_programming_status(rxq) < 0) {
2119                 PMD_DRV_LOG(ERR,
2120                     "Failed to program FDIR filter: programming status reported.");
2121                 return -ETIMEDOUT;
2122         }
2123
2124         return 0;
2125 }
2126
2127 /*
2128  * i40e_fdir_flush - clear all filters of Flow Director table
2129  * @pf: board private structure
2130  */
2131 int
2132 i40e_fdir_flush(struct rte_eth_dev *dev)
2133 {
2134         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2135         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2136         uint32_t reg;
2137         uint16_t guarant_cnt, best_cnt;
2138         uint16_t i;
2139
2140         I40E_WRITE_REG(hw, I40E_PFQF_CTL_1, I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
2141         I40E_WRITE_FLUSH(hw);
2142
2143         for (i = 0; i < I40E_FDIR_FLUSH_RETRY; i++) {
2144                 rte_delay_ms(I40E_FDIR_FLUSH_INTERVAL_MS);
2145                 reg = I40E_READ_REG(hw, I40E_PFQF_CTL_1);
2146                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
2147                         break;
2148         }
2149         if (i >= I40E_FDIR_FLUSH_RETRY) {
2150                 PMD_DRV_LOG(ERR, "FD table did not flush, may need more time.");
2151                 return -ETIMEDOUT;
2152         }
2153         guarant_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
2154                                 I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2155                                 I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2156         best_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
2157                                 I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2158                                 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2159         if (guarant_cnt != 0 || best_cnt != 0) {
2160                 PMD_DRV_LOG(ERR, "Failed to flush FD table.");
2161                 return -ENOSYS;
2162         } else
2163                 PMD_DRV_LOG(INFO, "FD table Flush success.");
2164         return 0;
2165 }
2166
2167 static inline void
2168 i40e_fdir_info_get_flex_set(struct i40e_pf *pf,
2169                         struct rte_eth_flex_payload_cfg *flex_set,
2170                         uint16_t *num)
2171 {
2172         struct i40e_fdir_flex_pit *flex_pit;
2173         struct rte_eth_flex_payload_cfg *ptr = flex_set;
2174         uint16_t src, dst, size, j, k;
2175         uint8_t i, layer_idx;
2176
2177         for (layer_idx = I40E_FLXPLD_L2_IDX;
2178              layer_idx <= I40E_FLXPLD_L4_IDX;
2179              layer_idx++) {
2180                 if (layer_idx == I40E_FLXPLD_L2_IDX)
2181                         ptr->type = RTE_ETH_L2_PAYLOAD;
2182                 else if (layer_idx == I40E_FLXPLD_L3_IDX)
2183                         ptr->type = RTE_ETH_L3_PAYLOAD;
2184                 else if (layer_idx == I40E_FLXPLD_L4_IDX)
2185                         ptr->type = RTE_ETH_L4_PAYLOAD;
2186
2187                 for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
2188                         flex_pit = &pf->fdir.flex_set[layer_idx *
2189                                 I40E_MAX_FLXPLD_FIED + i];
2190                         if (flex_pit->size == 0)
2191                                 continue;
2192                         src = flex_pit->src_offset * sizeof(uint16_t);
2193                         dst = flex_pit->dst_offset * sizeof(uint16_t);
2194                         size = flex_pit->size * sizeof(uint16_t);
2195                         for (j = src, k = dst; j < src + size; j++, k++)
2196                                 ptr->src_offset[k] = j;
2197                 }
2198                 (*num)++;
2199                 ptr++;
2200         }
2201 }
2202
2203 static inline void
2204 i40e_fdir_info_get_flex_mask(struct i40e_pf *pf,
2205                         struct rte_eth_fdir_flex_mask *flex_mask,
2206                         uint16_t *num)
2207 {
2208         struct i40e_fdir_flex_mask *mask;
2209         struct rte_eth_fdir_flex_mask *ptr = flex_mask;
2210         uint16_t flow_type;
2211         uint8_t i, j;
2212         uint16_t off_bytes, mask_tmp;
2213
2214         for (i = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
2215              i <= I40E_FILTER_PCTYPE_L2_PAYLOAD;
2216              i++) {
2217                 mask =  &pf->fdir.flex_mask[i];
2218                 flow_type = i40e_pctype_to_flowtype(pf->adapter,
2219                                                     (enum i40e_filter_pctype)i);
2220                 if (flow_type == RTE_ETH_FLOW_UNKNOWN)
2221                         continue;
2222
2223                 for (j = 0; j < I40E_FDIR_MAX_FLEXWORD_NUM; j++) {
2224                         if (mask->word_mask & I40E_FLEX_WORD_MASK(j)) {
2225                                 ptr->mask[j * sizeof(uint16_t)] = UINT8_MAX;
2226                                 ptr->mask[j * sizeof(uint16_t) + 1] = UINT8_MAX;
2227                         } else {
2228                                 ptr->mask[j * sizeof(uint16_t)] = 0x0;
2229                                 ptr->mask[j * sizeof(uint16_t) + 1] = 0x0;
2230                         }
2231                 }
2232                 for (j = 0; j < I40E_FDIR_BITMASK_NUM_WORD; j++) {
2233                         off_bytes = mask->bitmask[j].offset * sizeof(uint16_t);
2234                         mask_tmp = ~mask->bitmask[j].mask;
2235                         ptr->mask[off_bytes] &= I40E_HI_BYTE(mask_tmp);
2236                         ptr->mask[off_bytes + 1] &= I40E_LO_BYTE(mask_tmp);
2237                 }
2238                 ptr->flow_type = flow_type;
2239                 ptr++;
2240                 (*num)++;
2241         }
2242 }
2243
2244 /*
2245  * i40e_fdir_info_get - get information of Flow Director
2246  * @pf: ethernet device to get info from
2247  * @fdir: a pointer to a structure of type *rte_eth_fdir_info* to be filled with
2248  *    the flow director information.
2249  */
2250 void
2251 i40e_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir)
2252 {
2253         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2254         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2255         uint16_t num_flex_set = 0;
2256         uint16_t num_flex_mask = 0;
2257         uint16_t i;
2258
2259         if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT)
2260                 fdir->mode = RTE_FDIR_MODE_PERFECT;
2261         else
2262                 fdir->mode = RTE_FDIR_MODE_NONE;
2263
2264         fdir->guarant_spc =
2265                 (uint32_t)hw->func_caps.fd_filters_guaranteed;
2266         fdir->best_spc =
2267                 (uint32_t)hw->func_caps.fd_filters_best_effort;
2268         fdir->max_flexpayload = I40E_FDIR_MAX_FLEX_LEN;
2269         fdir->flow_types_mask[0] = I40E_FDIR_FLOWS;
2270         for (i = 1; i < RTE_FLOW_MASK_ARRAY_SIZE; i++)
2271                 fdir->flow_types_mask[i] = 0ULL;
2272         fdir->flex_payload_unit = sizeof(uint16_t);
2273         fdir->flex_bitmask_unit = sizeof(uint16_t);
2274         fdir->max_flex_payload_segment_num = I40E_MAX_FLXPLD_FIED;
2275         fdir->flex_payload_limit = I40E_MAX_FLX_SOURCE_OFF;
2276         fdir->max_flex_bitmask_num = I40E_FDIR_BITMASK_NUM_WORD;
2277
2278         i40e_fdir_info_get_flex_set(pf,
2279                                 fdir->flex_conf.flex_set,
2280                                 &num_flex_set);
2281         i40e_fdir_info_get_flex_mask(pf,
2282                                 fdir->flex_conf.flex_mask,
2283                                 &num_flex_mask);
2284
2285         fdir->flex_conf.nb_payloads = num_flex_set;
2286         fdir->flex_conf.nb_flexmasks = num_flex_mask;
2287 }
2288
2289 /*
2290  * i40e_fdir_stat_get - get statistics of Flow Director
2291  * @pf: ethernet device to get info from
2292  * @stat: a pointer to a structure of type *rte_eth_fdir_stats* to be filled with
2293  *    the flow director statistics.
2294  */
2295 void
2296 i40e_fdir_stats_get(struct rte_eth_dev *dev, struct rte_eth_fdir_stats *stat)
2297 {
2298         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2299         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2300         uint32_t fdstat;
2301
2302         fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT);
2303         stat->guarant_cnt =
2304                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2305                             I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2306         stat->best_cnt =
2307                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2308                             I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2309 }
2310
2311 static int
2312 i40e_fdir_filter_set(struct rte_eth_dev *dev,
2313                      struct rte_eth_fdir_filter_info *info)
2314 {
2315         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2316         int ret = 0;
2317
2318         if (!info) {
2319                 PMD_DRV_LOG(ERR, "Invalid pointer");
2320                 return -EFAULT;
2321         }
2322
2323         switch (info->info_type) {
2324         case RTE_ETH_FDIR_FILTER_INPUT_SET_SELECT:
2325                 ret = i40e_fdir_filter_inset_select(pf,
2326                                 &(info->info.input_set_conf));
2327                 break;
2328         default:
2329                 PMD_DRV_LOG(ERR, "FD filter info type (%d) not supported",
2330                             info->info_type);
2331                 return -EINVAL;
2332         }
2333
2334         return ret;
2335 }
2336
2337 /*
2338  * i40e_fdir_ctrl_func - deal with all operations on flow director.
2339  * @pf: board private structure
2340  * @filter_op:operation will be taken.
2341  * @arg: a pointer to specific structure corresponding to the filter_op
2342  */
2343 int
2344 i40e_fdir_ctrl_func(struct rte_eth_dev *dev,
2345                        enum rte_filter_op filter_op,
2346                        void *arg)
2347 {
2348         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2349         int ret = 0;
2350
2351         if ((pf->flags & I40E_FLAG_FDIR) == 0)
2352                 return -ENOTSUP;
2353
2354         if (filter_op == RTE_ETH_FILTER_NOP)
2355                 return 0;
2356
2357         if (arg == NULL && filter_op != RTE_ETH_FILTER_FLUSH)
2358                 return -EINVAL;
2359
2360         switch (filter_op) {
2361         case RTE_ETH_FILTER_ADD:
2362                 ret = i40e_add_del_fdir_filter(dev,
2363                         (struct rte_eth_fdir_filter *)arg,
2364                         TRUE);
2365                 break;
2366         case RTE_ETH_FILTER_DELETE:
2367                 ret = i40e_add_del_fdir_filter(dev,
2368                         (struct rte_eth_fdir_filter *)arg,
2369                         FALSE);
2370                 break;
2371         case RTE_ETH_FILTER_FLUSH:
2372                 ret = i40e_fdir_flush(dev);
2373                 break;
2374         case RTE_ETH_FILTER_INFO:
2375                 i40e_fdir_info_get(dev, (struct rte_eth_fdir_info *)arg);
2376                 break;
2377         case RTE_ETH_FILTER_SET:
2378                 ret = i40e_fdir_filter_set(dev,
2379                         (struct rte_eth_fdir_filter_info *)arg);
2380                 break;
2381         case RTE_ETH_FILTER_STATS:
2382                 i40e_fdir_stats_get(dev, (struct rte_eth_fdir_stats *)arg);
2383                 break;
2384         default:
2385                 PMD_DRV_LOG(ERR, "unknown operation %u.", filter_op);
2386                 ret = -EINVAL;
2387                 break;
2388         }
2389         return ret;
2390 }
2391
2392 /* Restore flow director filter */
2393 void
2394 i40e_fdir_filter_restore(struct i40e_pf *pf)
2395 {
2396         struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(pf->main_vsi);
2397         struct i40e_fdir_filter_list *fdir_list = &pf->fdir.fdir_list;
2398         struct i40e_fdir_filter *f;
2399         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2400         uint32_t fdstat;
2401         uint32_t guarant_cnt;  /**< Number of filters in guaranteed spaces. */
2402         uint32_t best_cnt;     /**< Number of filters in best effort spaces. */
2403
2404         TAILQ_FOREACH(f, fdir_list, rules)
2405                 i40e_flow_add_del_fdir_filter(dev, &f->fdir, TRUE);
2406
2407         fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT);
2408         guarant_cnt =
2409                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2410                            I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2411         best_cnt =
2412                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2413                            I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2414
2415         PMD_DRV_LOG(INFO, "FDIR: Guarant count: %d,  Best count: %d",
2416                     guarant_cnt, best_cnt);
2417 }