drivers/net/i40e/i40e_fdir.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_ether.h>
  14 #include <rte_ethdev_driver.h>
  15 #include <rte_log.h>
  16 #include <rte_memzone.h>
  17 #include <rte_malloc.h>
  18 #include <rte_arp.h>
  19 #include <rte_ip.h>
  20 #include <rte_udp.h>
  21 #include <rte_tcp.h>
  22 #include <rte_sctp.h>
  23 #include <rte_hash_crc.h>
  24 #include <rte_bitmap.h>
  25
  26 #include "i40e_logs.h"
  27 #include "base/i40e_type.h"
  28 #include "base/i40e_prototype.h"
  29 #include "i40e_ethdev.h"
  30 #include "i40e_rxtx.h"
  31
  32 #define I40E_FDIR_MZ_NAME          "FDIR_MEMZONE"
  33 #ifndef IPV6_ADDR_LEN
  34 #define IPV6_ADDR_LEN              16
  35 #endif
  36
  37 #ifndef IPPROTO_L2TP
  38 #define IPPROTO_L2TP              115
  39 #endif
  40
  41 #define I40E_FDIR_PKT_LEN                   512
  42 #define I40E_FDIR_IP_DEFAULT_LEN            420
  43 #define I40E_FDIR_IP_DEFAULT_TTL            0x40
  44 #define I40E_FDIR_IP_DEFAULT_VERSION_IHL    0x45
  45 #define I40E_FDIR_TCP_DEFAULT_DATAOFF       0x50
  46 #define I40E_FDIR_IPv6_DEFAULT_VTC_FLOW     0x60000000
  47
  48 #define I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS   0xFF
  49 #define I40E_FDIR_IPv6_PAYLOAD_LEN          380
  50 #define I40E_FDIR_UDP_DEFAULT_LEN           400
  51 #define I40E_FDIR_GTP_DEFAULT_LEN           384
  52 #define I40E_FDIR_INNER_IP_DEFAULT_LEN      384
  53 #define I40E_FDIR_INNER_IPV6_DEFAULT_LEN    344
  54
  55 #define I40E_FDIR_GTPC_DST_PORT             2123
  56 #define I40E_FDIR_GTPU_DST_PORT             2152
  57 #define I40E_FDIR_GTP_VER_FLAG_0X30         0x30
  58 #define I40E_FDIR_GTP_VER_FLAG_0X32         0x32
  59 #define I40E_FDIR_GTP_MSG_TYPE_0X01         0x01
  60 #define I40E_FDIR_GTP_MSG_TYPE_0XFF         0xFF
  61
  62 #define I40E_FDIR_ESP_DST_PORT              4500
  63
  64 /* Wait time for fdir filter programming */
  65 #define I40E_FDIR_MAX_WAIT_US 10000
  66
  67 /* Wait count and interval for fdir filter flush */
  68 #define I40E_FDIR_FLUSH_RETRY       50
  69 #define I40E_FDIR_FLUSH_INTERVAL_MS 5
  70
  71 #define I40E_COUNTER_PF           2
  72 /* Statistic counter index for one pf */
  73 #define I40E_COUNTER_INDEX_FDIR(pf_id)   (0 + (pf_id) * I40E_COUNTER_PF)
  74
  75 #define I40E_FDIR_FLOWS ( \
  76         (1ULL << RTE_ETH_FLOW_FRAG_IPV4) | \
  77         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \
  78         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \
  79         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \
  80         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \
  81         (1ULL << RTE_ETH_FLOW_FRAG_IPV6) | \
  82         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \
  83         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \
  84         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \
  85         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \
  86         (1ULL << RTE_ETH_FLOW_L2_PAYLOAD))
  87
  88 static int i40e_fdir_filter_programming(struct i40e_pf *pf,
  89                         enum i40e_filter_pctype pctype,
  90                         const struct rte_eth_fdir_filter *filter,
  91                         bool add);
  92 static int i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input,
  93                          struct i40e_fdir_filter *filter);
  94 static struct i40e_fdir_filter *
  95 i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info,
  96                         const struct i40e_fdir_input *input);
  97 static int i40e_sw_fdir_filter_insert(struct i40e_pf *pf,
  98                                    struct i40e_fdir_filter *filter);
  99 static int
 100 i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
 101                                   enum i40e_filter_pctype pctype,
 102                                   const struct i40e_fdir_filter_conf *filter,
 103                                   bool add);
 104
 105 static int
 106 i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq)
 107 {
 108         struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
 109         struct i40e_hmc_obj_rxq rx_ctx;
 110         int err = I40E_SUCCESS;
 111
 112         memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq));
 113         /* Init the RX queue in hardware */
 114         rx_ctx.dbuff = I40E_RXBUF_SZ_1024 >> I40E_RXQ_CTX_DBUFF_SHIFT;
 115         rx_ctx.hbuff = 0;
 116         rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
 117         rx_ctx.qlen = rxq->nb_rx_desc;
 118 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
 119         rx_ctx.dsize = 1;
 120 #endif
 121         rx_ctx.dtype = i40e_header_split_none;
 122         rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE;
 123         rx_ctx.rxmax = RTE_ETHER_MAX_LEN;
 124         rx_ctx.tphrdesc_ena = 1;
 125         rx_ctx.tphwdesc_ena = 1;
 126         rx_ctx.tphdata_ena = 1;
 127         rx_ctx.tphhead_ena = 1;
 128         rx_ctx.lrxqthresh = 2;
 129         rx_ctx.crcstrip = 0;
 130         rx_ctx.l2tsel = 1;
 131         rx_ctx.showiv = 0;
 132         rx_ctx.prefena = 1;
 133
 134         err = i40e_clear_lan_rx_queue_context(hw, rxq->reg_idx);
 135         if (err != I40E_SUCCESS) {
 136                 PMD_DRV_LOG(ERR, "Failed to clear FDIR RX queue context.");
 137                 return err;
 138         }
 139         err = i40e_set_lan_rx_queue_context(hw, rxq->reg_idx, &rx_ctx);
 140         if (err != I40E_SUCCESS) {
 141                 PMD_DRV_LOG(ERR, "Failed to set FDIR RX queue context.");
 142                 return err;
 143         }
 144         rxq->qrx_tail = hw->hw_addr +
 145                 I40E_QRX_TAIL(rxq->vsi->base_queue);
 146
 147         rte_wmb();
 148         /* Init the RX tail regieter. */
 149         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 150
 151         return err;
 152 }
 153
 154 /*
 155  * i40e_fdir_setup - reserve and initialize the Flow Director resources
 156  * @pf: board private structure
 157  */
 158 int
 159 i40e_fdir_setup(struct i40e_pf *pf)
 160 {
 161         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 162         struct i40e_vsi *vsi;
 163         int err = I40E_SUCCESS;
 164         char z_name[RTE_MEMZONE_NAMESIZE];
 165         const struct rte_memzone *mz = NULL;
 166         struct rte_eth_dev *eth_dev = pf->adapter->eth_dev;
 167
 168         if ((pf->flags & I40E_FLAG_FDIR) == 0) {
 169                 PMD_INIT_LOG(ERR, "HW doesn't support FDIR");
 170                 return I40E_NOT_SUPPORTED;
 171         }
 172
 173         PMD_DRV_LOG(INFO, "FDIR HW Capabilities: num_filters_guaranteed = %u,"
 174                         " num_filters_best_effort = %u.",
 175                         hw->func_caps.fd_filters_guaranteed,
 176                         hw->func_caps.fd_filters_best_effort);
 177
 178         vsi = pf->fdir.fdir_vsi;
 179         if (vsi) {
 180                 PMD_DRV_LOG(INFO, "FDIR initialization has been done.");
 181                 return I40E_SUCCESS;
 182         }
 183         /* make new FDIR VSI */
 184         vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->main_vsi, 0);
 185         if (!vsi) {
 186                 PMD_DRV_LOG(ERR, "Couldn't create FDIR VSI.");
 187                 return I40E_ERR_NO_AVAILABLE_VSI;
 188         }
 189         pf->fdir.fdir_vsi = vsi;
 190
 191         /*Fdir tx queue setup*/
 192         err = i40e_fdir_setup_tx_resources(pf);
 193         if (err) {
 194                 PMD_DRV_LOG(ERR, "Failed to setup FDIR TX resources.");
 195                 goto fail_setup_tx;
 196         }
 197
 198         /*Fdir rx queue setup*/
 199         err = i40e_fdir_setup_rx_resources(pf);
 200         if (err) {
 201                 PMD_DRV_LOG(ERR, "Failed to setup FDIR RX resources.");
 202                 goto fail_setup_rx;
 203         }
 204
 205         err = i40e_tx_queue_init(pf->fdir.txq);
 206         if (err) {
 207                 PMD_DRV_LOG(ERR, "Failed to do FDIR TX initialization.");
 208                 goto fail_mem;
 209         }
 210
 211         /* need switch on before dev start*/
 212         err = i40e_switch_tx_queue(hw, vsi->base_queue, TRUE);
 213         if (err) {
 214                 PMD_DRV_LOG(ERR, "Failed to do fdir TX switch on.");
 215                 goto fail_mem;
 216         }
 217
 218         /* Init the rx queue in hardware */
 219         err = i40e_fdir_rx_queue_init(pf->fdir.rxq);
 220         if (err) {
 221                 PMD_DRV_LOG(ERR, "Failed to do FDIR RX initialization.");
 222                 goto fail_mem;
 223         }
 224
 225         /* switch on rx queue */
 226         err = i40e_switch_rx_queue(hw, vsi->base_queue, TRUE);
 227         if (err) {
 228                 PMD_DRV_LOG(ERR, "Failed to do FDIR RX switch on.");
 229                 goto fail_mem;
 230         }
 231
 232         /* reserve memory for the fdir programming packet */
 233         snprintf(z_name, sizeof(z_name), "%s_%s_%d",
 234                         eth_dev->device->driver->name,
 235                         I40E_FDIR_MZ_NAME,
 236                         eth_dev->data->port_id);
 237         mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN, SOCKET_ID_ANY);
 238         if (!mz) {
 239                 PMD_DRV_LOG(ERR, "Cannot init memzone for "
 240                                  "flow director program packet.");
 241                 err = I40E_ERR_NO_MEMORY;
 242                 goto fail_mem;
 243         }
 244         pf->fdir.prg_pkt = mz->addr;
 245         pf->fdir.dma_addr = mz->iova;
 246
 247         pf->fdir.match_counter_index = I40E_COUNTER_INDEX_FDIR(hw->pf_id);
 248         pf->fdir.fdir_actual_cnt = 0;
 249         pf->fdir.fdir_guarantee_free_space =
 250                 pf->fdir.fdir_guarantee_total_space;
 251
 252         PMD_DRV_LOG(INFO, "FDIR setup successfully, with programming queue %u.",
 253                     vsi->base_queue);
 254         return I40E_SUCCESS;
 255
 256 fail_mem:
 257         i40e_dev_rx_queue_release(pf->fdir.rxq);
 258         pf->fdir.rxq = NULL;
 259 fail_setup_rx:
 260         i40e_dev_tx_queue_release(pf->fdir.txq);
 261         pf->fdir.txq = NULL;
 262 fail_setup_tx:
 263         i40e_vsi_release(vsi);
 264         pf->fdir.fdir_vsi = NULL;
 265         return err;
 266 }
 267
 268 /*
 269  * i40e_fdir_teardown - release the Flow Director resources
 270  * @pf: board private structure
 271  */
 272 void
 273 i40e_fdir_teardown(struct i40e_pf *pf)
 274 {
 275         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 276         struct i40e_vsi *vsi;
 277         struct rte_eth_dev *dev = pf->adapter->eth_dev;
 278
 279         vsi = pf->fdir.fdir_vsi;
 280         if (!vsi)
 281                 return;
 282         int err = i40e_switch_tx_queue(hw, vsi->base_queue, FALSE);
 283         if (err)
 284                 PMD_DRV_LOG(DEBUG, "Failed to do FDIR TX switch off");
 285         err = i40e_switch_rx_queue(hw, vsi->base_queue, FALSE);
 286         if (err)
 287                 PMD_DRV_LOG(DEBUG, "Failed to do FDIR RX switch off");
 288         i40e_dev_rx_queue_release(pf->fdir.rxq);
 289         rte_eth_dma_zone_free(dev, "fdir_rx_ring", pf->fdir.rxq->queue_id);
 290         pf->fdir.rxq = NULL;
 291         i40e_dev_tx_queue_release(pf->fdir.txq);
 292         rte_eth_dma_zone_free(dev, "fdir_tx_ring", pf->fdir.txq->queue_id);
 293         pf->fdir.txq = NULL;
 294         i40e_vsi_release(vsi);
 295         pf->fdir.fdir_vsi = NULL;
 296 }
 297
 298 /* check whether the flow director table in empty */
 299 static inline int
 300 i40e_fdir_empty(struct i40e_hw *hw)
 301 {
 302         uint32_t guarant_cnt, best_cnt;
 303
 304         guarant_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
 305                                  I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
 306                                  I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
 307         best_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
 308                               I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
 309                               I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
 310         if (best_cnt + guarant_cnt > 0)
 311                 return -1;
 312
 313         return 0;
 314 }
 315
 316 /*
 317  * Initialize the configuration about bytes stream extracted as flexible payload
 318  * and mask setting
 319  */
 320 static inline void
 321 i40e_init_flx_pld(struct i40e_pf *pf)
 322 {
 323         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 324         uint8_t pctype;
 325         int i, index;
 326         uint16_t flow_type;
 327
 328         /*
 329          * Define the bytes stream extracted as flexible payload in
 330          * field vector. By default, select 8 words from the beginning
 331          * of payload as flexible payload.
 332          */
 333         for (i = I40E_FLXPLD_L2_IDX; i < I40E_MAX_FLXPLD_LAYER; i++) {
 334                 index = i * I40E_MAX_FLXPLD_FIED;
 335                 pf->fdir.flex_set[index].src_offset = 0;
 336                 pf->fdir.flex_set[index].size = I40E_FDIR_MAX_FLEXWORD_NUM;
 337                 pf->fdir.flex_set[index].dst_offset = 0;
 338                 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(index), 0x0000C900);
 339                 I40E_WRITE_REG(hw,
 340                         I40E_PRTQF_FLX_PIT(index + 1), 0x0000FC29);/*non-used*/
 341                 I40E_WRITE_REG(hw,
 342                         I40E_PRTQF_FLX_PIT(index + 2), 0x0000FC2A);/*non-used*/
 343         }
 344
 345         /* initialize the masks */
 346         for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
 347              pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) {
 348                 flow_type = i40e_pctype_to_flowtype(pf->adapter, pctype);
 349
 350                 if (flow_type == RTE_ETH_FLOW_UNKNOWN)
 351                         continue;
 352                 pf->fdir.flex_mask[pctype].word_mask = 0;
 353                 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), 0);
 354                 for (i = 0; i < I40E_FDIR_BITMASK_NUM_WORD; i++) {
 355                         pf->fdir.flex_mask[pctype].bitmask[i].offset = 0;
 356                         pf->fdir.flex_mask[pctype].bitmask[i].mask = 0;
 357                         i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), 0);
 358                 }
 359         }
 360 }
 361
 362 #define I40E_VALIDATE_FLEX_PIT(flex_pit1, flex_pit2) do { \
 363         if ((flex_pit2).src_offset < \
 364                 (flex_pit1).src_offset + (flex_pit1).size) { \
 365                 PMD_DRV_LOG(ERR, "src_offset should be not" \
 366                         " less than than previous offset" \
 367                         " + previous FSIZE."); \
 368                 return -EINVAL; \
 369         } \
 370 } while (0)
 371
 372 /*
 373  * i40e_srcoff_to_flx_pit - transform the src_offset into flex_pit structure,
 374  * and the flex_pit will be sorted by it's src_offset value
 375  */
 376 static inline uint16_t
 377 i40e_srcoff_to_flx_pit(const uint16_t *src_offset,
 378                         struct i40e_fdir_flex_pit *flex_pit)
 379 {
 380         uint16_t src_tmp, size, num = 0;
 381         uint16_t i, k, j = 0;
 382
 383         while (j < I40E_FDIR_MAX_FLEX_LEN) {
 384                 size = 1;
 385                 for (; j < I40E_FDIR_MAX_FLEX_LEN - 1; j++) {
 386                         if (src_offset[j + 1] == src_offset[j] + 1)
 387                                 size++;
 388                         else
 389                                 break;
 390                 }
 391                 src_tmp = src_offset[j] + 1 - size;
 392                 /* the flex_pit need to be sort by src_offset */
 393                 for (i = 0; i < num; i++) {
 394                         if (src_tmp < flex_pit[i].src_offset)
 395                                 break;
 396                 }
 397                 /* if insert required, move backward */
 398                 for (k = num; k > i; k--)
 399                         flex_pit[k] = flex_pit[k - 1];
 400                 /* insert */
 401                 flex_pit[i].dst_offset = j + 1 - size;
 402                 flex_pit[i].src_offset = src_tmp;
 403                 flex_pit[i].size = size;
 404                 j++;
 405                 num++;
 406         }
 407         return num;
 408 }
 409
 410 /* i40e_check_fdir_flex_payload -check flex payload configuration arguments */
 411 static inline int
 412 i40e_check_fdir_flex_payload(const struct rte_eth_flex_payload_cfg *flex_cfg)
 413 {
 414         struct i40e_fdir_flex_pit flex_pit[I40E_FDIR_MAX_FLEX_LEN];
 415         uint16_t num, i;
 416
 417         for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i++) {
 418                 if (flex_cfg->src_offset[i] >= I40E_MAX_FLX_SOURCE_OFF) {
 419                         PMD_DRV_LOG(ERR, "exceeds maxmial payload limit.");
 420                         return -EINVAL;
 421                 }
 422         }
 423
 424         memset(flex_pit, 0, sizeof(flex_pit));
 425         num = i40e_srcoff_to_flx_pit(flex_cfg->src_offset, flex_pit);
 426         if (num > I40E_MAX_FLXPLD_FIED) {
 427                 PMD_DRV_LOG(ERR, "exceeds maxmial number of flex fields.");
 428                 return -EINVAL;
 429         }
 430         for (i = 0; i < num; i++) {
 431                 if (flex_pit[i].size & 0x01 || flex_pit[i].dst_offset & 0x01 ||
 432                         flex_pit[i].src_offset & 0x01) {
 433                         PMD_DRV_LOG(ERR, "flexpayload should be measured"
 434                                 " in word");
 435                         return -EINVAL;
 436                 }
 437                 if (i != num - 1)
 438                         I40E_VALIDATE_FLEX_PIT(flex_pit[i], flex_pit[i + 1]);
 439         }
 440         return 0;
 441 }
 442
 443 /*
 444  * i40e_check_fdir_flex_conf -check if the flex payload and mask configuration
 445  * arguments are valid
 446  */
 447 static int
 448 i40e_check_fdir_flex_conf(const struct i40e_adapter *adapter,
 449                           const struct rte_eth_fdir_flex_conf *conf)
 450 {
 451         const struct rte_eth_flex_payload_cfg *flex_cfg;
 452         const struct rte_eth_fdir_flex_mask *flex_mask;
 453         uint16_t mask_tmp;
 454         uint8_t nb_bitmask;
 455         uint16_t i, j;
 456         int ret = 0;
 457         enum i40e_filter_pctype pctype;
 458
 459         if (conf == NULL) {
 460                 PMD_DRV_LOG(INFO, "NULL pointer.");
 461                 return -EINVAL;
 462         }
 463         /* check flexible payload setting configuration */
 464         if (conf->nb_payloads > RTE_ETH_L4_PAYLOAD) {
 465                 PMD_DRV_LOG(ERR, "invalid number of payload setting.");
 466                 return -EINVAL;
 467         }
 468         for (i = 0; i < conf->nb_payloads; i++) {
 469                 flex_cfg = &conf->flex_set[i];
 470                 if (flex_cfg->type > RTE_ETH_L4_PAYLOAD) {
 471                         PMD_DRV_LOG(ERR, "invalid payload type.");
 472                         return -EINVAL;
 473                 }
 474                 ret = i40e_check_fdir_flex_payload(flex_cfg);
 475                 if (ret < 0) {
 476                         PMD_DRV_LOG(ERR, "invalid flex payload arguments.");
 477                         return -EINVAL;
 478                 }
 479         }
 480
 481         /* check flex mask setting configuration */
 482         if (conf->nb_flexmasks >= RTE_ETH_FLOW_MAX) {
 483                 PMD_DRV_LOG(ERR, "invalid number of flex masks.");
 484                 return -EINVAL;
 485         }
 486         for (i = 0; i < conf->nb_flexmasks; i++) {
 487                 flex_mask = &conf->flex_mask[i];
 488                 pctype = i40e_flowtype_to_pctype(adapter, flex_mask->flow_type);
 489                 if (pctype == I40E_FILTER_PCTYPE_INVALID) {
 490                         PMD_DRV_LOG(WARNING, "invalid flow type.");
 491                         return -EINVAL;
 492                 }
 493                 nb_bitmask = 0;
 494                 for (j = 0; j < I40E_FDIR_MAX_FLEX_LEN; j += sizeof(uint16_t)) {
 495                         mask_tmp = I40E_WORD(flex_mask->mask[j],
 496                                              flex_mask->mask[j + 1]);
 497                         if (mask_tmp != 0x0 && mask_tmp != UINT16_MAX) {
 498                                 nb_bitmask++;
 499                                 if (nb_bitmask > I40E_FDIR_BITMASK_NUM_WORD) {
 500                                         PMD_DRV_LOG(ERR, " exceed maximal"
 501                                                 " number of bitmasks.");
 502                                         return -EINVAL;
 503                                 }
 504                         }
 505                 }
 506         }
 507         return 0;
 508 }
 509
 510 /*
 511  * i40e_set_flx_pld_cfg -configure the rule how bytes stream is extracted as flexible payload
 512  * @pf: board private structure
 513  * @cfg: the rule how bytes stream is extracted as flexible payload
 514  */
 515 static void
 516 i40e_set_flx_pld_cfg(struct i40e_pf *pf,
 517                          const struct rte_eth_flex_payload_cfg *cfg)
 518 {
 519         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 520         struct i40e_fdir_flex_pit flex_pit[I40E_MAX_FLXPLD_FIED];
 521         uint32_t flx_pit, flx_ort;
 522         uint16_t num, min_next_off;  /* in words */
 523         uint8_t field_idx = 0;
 524         uint8_t layer_idx = 0;
 525         uint16_t i;
 526
 527         if (cfg->type == RTE_ETH_L2_PAYLOAD)
 528                 layer_idx = I40E_FLXPLD_L2_IDX;
 529         else if (cfg->type == RTE_ETH_L3_PAYLOAD)
 530                 layer_idx = I40E_FLXPLD_L3_IDX;
 531         else if (cfg->type == RTE_ETH_L4_PAYLOAD)
 532                 layer_idx = I40E_FLXPLD_L4_IDX;
 533
 534         memset(flex_pit, 0, sizeof(flex_pit));
 535         num = RTE_MIN(i40e_srcoff_to_flx_pit(cfg->src_offset, flex_pit),
 536                       RTE_DIM(flex_pit));
 537
 538         if (num) {
 539                 flx_ort = (1 << I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT) |
 540                           (num << I40E_GLQF_ORT_FIELD_CNT_SHIFT) |
 541                           (layer_idx * I40E_MAX_FLXPLD_FIED);
 542                 I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33 + layer_idx), flx_ort);
 543         }
 544
 545         for (i = 0; i < num; i++) {
 546                 field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i;
 547                 /* record the info in fdir structure */
 548                 pf->fdir.flex_set[field_idx].src_offset =
 549                         flex_pit[i].src_offset / sizeof(uint16_t);
 550                 pf->fdir.flex_set[field_idx].size =
 551                         flex_pit[i].size / sizeof(uint16_t);
 552                 pf->fdir.flex_set[field_idx].dst_offset =
 553                         flex_pit[i].dst_offset / sizeof(uint16_t);
 554                 flx_pit = MK_FLX_PIT(pf->fdir.flex_set[field_idx].src_offset,
 555                                 pf->fdir.flex_set[field_idx].size,
 556                                 pf->fdir.flex_set[field_idx].dst_offset);
 557
 558                 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit);
 559         }
 560         min_next_off = pf->fdir.flex_set[field_idx].src_offset +
 561                                 pf->fdir.flex_set[field_idx].size;
 562
 563         for (; i < I40E_MAX_FLXPLD_FIED; i++) {
 564                 /* set the non-used register obeying register's constrain */
 565                 flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE,
 566                            NONUSE_FLX_PIT_DEST_OFF);
 567                 I40E_WRITE_REG(hw,
 568                         I40E_PRTQF_FLX_PIT(layer_idx * I40E_MAX_FLXPLD_FIED + i),
 569                         flx_pit);
 570                 min_next_off++;
 571         }
 572 }
 573
 574 /*
 575  * i40e_set_flex_mask_on_pctype - configure the mask on flexible payload
 576  * @pf: board private structure
 577  * @pctype: packet classify type
 578  * @flex_masks: mask for flexible payload
 579  */
 580 static void
 581 i40e_set_flex_mask_on_pctype(struct i40e_pf *pf,
 582                 enum i40e_filter_pctype pctype,
 583                 const struct rte_eth_fdir_flex_mask *mask_cfg)
 584 {
 585         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 586         struct i40e_fdir_flex_mask *flex_mask;
 587         uint32_t flxinset, fd_mask;
 588         uint16_t mask_tmp;
 589         uint8_t i, nb_bitmask = 0;
 590
 591         flex_mask = &pf->fdir.flex_mask[pctype];
 592         memset(flex_mask, 0, sizeof(struct i40e_fdir_flex_mask));
 593         for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i += sizeof(uint16_t)) {
 594                 mask_tmp = I40E_WORD(mask_cfg->mask[i], mask_cfg->mask[i + 1]);
 595                 if (mask_tmp != 0x0) {
 596                         flex_mask->word_mask |=
 597                                 I40E_FLEX_WORD_MASK(i / sizeof(uint16_t));
 598                         if (mask_tmp != UINT16_MAX) {
 599                                 /* set bit mask */
 600                                 flex_mask->bitmask[nb_bitmask].mask = ~mask_tmp;
 601                                 flex_mask->bitmask[nb_bitmask].offset =
 602                                         i / sizeof(uint16_t);
 603                                 nb_bitmask++;
 604                         }
 605                 }
 606         }
 607         /* write mask to hw */
 608         flxinset = (flex_mask->word_mask <<
 609                 I40E_PRTQF_FD_FLXINSET_INSET_SHIFT) &
 610                 I40E_PRTQF_FD_FLXINSET_INSET_MASK;
 611         i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), flxinset);
 612
 613         for (i = 0; i < nb_bitmask; i++) {
 614                 fd_mask = (flex_mask->bitmask[i].mask <<
 615                         I40E_PRTQF_FD_MSK_MASK_SHIFT) &
 616                         I40E_PRTQF_FD_MSK_MASK_MASK;
 617                 fd_mask |= ((flex_mask->bitmask[i].offset +
 618                         I40E_FLX_OFFSET_IN_FIELD_VECTOR) <<
 619                         I40E_PRTQF_FD_MSK_OFFSET_SHIFT) &
 620                         I40E_PRTQF_FD_MSK_OFFSET_MASK;
 621                 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), fd_mask);
 622         }
 623 }
 624
 625 /*
 626  * Enable/disable flow director RX processing in vector routines.
 627  */
 628 void
 629 i40e_fdir_rx_proc_enable(struct rte_eth_dev *dev, bool on)
 630 {
 631         int32_t i;
 632
 633         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 634                 struct i40e_rx_queue *rxq = dev->data->rx_queues[i];
 635                 if (!rxq)
 636                         continue;
 637                 rxq->fdir_enabled = on;
 638         }
 639         PMD_DRV_LOG(DEBUG, "Flow Director processing on RX set to %d", on);
 640 }
 641
 642 /*
 643  * Configure flow director related setting
 644  */
 645 int
 646 i40e_fdir_configure(struct rte_eth_dev *dev)
 647 {
 648         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 649         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 650         struct rte_eth_fdir_flex_conf *conf;
 651         enum i40e_filter_pctype pctype;
 652         uint32_t val;
 653         uint8_t i;
 654         int ret = 0;
 655
 656         /*
 657         * configuration need to be done before
 658         * flow director filters are added
 659         * If filters exist, flush them.
 660         */
 661         if (i40e_fdir_empty(hw) < 0) {
 662                 ret = i40e_fdir_flush(dev);
 663                 if (ret) {
 664                         PMD_DRV_LOG(ERR, "failed to flush fdir table.");
 665                         return ret;
 666                 }
 667         }
 668
 669         /* enable FDIR filter */
 670         val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
 671         val |= I40E_PFQF_CTL_0_FD_ENA_MASK;
 672         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, val);
 673
 674         i40e_init_flx_pld(pf); /* set flex config to default value */
 675
 676         conf = &dev->data->dev_conf.fdir_conf.flex_conf;
 677         ret = i40e_check_fdir_flex_conf(pf->adapter, conf);
 678         if (ret < 0) {
 679                 PMD_DRV_LOG(ERR, " invalid configuration arguments.");
 680                 return -EINVAL;
 681         }
 682
 683         if (!pf->support_multi_driver) {
 684                 /* configure flex payload */
 685                 for (i = 0; i < conf->nb_payloads; i++)
 686                         i40e_set_flx_pld_cfg(pf, &conf->flex_set[i]);
 687                 /* configure flex mask*/
 688                 for (i = 0; i < conf->nb_flexmasks; i++) {
 689                         if (hw->mac.type == I40E_MAC_X722) {
 690                                 /* get pctype value in fd pctype register */
 691                                 pctype = (enum i40e_filter_pctype)
 692                                           i40e_read_rx_ctl(hw,
 693                                                 I40E_GLQF_FD_PCTYPES(
 694                                                 (int)i40e_flowtype_to_pctype(
 695                                                 pf->adapter,
 696                                                 conf->flex_mask[i].flow_type)));
 697                         } else {
 698                                 pctype = i40e_flowtype_to_pctype(pf->adapter,
 699                                                   conf->flex_mask[i].flow_type);
 700                         }
 701
 702                         i40e_set_flex_mask_on_pctype(pf, pctype,
 703                                                      &conf->flex_mask[i]);
 704                 }
 705         } else {
 706                 PMD_DRV_LOG(ERR, "Not support flexible payload.");
 707         }
 708
 709         /* Enable FDIR processing in RX routines */
 710         i40e_fdir_rx_proc_enable(dev, 1);
 711
 712         return ret;
 713 }
 714
 715 static inline int
 716 i40e_fdir_fill_eth_ip_head(const struct rte_eth_fdir_input *fdir_input,
 717                            unsigned char *raw_pkt,
 718                            bool vlan)
 719 {
 720         static uint8_t vlan_frame[] = {0x81, 0, 0, 0};
 721         uint16_t *ether_type;
 722         uint8_t len = 2 * sizeof(struct rte_ether_addr);
 723         struct rte_ipv4_hdr *ip;
 724         struct rte_ipv6_hdr *ip6;
 725         static const uint8_t next_proto[] = {
 726                 [RTE_ETH_FLOW_FRAG_IPV4] = IPPROTO_IP,
 727                 [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = IPPROTO_TCP,
 728                 [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = IPPROTO_UDP,
 729                 [RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] = IPPROTO_SCTP,
 730                 [RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] = IPPROTO_IP,
 731                 [RTE_ETH_FLOW_FRAG_IPV6] = IPPROTO_NONE,
 732                 [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = IPPROTO_TCP,
 733                 [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = IPPROTO_UDP,
 734                 [RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] = IPPROTO_SCTP,
 735                 [RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] = IPPROTO_NONE,
 736         };
 737
 738         raw_pkt += 2 * sizeof(struct rte_ether_addr);
 739         if (vlan && fdir_input->flow_ext.vlan_tci) {
 740                 rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame));
 741                 rte_memcpy(raw_pkt + sizeof(uint16_t),
 742                            &fdir_input->flow_ext.vlan_tci,
 743                            sizeof(uint16_t));
 744                 raw_pkt += sizeof(vlan_frame);
 745                 len += sizeof(vlan_frame);
 746         }
 747         ether_type = (uint16_t *)raw_pkt;
 748         raw_pkt += sizeof(uint16_t);
 749         len += sizeof(uint16_t);
 750
 751         switch (fdir_input->flow_type) {
 752         case RTE_ETH_FLOW_L2_PAYLOAD:
 753                 *ether_type = fdir_input->flow.l2_flow.ether_type;
 754                 break;
 755         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 756         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 757         case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP:
 758         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 759         case RTE_ETH_FLOW_FRAG_IPV4:
 760                 ip = (struct rte_ipv4_hdr *)raw_pkt;
 761
 762                 *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
 763                 ip->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL;
 764                 /* set len to by default */
 765                 ip->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN);
 766                 ip->next_proto_id = fdir_input->flow.ip4_flow.proto ?
 767                                         fdir_input->flow.ip4_flow.proto :
 768                                         next_proto[fdir_input->flow_type];
 769                 ip->time_to_live = fdir_input->flow.ip4_flow.ttl ?
 770                                         fdir_input->flow.ip4_flow.ttl :
 771                                         I40E_FDIR_IP_DEFAULT_TTL;
 772                 ip->type_of_service = fdir_input->flow.ip4_flow.tos;
 773                 /*
 774                  * The source and destination fields in the transmitted packet
 775                  * need to be presented in a reversed order with respect
 776                  * to the expected received packets.
 777                  */
 778                 ip->src_addr = fdir_input->flow.ip4_flow.dst_ip;
 779                 ip->dst_addr = fdir_input->flow.ip4_flow.src_ip;
 780                 len += sizeof(struct rte_ipv4_hdr);
 781                 break;
 782         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 783         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 784         case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP:
 785         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 786         case RTE_ETH_FLOW_FRAG_IPV6:
 787                 ip6 = (struct rte_ipv6_hdr *)raw_pkt;
 788
 789                 *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
 790                 ip6->vtc_flow =
 791                         rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
 792                                          (fdir_input->flow.ipv6_flow.tc <<
 793                                           I40E_FDIR_IPv6_TC_OFFSET));
 794                 ip6->payload_len =
 795                         rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
 796                 ip6->proto = fdir_input->flow.ipv6_flow.proto ?
 797                                         fdir_input->flow.ipv6_flow.proto :
 798                                         next_proto[fdir_input->flow_type];
 799                 ip6->hop_limits = fdir_input->flow.ipv6_flow.hop_limits ?
 800                                         fdir_input->flow.ipv6_flow.hop_limits :
 801                                         I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
 802                 /*
 803                  * The source and destination fields in the transmitted packet
 804                  * need to be presented in a reversed order with respect
 805                  * to the expected received packets.
 806                  */
 807                 rte_memcpy(&(ip6->src_addr),
 808                            &(fdir_input->flow.ipv6_flow.dst_ip),
 809                            IPV6_ADDR_LEN);
 810                 rte_memcpy(&(ip6->dst_addr),
 811                            &(fdir_input->flow.ipv6_flow.src_ip),
 812                            IPV6_ADDR_LEN);
 813                 len += sizeof(struct rte_ipv6_hdr);
 814                 break;
 815         default:
 816                 PMD_DRV_LOG(ERR, "unknown flow type %u.",
 817                             fdir_input->flow_type);
 818                 return -1;
 819         }
 820         return len;
 821 }
 822
 823
 824 /*
 825  * i40e_fdir_construct_pkt - construct packet based on fields in input
 826  * @pf: board private structure
 827  * @fdir_input: input set of the flow director entry
 828  * @raw_pkt: a packet to be constructed
 829  */
 830 static int
 831 i40e_fdir_construct_pkt(struct i40e_pf *pf,
 832                              const struct rte_eth_fdir_input *fdir_input,
 833                              unsigned char *raw_pkt)
 834 {
 835         unsigned char *payload, *ptr;
 836         struct rte_udp_hdr *udp;
 837         struct rte_tcp_hdr *tcp;
 838         struct rte_sctp_hdr *sctp;
 839         uint8_t size, dst = 0;
 840         uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/
 841         int len;
 842
 843         /* fill the ethernet and IP head */
 844         len = i40e_fdir_fill_eth_ip_head(fdir_input, raw_pkt,
 845                                          !!fdir_input->flow_ext.vlan_tci);
 846         if (len < 0)
 847                 return -EINVAL;
 848
 849         /* fill the L4 head */
 850         switch (fdir_input->flow_type) {
 851         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 852                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
 853                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
 854                 /*
 855                  * The source and destination fields in the transmitted packet
 856                  * need to be presented in a reversed order with respect
 857                  * to the expected received packets.
 858                  */
 859                 udp->src_port = fdir_input->flow.udp4_flow.dst_port;
 860                 udp->dst_port = fdir_input->flow.udp4_flow.src_port;
 861                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
 862                 break;
 863
 864         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 865                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
 866                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
 867                 /*
 868                  * The source and destination fields in the transmitted packet
 869                  * need to be presented in a reversed order with respect
 870                  * to the expected received packets.
 871                  */
 872                 tcp->src_port = fdir_input->flow.tcp4_flow.dst_port;
 873                 tcp->dst_port = fdir_input->flow.tcp4_flow.src_port;
 874                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
 875                 break;
 876
 877         case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP:
 878                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
 879                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
 880                 /*
 881                  * The source and destination fields in the transmitted packet
 882                  * need to be presented in a reversed order with respect
 883                  * to the expected received packets.
 884                  */
 885                 sctp->src_port = fdir_input->flow.sctp4_flow.dst_port;
 886                 sctp->dst_port = fdir_input->flow.sctp4_flow.src_port;
 887                 sctp->tag = fdir_input->flow.sctp4_flow.verify_tag;
 888                 break;
 889
 890         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 891         case RTE_ETH_FLOW_FRAG_IPV4:
 892                 payload = raw_pkt + len;
 893                 set_idx = I40E_FLXPLD_L3_IDX;
 894                 break;
 895
 896         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 897                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
 898                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
 899                 /*
 900                  * The source and destination fields in the transmitted packet
 901                  * need to be presented in a reversed order with respect
 902                  * to the expected received packets.
 903                  */
 904                 udp->src_port = fdir_input->flow.udp6_flow.dst_port;
 905                 udp->dst_port = fdir_input->flow.udp6_flow.src_port;
 906                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
 907                 break;
 908
 909         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 910                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
 911                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
 912                 /*
 913                  * The source and destination fields in the transmitted packet
 914                  * need to be presented in a reversed order with respect
 915                  * to the expected received packets.
 916                  */
 917                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
 918                 tcp->src_port = fdir_input->flow.udp6_flow.dst_port;
 919                 tcp->dst_port = fdir_input->flow.udp6_flow.src_port;
 920                 break;
 921
 922         case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP:
 923                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
 924                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
 925                 /*
 926                  * The source and destination fields in the transmitted packet
 927                  * need to be presented in a reversed order with respect
 928                  * to the expected received packets.
 929                  */
 930                 sctp->src_port = fdir_input->flow.sctp6_flow.dst_port;
 931                 sctp->dst_port = fdir_input->flow.sctp6_flow.src_port;
 932                 sctp->tag = fdir_input->flow.sctp6_flow.verify_tag;
 933                 break;
 934
 935         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 936         case RTE_ETH_FLOW_FRAG_IPV6:
 937                 payload = raw_pkt + len;
 938                 set_idx = I40E_FLXPLD_L3_IDX;
 939                 break;
 940         case RTE_ETH_FLOW_L2_PAYLOAD:
 941                 payload = raw_pkt + len;
 942                 /*
 943                  * ARP packet is a special case on which the payload
 944                  * starts after the whole ARP header
 945                  */
 946                 if (fdir_input->flow.l2_flow.ether_type ==
 947                                 rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
 948                         payload += sizeof(struct rte_arp_hdr);
 949                 set_idx = I40E_FLXPLD_L2_IDX;
 950                 break;
 951         default:
 952                 PMD_DRV_LOG(ERR, "unknown flow type %u.", fdir_input->flow_type);
 953                 return -EINVAL;
 954         }
 955
 956         /* fill the flexbytes to payload */
 957         for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
 958                 pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i;
 959                 size = pf->fdir.flex_set[pit_idx].size;
 960                 if (size == 0)
 961                         continue;
 962                 dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t);
 963                 ptr = payload +
 964                         pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t);
 965                 rte_memcpy(ptr,
 966                                  &fdir_input->flow_ext.flexbytes[dst],
 967                                  size * sizeof(uint16_t));
 968         }
 969
 970         return 0;
 971 }
 972
 973 static struct i40e_customized_pctype *
 974 i40e_flow_fdir_find_customized_pctype(struct i40e_pf *pf, uint8_t pctype)
 975 {
 976         struct i40e_customized_pctype *cus_pctype;
 977         enum i40e_new_pctype i = I40E_CUSTOMIZED_GTPC;
 978
 979         for (; i < I40E_CUSTOMIZED_MAX; i++) {
 980                 cus_pctype = &pf->customized_pctype[i];
 981                 if (pctype == cus_pctype->pctype)
 982                         return cus_pctype;
 983         }
 984         return NULL;
 985 }
 986
 987 static inline int
 988 fill_ip6_head(const struct i40e_fdir_input *fdir_input, unsigned char *raw_pkt,
 989                 uint8_t next_proto, uint8_t len, uint16_t *ether_type)
 990 {
 991         struct rte_ipv6_hdr *ip6;
 992
 993         ip6 = (struct rte_ipv6_hdr *)raw_pkt;
 994
 995         *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
 996         ip6->vtc_flow = rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
 997                 (fdir_input->flow.ipv6_flow.tc << I40E_FDIR_IPv6_TC_OFFSET));
 998         ip6->payload_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
 999         ip6->proto = fdir_input->flow.ipv6_flow.proto ?
1000                 fdir_input->flow.ipv6_flow.proto : next_proto;
1001         ip6->hop_limits = fdir_input->flow.ipv6_flow.hop_limits ?
1002                 fdir_input->flow.ipv6_flow.hop_limits :
1003                 I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
1004         /**
1005          * The source and destination fields in the transmitted packet
1006          * need to be presented in a reversed order with respect
1007          * to the expected received packets.
1008          */
1009         rte_memcpy(&ip6->src_addr, &fdir_input->flow.ipv6_flow.dst_ip,
1010                 IPV6_ADDR_LEN);
1011         rte_memcpy(&ip6->dst_addr, &fdir_input->flow.ipv6_flow.src_ip,
1012                 IPV6_ADDR_LEN);
1013         len += sizeof(struct rte_ipv6_hdr);
1014
1015         return len;
1016 }
1017
1018 static inline int
1019 fill_ip4_head(const struct i40e_fdir_input *fdir_input, unsigned char *raw_pkt,
1020                 uint8_t next_proto, uint8_t len, uint16_t *ether_type)
1021 {
1022         struct rte_ipv4_hdr *ip4;
1023
1024         ip4 = (struct rte_ipv4_hdr *)raw_pkt;
1025
1026         *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1027         ip4->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL;
1028         /* set len to by default */
1029         ip4->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN);
1030         ip4->time_to_live = fdir_input->flow.ip4_flow.ttl ?
1031                 fdir_input->flow.ip4_flow.ttl :
1032                 I40E_FDIR_IP_DEFAULT_TTL;
1033         ip4->type_of_service = fdir_input->flow.ip4_flow.tos;
1034         ip4->next_proto_id = fdir_input->flow.ip4_flow.proto ?
1035                 fdir_input->flow.ip4_flow.proto : next_proto;
1036         /**
1037          * The source and destination fields in the transmitted packet
1038          * need to be presented in a reversed order with respect
1039          * to the expected received packets.
1040          */
1041         ip4->src_addr = fdir_input->flow.ip4_flow.dst_ip;
1042         ip4->dst_addr = fdir_input->flow.ip4_flow.src_ip;
1043         len += sizeof(struct rte_ipv4_hdr);
1044
1045         return len;
1046 }
1047
1048 static inline int
1049 i40e_flow_fdir_fill_eth_ip_head(struct i40e_pf *pf,
1050                                 const struct i40e_fdir_input *fdir_input,
1051                                 unsigned char *raw_pkt,
1052                                 bool vlan)
1053 {
1054         struct i40e_customized_pctype *cus_pctype = NULL;
1055         static uint8_t vlan_frame[] = {0x81, 0, 0, 0};
1056         uint16_t *ether_type;
1057         uint8_t len = 2 * sizeof(struct rte_ether_addr);
1058         uint8_t pctype = fdir_input->pctype;
1059         bool is_customized_pctype = fdir_input->flow_ext.customized_pctype;
1060         static const uint8_t next_proto[] = {
1061                 [I40E_FILTER_PCTYPE_FRAG_IPV4] = IPPROTO_IP,
1062                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = IPPROTO_TCP,
1063                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = IPPROTO_UDP,
1064                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = IPPROTO_SCTP,
1065                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = IPPROTO_IP,
1066                 [I40E_FILTER_PCTYPE_FRAG_IPV6] = IPPROTO_NONE,
1067                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = IPPROTO_TCP,
1068                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = IPPROTO_UDP,
1069                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = IPPROTO_SCTP,
1070                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = IPPROTO_NONE,
1071         };
1072
1073         rte_memcpy(raw_pkt, &fdir_input->flow.l2_flow.dst,
1074                 sizeof(struct rte_ether_addr));
1075         rte_memcpy(raw_pkt + sizeof(struct rte_ether_addr),
1076                 &fdir_input->flow.l2_flow.src,
1077                 sizeof(struct rte_ether_addr));
1078         raw_pkt += 2 * sizeof(struct rte_ether_addr);
1079
1080         if (vlan && fdir_input->flow_ext.vlan_tci) {
1081                 rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame));
1082                 rte_memcpy(raw_pkt + sizeof(uint16_t),
1083                            &fdir_input->flow_ext.vlan_tci,
1084                            sizeof(uint16_t));
1085                 raw_pkt += sizeof(vlan_frame);
1086                 len += sizeof(vlan_frame);
1087         }
1088         ether_type = (uint16_t *)raw_pkt;
1089         raw_pkt += sizeof(uint16_t);
1090         len += sizeof(uint16_t);
1091
1092         if (is_customized_pctype) {
1093                 cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype);
1094                 if (!cus_pctype) {
1095                         PMD_DRV_LOG(ERR, "unknown pctype %u.",
1096                                     fdir_input->pctype);
1097                         return -1;
1098                 }
1099         }
1100
1101         if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD)
1102                 *ether_type = fdir_input->flow.l2_flow.ether_type;
1103         else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP ||
1104                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP ||
1105                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP ||
1106                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1107                  pctype == I40E_FILTER_PCTYPE_FRAG_IPV4 ||
1108                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP ||
1109                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP ||
1110                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP ||
1111                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1112                  pctype == I40E_FILTER_PCTYPE_FRAG_IPV6 ||
1113                  is_customized_pctype) {
1114                 if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP ||
1115                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP ||
1116                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP ||
1117                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1118                         pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) {
1119                         len = fill_ip4_head(fdir_input, raw_pkt,
1120                                         next_proto[pctype], len, ether_type);
1121                 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP ||
1122                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP ||
1123                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP ||
1124                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1125                         pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) {
1126                         len = fill_ip6_head(fdir_input, raw_pkt,
1127                                         next_proto[pctype], len,
1128                                         ether_type);
1129                 } else if (cus_pctype->index == I40E_CUSTOMIZED_GTPC ||
1130                          cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 ||
1131                          cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 ||
1132                          cus_pctype->index == I40E_CUSTOMIZED_GTPU) {
1133                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1134                                         len, ether_type);
1135                 } else if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3) {
1136                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_L2TP,
1137                                         len, ether_type);
1138                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4) {
1139                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_ESP,
1140                                         len, ether_type);
1141                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1142                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1143                                         len, ether_type);
1144                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1145                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1146                                         len, ether_type);
1147                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6)
1148                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_ESP,
1149                                         len, ether_type);
1150                 else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6_UDP)
1151                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_UDP,
1152                                         len, ether_type);
1153                 else if (cus_pctype->index == I40E_CUSTOMIZED_IPV6_L2TPV3)
1154                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_L2TP,
1155                                         len, ether_type);
1156         } else {
1157                 PMD_DRV_LOG(ERR, "unknown pctype %u.", fdir_input->pctype);
1158                 return -1;
1159         }
1160
1161         return len;
1162 }
1163
1164 /**
1165  * i40e_flow_fdir_construct_pkt - construct packet based on fields in input
1166  * @pf: board private structure
1167  * @fdir_input: input set of the flow director entry
1168  * @raw_pkt: a packet to be constructed
1169  */
1170 static int
1171 i40e_flow_fdir_construct_pkt(struct i40e_pf *pf,
1172                              const struct i40e_fdir_input *fdir_input,
1173                              unsigned char *raw_pkt)
1174 {
1175         unsigned char *payload = NULL;
1176         unsigned char *ptr;
1177         struct rte_udp_hdr *udp;
1178         struct rte_tcp_hdr *tcp;
1179         struct rte_sctp_hdr *sctp;
1180         struct rte_flow_item_gtp *gtp;
1181         struct rte_ipv4_hdr *gtp_ipv4;
1182         struct rte_ipv6_hdr *gtp_ipv6;
1183         struct rte_flow_item_l2tpv3oip *l2tpv3oip;
1184         struct rte_flow_item_esp *esp;
1185         struct rte_ipv4_hdr *esp_ipv4;
1186         struct rte_ipv6_hdr *esp_ipv6;
1187
1188         uint8_t size, dst = 0;
1189         uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/
1190         int len;
1191         uint8_t pctype = fdir_input->pctype;
1192         struct i40e_customized_pctype *cus_pctype;
1193
1194         /* raw pcket template - just copy contents of the raw packet */
1195         if (fdir_input->flow_ext.pkt_template) {
1196                 memcpy(raw_pkt, fdir_input->flow.raw_flow.packet,
1197                        fdir_input->flow.raw_flow.length);
1198                 return 0;
1199         }
1200
1201         /* fill the ethernet and IP head */
1202         len = i40e_flow_fdir_fill_eth_ip_head(pf, fdir_input, raw_pkt,
1203                                               !!fdir_input->flow_ext.vlan_tci);
1204         if (len < 0)
1205                 return -EINVAL;
1206
1207         /* fill the L4 head */
1208         if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP) {
1209                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
1210                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
1211                 /**
1212                  * The source and destination fields in the transmitted packet
1213                  * need to be presented in a reversed order with respect
1214                  * to the expected received packets.
1215                  */
1216                 udp->src_port = fdir_input->flow.udp4_flow.dst_port;
1217                 udp->dst_port = fdir_input->flow.udp4_flow.src_port;
1218                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
1219         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP) {
1220                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
1221                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
1222                 /**
1223                  * The source and destination fields in the transmitted packet
1224                  * need to be presented in a reversed order with respect
1225                  * to the expected received packets.
1226                  */
1227                 tcp->src_port = fdir_input->flow.tcp4_flow.dst_port;
1228                 tcp->dst_port = fdir_input->flow.tcp4_flow.src_port;
1229                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
1230         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) {
1231                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
1232                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
1233                 /**
1234                  * The source and destination fields in the transmitted packet
1235                  * need to be presented in a reversed order with respect
1236                  * to the expected received packets.
1237                  */
1238                 sctp->src_port = fdir_input->flow.sctp4_flow.dst_port;
1239                 sctp->dst_port = fdir_input->flow.sctp4_flow.src_port;
1240                 sctp->tag = fdir_input->flow.sctp4_flow.verify_tag;
1241         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1242                    pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) {
1243                 payload = raw_pkt + len;
1244                 set_idx = I40E_FLXPLD_L3_IDX;
1245         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP) {
1246                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
1247                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
1248                 /**
1249                  * The source and destination fields in the transmitted packet
1250                  * need to be presented in a reversed order with respect
1251                  * to the expected received packets.
1252                  */
1253                 udp->src_port = fdir_input->flow.udp6_flow.dst_port;
1254                 udp->dst_port = fdir_input->flow.udp6_flow.src_port;
1255                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
1256         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP) {
1257                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
1258                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
1259                 /**
1260                  * The source and destination fields in the transmitted packet
1261                  * need to be presented in a reversed order with respect
1262                  * to the expected received packets.
1263                  */
1264                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
1265                 tcp->src_port = fdir_input->flow.udp6_flow.dst_port;
1266                 tcp->dst_port = fdir_input->flow.udp6_flow.src_port;
1267         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) {
1268                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
1269                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
1270                 /**
1271                  * The source and destination fields in the transmitted packet
1272                  * need to be presented in a reversed order with respect
1273                  * to the expected received packets.
1274                  */
1275                 sctp->src_port = fdir_input->flow.sctp6_flow.dst_port;
1276                 sctp->dst_port = fdir_input->flow.sctp6_flow.src_port;
1277                 sctp->tag = fdir_input->flow.sctp6_flow.verify_tag;
1278         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1279                    pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) {
1280                 payload = raw_pkt + len;
1281                 set_idx = I40E_FLXPLD_L3_IDX;
1282         } else if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD) {
1283                 payload = raw_pkt + len;
1284                 /**
1285                  * ARP packet is a special case on which the payload
1286                  * starts after the whole ARP header
1287                  */
1288                 if (fdir_input->flow.l2_flow.ether_type ==
1289                                 rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
1290                         payload += sizeof(struct rte_arp_hdr);
1291                 set_idx = I40E_FLXPLD_L2_IDX;
1292         } else if (fdir_input->flow_ext.customized_pctype) {
1293                 /* If customized pctype is used */
1294                 cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype);
1295                 if (cus_pctype->index == I40E_CUSTOMIZED_GTPC ||
1296                     cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 ||
1297                     cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 ||
1298                     cus_pctype->index == I40E_CUSTOMIZED_GTPU) {
1299                         udp = (struct rte_udp_hdr *)(raw_pkt + len);
1300                         udp->dgram_len =
1301                                 rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
1302
1303                         gtp = (struct rte_flow_item_gtp *)
1304                                 ((unsigned char *)udp +
1305                                         sizeof(struct rte_udp_hdr));
1306                         gtp->msg_len =
1307                                 rte_cpu_to_be_16(I40E_FDIR_GTP_DEFAULT_LEN);
1308                         gtp->teid = fdir_input->flow.gtp_flow.teid;
1309                         gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0X01;
1310
1311                         /* GTP-C message type is not supported. */
1312                         if (cus_pctype->index == I40E_CUSTOMIZED_GTPC) {
1313                                 udp->dst_port =
1314                                       rte_cpu_to_be_16(I40E_FDIR_GTPC_DST_PORT);
1315                                 gtp->v_pt_rsv_flags =
1316                                         I40E_FDIR_GTP_VER_FLAG_0X32;
1317                         } else {
1318                                 udp->dst_port =
1319                                       rte_cpu_to_be_16(I40E_FDIR_GTPU_DST_PORT);
1320                                 gtp->v_pt_rsv_flags =
1321                                         I40E_FDIR_GTP_VER_FLAG_0X30;
1322                         }
1323
1324                         if (cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4) {
1325                                 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF;
1326                                 gtp_ipv4 = (struct rte_ipv4_hdr *)
1327                                         ((unsigned char *)gtp +
1328                                          sizeof(struct rte_flow_item_gtp));
1329                                 gtp_ipv4->version_ihl =
1330                                         I40E_FDIR_IP_DEFAULT_VERSION_IHL;
1331                                 gtp_ipv4->next_proto_id = IPPROTO_IP;
1332                                 gtp_ipv4->total_length =
1333                                         rte_cpu_to_be_16(
1334                                                 I40E_FDIR_INNER_IP_DEFAULT_LEN);
1335                                 payload = (unsigned char *)gtp_ipv4 +
1336                                         sizeof(struct rte_ipv4_hdr);
1337                         } else if (cus_pctype->index ==
1338                                    I40E_CUSTOMIZED_GTPU_IPV6) {
1339                                 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF;
1340                                 gtp_ipv6 = (struct rte_ipv6_hdr *)
1341                                         ((unsigned char *)gtp +
1342                                          sizeof(struct rte_flow_item_gtp));
1343                                 gtp_ipv6->vtc_flow =
1344                                         rte_cpu_to_be_32(
1345                                                I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
1346                                                (0 << I40E_FDIR_IPv6_TC_OFFSET));
1347                                 gtp_ipv6->proto = IPPROTO_NONE;
1348                                 gtp_ipv6->payload_len =
1349                                         rte_cpu_to_be_16(
1350                                               I40E_FDIR_INNER_IPV6_DEFAULT_LEN);
1351                                 gtp_ipv6->hop_limits =
1352                                         I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
1353                                 payload = (unsigned char *)gtp_ipv6 +
1354                                         sizeof(struct rte_ipv6_hdr);
1355                         } else
1356                                 payload = (unsigned char *)gtp +
1357                                         sizeof(struct rte_flow_item_gtp);
1358                 } else if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3 ||
1359                            cus_pctype->index == I40E_CUSTOMIZED_IPV6_L2TPV3) {
1360                         l2tpv3oip = (struct rte_flow_item_l2tpv3oip *)(raw_pkt
1361                                                                        + len);
1362
1363                         if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3)
1364                                 l2tpv3oip->session_id =
1365                                  fdir_input->flow.ip4_l2tpv3oip_flow.session_id;
1366                         else
1367                                 l2tpv3oip->session_id =
1368                                  fdir_input->flow.ip6_l2tpv3oip_flow.session_id;
1369                         payload = (unsigned char *)l2tpv3oip +
1370                                 sizeof(struct rte_flow_item_l2tpv3oip);
1371                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4 ||
1372                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6 ||
1373                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP ||
1374                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6_UDP) {
1375                         if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4) {
1376                                 esp_ipv4 = (struct rte_ipv4_hdr *)
1377                                         (raw_pkt + len);
1378                                 esp = (struct rte_flow_item_esp *)esp_ipv4;
1379                                 esp->hdr.spi =
1380                                         fdir_input->flow.esp_ipv4_flow.spi;
1381                                 payload = (unsigned char *)esp +
1382                                         sizeof(struct rte_esp_hdr);
1383                                 len += sizeof(struct rte_esp_hdr);
1384                         } else if (cus_pctype->index ==
1385                                         I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1386                                 esp_ipv4 = (struct rte_ipv4_hdr *)
1387                                         (raw_pkt + len);
1388                                 udp = (struct rte_udp_hdr *)esp_ipv4;
1389                                 udp->dst_port = rte_cpu_to_be_16
1390                                         (I40E_FDIR_ESP_DST_PORT);
1391
1392                                 udp->dgram_len = rte_cpu_to_be_16
1393                                                 (I40E_FDIR_UDP_DEFAULT_LEN);
1394                                 esp = (struct rte_flow_item_esp *)
1395                                         ((unsigned char *)esp_ipv4 +
1396                                                 sizeof(struct rte_udp_hdr));
1397                                 esp->hdr.spi =
1398                                         fdir_input->flow.esp_ipv4_udp_flow.spi;
1399                                 payload = (unsigned char *)esp +
1400                                         sizeof(struct rte_esp_hdr);
1401                                 len += sizeof(struct rte_udp_hdr) +
1402                                                 sizeof(struct rte_esp_hdr);
1403                         } else if (cus_pctype->index ==
1404                                         I40E_CUSTOMIZED_ESP_IPV6) {
1405                                 esp_ipv6 = (struct rte_ipv6_hdr *)
1406                                         (raw_pkt + len);
1407                                 esp = (struct rte_flow_item_esp *)esp_ipv6;
1408                                 esp->hdr.spi =
1409                                         fdir_input->flow.esp_ipv6_flow.spi;
1410                                 payload = (unsigned char *)esp +
1411                                         sizeof(struct rte_esp_hdr);
1412                                 len += sizeof(struct rte_esp_hdr);
1413                         } else if (cus_pctype->index ==
1414                                         I40E_CUSTOMIZED_ESP_IPV6_UDP) {
1415                                 esp_ipv6 = (struct rte_ipv6_hdr *)
1416                                         (raw_pkt + len);
1417                                 udp = (struct rte_udp_hdr *)esp_ipv6;
1418                                 udp->dst_port = rte_cpu_to_be_16
1419                                         (I40E_FDIR_ESP_DST_PORT);
1420
1421                                 udp->dgram_len = rte_cpu_to_be_16
1422                                         (I40E_FDIR_UDP_DEFAULT_LEN);
1423                                 esp = (struct rte_flow_item_esp *)
1424                                         ((unsigned char *)esp_ipv6 +
1425                                                 sizeof(struct rte_udp_hdr));
1426                                 esp->hdr.spi =
1427                                         fdir_input->flow.esp_ipv6_udp_flow.spi;
1428                                 payload = (unsigned char *)esp +
1429                                         sizeof(struct rte_esp_hdr);
1430                                 len += sizeof(struct rte_udp_hdr) +
1431                                                 sizeof(struct rte_esp_hdr);
1432                         }
1433                 }
1434         } else {
1435                 PMD_DRV_LOG(ERR, "unknown pctype %u.", fdir_input->pctype);
1436                 return -1;
1437         }
1438
1439         /* fill the flexbytes to payload */
1440         for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
1441                 pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i;
1442                 size = pf->fdir.flex_set[pit_idx].size;
1443                 if (size == 0)
1444                         continue;
1445                 dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t);
1446                 ptr = payload +
1447                       pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t);
1448                 (void)rte_memcpy(ptr,
1449                                  &fdir_input->flow_ext.flexbytes[dst],
1450                                  size * sizeof(uint16_t));
1451         }
1452
1453         return 0;
1454 }
1455
1456 /* Construct the tx flags */
1457 static inline uint64_t
1458 i40e_build_ctob(uint32_t td_cmd,
1459                 uint32_t td_offset,
1460                 unsigned int size,
1461                 uint32_t td_tag)
1462 {
1463         return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA |
1464                         ((uint64_t)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
1465                         ((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
1466                         ((uint64_t)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
1467                         ((uint64_t)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
1468 }
1469
1470 /*
1471  * check the programming status descriptor in rx queue.
1472  * done after Programming Flow Director is programmed on
1473  * tx queue
1474  */
1475 static inline int
1476 i40e_check_fdir_programming_status(struct i40e_rx_queue *rxq)
1477 {
1478         volatile union i40e_rx_desc *rxdp;
1479         uint64_t qword1;
1480         uint32_t rx_status;
1481         uint32_t len, id;
1482         uint32_t error;
1483         int ret = 0;
1484
1485         rxdp = &rxq->rx_ring[rxq->rx_tail];
1486         qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
1487         rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK)
1488                         >> I40E_RXD_QW1_STATUS_SHIFT;
1489
1490         if (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1491                 len = qword1 >> I40E_RX_PROG_STATUS_DESC_LENGTH_SHIFT;
1492                 id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
1493                             I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
1494
1495                 if (len  == I40E_RX_PROG_STATUS_DESC_LENGTH &&
1496                     id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) {
1497                         error = (qword1 &
1498                                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
1499                                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
1500                         if (error == (0x1 <<
1501                                 I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
1502                                 PMD_DRV_LOG(ERR, "Failed to add FDIR filter"
1503                                             " (FD_ID %u): programming status"
1504                                             " reported.",
1505                                             rxdp->wb.qword0.hi_dword.fd_id);
1506                                 ret = -1;
1507                         } else if (error == (0x1 <<
1508                                 I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
1509                                 PMD_DRV_LOG(ERR, "Failed to delete FDIR filter"
1510                                             " (FD_ID %u): programming status"
1511                                             " reported.",
1512                                             rxdp->wb.qword0.hi_dword.fd_id);
1513                                 ret = -1;
1514                         } else
1515                                 PMD_DRV_LOG(ERR, "invalid programming status"
1516                                             " reported, error = %u.", error);
1517                 } else
1518                         PMD_DRV_LOG(INFO, "unknown programming status"
1519                                     " reported, len = %d, id = %u.", len, id);
1520                 rxdp->wb.qword1.status_error_len = 0;
1521                 rxq->rx_tail++;
1522                 if (unlikely(rxq->rx_tail == rxq->nb_rx_desc))
1523                         rxq->rx_tail = 0;
1524                 if (rxq->rx_tail == 0)
1525                         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1526                 else
1527                         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_tail - 1);
1528         }
1529
1530         return ret;
1531 }
1532
1533 static int
1534 i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input,
1535                          struct i40e_fdir_filter *filter)
1536 {
1537         rte_memcpy(&filter->fdir, input, sizeof(struct i40e_fdir_filter_conf));
1538         if (input->input.flow_ext.pkt_template) {
1539                 filter->fdir.input.flow.raw_flow.packet = NULL;
1540                 filter->fdir.input.flow.raw_flow.length =
1541                         rte_hash_crc(input->input.flow.raw_flow.packet,
1542                                      input->input.flow.raw_flow.length,
1543                                      input->input.flow.raw_flow.pctype);
1544         }
1545         return 0;
1546 }
1547
1548 /* Check if there exists the flow director filter */
1549 static struct i40e_fdir_filter *
1550 i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info,
1551                         const struct i40e_fdir_input *input)
1552 {
1553         int ret;
1554
1555         if (input->flow_ext.pkt_template)
1556                 ret = rte_hash_lookup_with_hash(fdir_info->hash_table,
1557                                                 (const void *)input,
1558                                                 input->flow.raw_flow.length);
1559         else
1560                 ret = rte_hash_lookup(fdir_info->hash_table,
1561                                       (const void *)input);
1562         if (ret < 0)
1563                 return NULL;
1564
1565         return fdir_info->hash_map[ret];
1566 }
1567
1568 /* Add a flow director filter into the SW list */
1569 static int
1570 i40e_sw_fdir_filter_insert(struct i40e_pf *pf, struct i40e_fdir_filter *filter)
1571 {
1572         struct i40e_fdir_info *fdir_info = &pf->fdir;
1573         int ret;
1574
1575         if (filter->fdir.input.flow_ext.pkt_template)
1576                 ret = rte_hash_add_key_with_hash(fdir_info->hash_table,
1577                                  &filter->fdir.input,
1578                                  filter->fdir.input.flow.raw_flow.length);
1579         else
1580                 ret = rte_hash_add_key(fdir_info->hash_table,
1581                                        &filter->fdir.input);
1582         if (ret < 0) {
1583                 PMD_DRV_LOG(ERR,
1584                             "Failed to insert fdir filter to hash table %d!",
1585                             ret);
1586                 return ret;
1587         }
1588         fdir_info->hash_map[ret] = filter;
1589
1590         TAILQ_INSERT_TAIL(&fdir_info->fdir_list, filter, rules);
1591
1592         return 0;
1593 }
1594
1595 /* Delete a flow director filter from the SW list */
1596 int
1597 i40e_sw_fdir_filter_del(struct i40e_pf *pf, struct i40e_fdir_input *input)
1598 {
1599         struct i40e_fdir_info *fdir_info = &pf->fdir;
1600         struct i40e_fdir_filter *filter;
1601         int ret;
1602
1603         if (input->flow_ext.pkt_template)
1604                 ret = rte_hash_del_key_with_hash(fdir_info->hash_table,
1605                                                  input,
1606                                                  input->flow.raw_flow.length);
1607         else
1608                 ret = rte_hash_del_key(fdir_info->hash_table, input);
1609         if (ret < 0) {
1610                 PMD_DRV_LOG(ERR,
1611                             "Failed to delete fdir filter to hash table %d!",
1612                             ret);
1613                 return ret;
1614         }
1615         filter = fdir_info->hash_map[ret];
1616         fdir_info->hash_map[ret] = NULL;
1617
1618         TAILQ_REMOVE(&fdir_info->fdir_list, filter, rules);
1619         rte_free(filter);
1620
1621         return 0;
1622 }
1623
1624 /*
1625  * i40e_add_del_fdir_filter - add or remove a flow director filter.
1626  * @pf: board private structure
1627  * @filter: fdir filter entry
1628  * @add: 0 - delete, 1 - add
1629  */
1630 int
1631 i40e_add_del_fdir_filter(struct rte_eth_dev *dev,
1632                          const struct rte_eth_fdir_filter *filter,
1633                          bool add)
1634 {
1635         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1636         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1637         unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
1638         enum i40e_filter_pctype pctype;
1639         int ret = 0;
1640
1641         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_PERFECT) {
1642                 PMD_DRV_LOG(ERR, "FDIR is not enabled, please"
1643                         " check the mode in fdir_conf.");
1644                 return -ENOTSUP;
1645         }
1646
1647         pctype = i40e_flowtype_to_pctype(pf->adapter, filter->input.flow_type);
1648         if (pctype == I40E_FILTER_PCTYPE_INVALID) {
1649                 PMD_DRV_LOG(ERR, "invalid flow_type input.");
1650                 return -EINVAL;
1651         }
1652         if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) {
1653                 PMD_DRV_LOG(ERR, "Invalid queue ID");
1654                 return -EINVAL;
1655         }
1656         if (filter->input.flow_ext.is_vf &&
1657                 filter->input.flow_ext.dst_id >= pf->vf_num) {
1658                 PMD_DRV_LOG(ERR, "Invalid VF ID");
1659                 return -EINVAL;
1660         }
1661
1662         memset(pkt, 0, I40E_FDIR_PKT_LEN);
1663
1664         ret = i40e_fdir_construct_pkt(pf, &filter->input, pkt);
1665         if (ret < 0) {
1666                 PMD_DRV_LOG(ERR, "construct packet for fdir fails.");
1667                 return ret;
1668         }
1669
1670         if (hw->mac.type == I40E_MAC_X722) {
1671                 /* get translated pctype value in fd pctype register */
1672                 pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(
1673                         hw, I40E_GLQF_FD_PCTYPES((int)pctype));
1674         }
1675
1676         ret = i40e_fdir_filter_programming(pf, pctype, filter, add);
1677         if (ret < 0) {
1678                 PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).",
1679                             pctype);
1680                 return ret;
1681         }
1682
1683         return ret;
1684 }
1685
1686 /**
1687  * i40e_flow_add_del_fdir_filter - add or remove a flow director filter.
1688  * @pf: board private structure
1689  * @filter: fdir filter entry
1690  * @add: 0 - delete, 1 - add
1691  */
1692 int
1693 i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev,
1694                               const struct i40e_fdir_filter_conf *filter,
1695                               bool add)
1696 {
1697         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1698         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1699         unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
1700         enum i40e_filter_pctype pctype;
1701         struct i40e_fdir_info *fdir_info = &pf->fdir;
1702         struct i40e_fdir_filter *fdir_filter, *node;
1703         struct i40e_fdir_filter check_filter; /* Check if the filter exists */
1704         int ret = 0;
1705
1706         if (pf->fdir.fdir_vsi == NULL) {
1707                 PMD_DRV_LOG(ERR, "FDIR is not enabled");
1708                 return -ENOTSUP;
1709         }
1710
1711         if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) {
1712                 PMD_DRV_LOG(ERR, "Invalid queue ID");
1713                 return -EINVAL;
1714         }
1715         if (filter->input.flow_ext.is_vf &&
1716             filter->input.flow_ext.dst_id >= pf->vf_num) {
1717                 PMD_DRV_LOG(ERR, "Invalid VF ID");
1718                 return -EINVAL;
1719         }
1720         if (filter->input.flow_ext.pkt_template) {
1721                 if (filter->input.flow.raw_flow.length > I40E_FDIR_PKT_LEN ||
1722                     !filter->input.flow.raw_flow.packet) {
1723                         PMD_DRV_LOG(ERR, "Invalid raw packet template"
1724                                 " flow filter parameters!");
1725                         return -EINVAL;
1726                 }
1727                 pctype = filter->input.flow.raw_flow.pctype;
1728         } else {
1729                 pctype = filter->input.pctype;
1730         }
1731
1732         /* Check if there is the filter in SW list */
1733         memset(&check_filter, 0, sizeof(check_filter));
1734         i40e_fdir_filter_convert(filter, &check_filter);
1735         node = i40e_sw_fdir_filter_lookup(fdir_info, &check_filter.fdir.input);
1736         if (add && node) {
1737                 PMD_DRV_LOG(ERR,
1738                             "Conflict with existing flow director rules!");
1739                 return -EINVAL;
1740         }
1741
1742         if (!add && !node) {
1743                 PMD_DRV_LOG(ERR,
1744                             "There's no corresponding flow firector filter!");
1745                 return -EINVAL;
1746         }
1747
1748         memset(pkt, 0, I40E_FDIR_PKT_LEN);
1749
1750         ret = i40e_flow_fdir_construct_pkt(pf, &filter->input, pkt);
1751         if (ret < 0) {
1752                 PMD_DRV_LOG(ERR, "construct packet for fdir fails.");
1753                 return ret;
1754         }
1755
1756         if (hw->mac.type == I40E_MAC_X722) {
1757                 /* get translated pctype value in fd pctype register */
1758                 pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(
1759                         hw, I40E_GLQF_FD_PCTYPES((int)pctype));
1760         }
1761
1762         ret = i40e_flow_fdir_filter_programming(pf, pctype, filter, add);
1763         if (ret < 0) {
1764                 PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).",
1765                             pctype);
1766                 return ret;
1767         }
1768
1769         if (add) {
1770                 fdir_info->fdir_actual_cnt++;
1771                 if (fdir_info->fdir_invalprio == 1 &&
1772                                 fdir_info->fdir_guarantee_free_space > 0)
1773                         fdir_info->fdir_guarantee_free_space--;
1774
1775                 fdir_filter = rte_zmalloc("fdir_filter",
1776                                           sizeof(*fdir_filter), 0);
1777                 if (fdir_filter == NULL) {
1778                         PMD_DRV_LOG(ERR, "Failed to alloc memory.");
1779                         return -ENOMEM;
1780                 }
1781
1782                 rte_memcpy(fdir_filter, &check_filter, sizeof(check_filter));
1783                 ret = i40e_sw_fdir_filter_insert(pf, fdir_filter);
1784                 if (ret < 0)
1785                         rte_free(fdir_filter);
1786         } else {
1787                 fdir_info->fdir_actual_cnt--;
1788                 if (fdir_info->fdir_invalprio == 1 &&
1789                                 fdir_info->fdir_guarantee_free_space <
1790                                 fdir_info->fdir_guarantee_total_space)
1791                         fdir_info->fdir_guarantee_free_space++;
1792
1793                 ret = i40e_sw_fdir_filter_del(pf, &node->fdir.input);
1794         }
1795
1796         return ret;
1797 }
1798
1799 /*
1800  * i40e_fdir_filter_programming - Program a flow director filter rule.
1801  * Is done by Flow Director Programming Descriptor followed by packet
1802  * structure that contains the filter fields need to match.
1803  * @pf: board private structure
1804  * @pctype: pctype
1805  * @filter: fdir filter entry
1806  * @add: 0 - delete, 1 - add
1807  */
1808 static int
1809 i40e_fdir_filter_programming(struct i40e_pf *pf,
1810                         enum i40e_filter_pctype pctype,
1811                         const struct rte_eth_fdir_filter *filter,
1812                         bool add)
1813 {
1814         struct i40e_tx_queue *txq = pf->fdir.txq;
1815         struct i40e_rx_queue *rxq = pf->fdir.rxq;
1816         const struct rte_eth_fdir_action *fdir_action = &filter->action;
1817         volatile struct i40e_tx_desc *txdp;
1818         volatile struct i40e_filter_program_desc *fdirdp;
1819         uint32_t td_cmd;
1820         uint16_t vsi_id, i;
1821         uint8_t dest;
1822
1823         PMD_DRV_LOG(INFO, "filling filter programming descriptor.");
1824         fdirdp = (volatile struct i40e_filter_program_desc *)
1825                         (&(txq->tx_ring[txq->tx_tail]));
1826
1827         fdirdp->qindex_flex_ptype_vsi =
1828                         rte_cpu_to_le_32((fdir_action->rx_queue <<
1829                                           I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1830                                           I40E_TXD_FLTR_QW0_QINDEX_MASK);
1831
1832         fdirdp->qindex_flex_ptype_vsi |=
1833                         rte_cpu_to_le_32((fdir_action->flex_off <<
1834                                           I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
1835                                           I40E_TXD_FLTR_QW0_FLEXOFF_MASK);
1836
1837         fdirdp->qindex_flex_ptype_vsi |=
1838                         rte_cpu_to_le_32((pctype <<
1839                                           I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
1840                                           I40E_TXD_FLTR_QW0_PCTYPE_MASK);
1841
1842         if (filter->input.flow_ext.is_vf)
1843                 vsi_id = pf->vfs[filter->input.flow_ext.dst_id].vsi->vsi_id;
1844         else
1845                 /* Use LAN VSI Id by default */
1846                 vsi_id = pf->main_vsi->vsi_id;
1847         fdirdp->qindex_flex_ptype_vsi |=
1848                 rte_cpu_to_le_32(((uint32_t)vsi_id <<
1849                                   I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
1850                                   I40E_TXD_FLTR_QW0_DEST_VSI_MASK);
1851
1852         fdirdp->dtype_cmd_cntindex =
1853                         rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG);
1854
1855         if (add)
1856                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1857                                 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1858                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1859         else
1860                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1861                                 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1862                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1863
1864         if (fdir_action->behavior == RTE_ETH_FDIR_REJECT)
1865                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
1866         else if (fdir_action->behavior == RTE_ETH_FDIR_ACCEPT)
1867                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX;
1868         else if (fdir_action->behavior == RTE_ETH_FDIR_PASSTHRU)
1869                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_OTHER;
1870         else {
1871                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter:"
1872                             " unsupported fdir behavior.");
1873                 return -EINVAL;
1874         }
1875
1876         fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest <<
1877                                 I40E_TXD_FLTR_QW1_DEST_SHIFT) &
1878                                 I40E_TXD_FLTR_QW1_DEST_MASK);
1879
1880         fdirdp->dtype_cmd_cntindex |=
1881                 rte_cpu_to_le_32((fdir_action->report_status<<
1882                                 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
1883                                 I40E_TXD_FLTR_QW1_FD_STATUS_MASK);
1884
1885         fdirdp->dtype_cmd_cntindex |=
1886                         rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK);
1887         fdirdp->dtype_cmd_cntindex |=
1888                         rte_cpu_to_le_32(
1889                         ((uint32_t)pf->fdir.match_counter_index <<
1890                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
1891                         I40E_TXD_FLTR_QW1_CNTINDEX_MASK);
1892
1893         fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id);
1894
1895         PMD_DRV_LOG(INFO, "filling transmit descriptor.");
1896         txdp = &(txq->tx_ring[txq->tx_tail + 1]);
1897         txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
1898         td_cmd = I40E_TX_DESC_CMD_EOP |
1899                  I40E_TX_DESC_CMD_RS  |
1900                  I40E_TX_DESC_CMD_DUMMY;
1901
1902         txdp->cmd_type_offset_bsz =
1903                 i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0);
1904
1905         txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */
1906         if (txq->tx_tail >= txq->nb_tx_desc)
1907                 txq->tx_tail = 0;
1908         /* Update the tx tail register */
1909         rte_wmb();
1910         I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
1911         for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
1912                 if ((txdp->cmd_type_offset_bsz &
1913                                 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
1914                                 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
1915                         break;
1916                 rte_delay_us(1);
1917         }
1918         if (i >= I40E_FDIR_MAX_WAIT_US) {
1919                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter:"
1920                             " time out to get DD on tx queue.");
1921                 return -ETIMEDOUT;
1922         }
1923         /* totally delay 10 ms to check programming status*/
1924         for (; i < I40E_FDIR_MAX_WAIT_US; i++) {
1925                 if (i40e_check_fdir_programming_status(rxq) >= 0)
1926                         return 0;
1927                 rte_delay_us(1);
1928         }
1929         PMD_DRV_LOG(ERR,
1930                 "Failed to program FDIR filter: programming status reported.");
1931         return -ETIMEDOUT;
1932 }
1933
1934 /*
1935  * i40e_flow_fdir_filter_programming - Program a flow director filter rule.
1936  * Is done by Flow Director Programming Descriptor followed by packet
1937  * structure that contains the filter fields need to match.
1938  * @pf: board private structure
1939  * @pctype: pctype
1940  * @filter: fdir filter entry
1941  * @add: 0 - delete, 1 - add
1942  */
1943 static int
1944 i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
1945                                   enum i40e_filter_pctype pctype,
1946                                   const struct i40e_fdir_filter_conf *filter,
1947                                   bool add)
1948 {
1949         struct i40e_tx_queue *txq = pf->fdir.txq;
1950         struct i40e_rx_queue *rxq = pf->fdir.rxq;
1951         const struct i40e_fdir_action *fdir_action = &filter->action;
1952         volatile struct i40e_tx_desc *txdp;
1953         volatile struct i40e_filter_program_desc *fdirdp;
1954         uint32_t td_cmd;
1955         uint16_t vsi_id, i;
1956         uint8_t dest;
1957
1958         PMD_DRV_LOG(INFO, "filling filter programming descriptor.");
1959         fdirdp = (volatile struct i40e_filter_program_desc *)
1960                                 (&txq->tx_ring[txq->tx_tail]);
1961
1962         fdirdp->qindex_flex_ptype_vsi =
1963                         rte_cpu_to_le_32((fdir_action->rx_queue <<
1964                                           I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1965                                           I40E_TXD_FLTR_QW0_QINDEX_MASK);
1966
1967         fdirdp->qindex_flex_ptype_vsi |=
1968                         rte_cpu_to_le_32((fdir_action->flex_off <<
1969                                           I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
1970                                           I40E_TXD_FLTR_QW0_FLEXOFF_MASK);
1971
1972         fdirdp->qindex_flex_ptype_vsi |=
1973                         rte_cpu_to_le_32((pctype <<
1974                                           I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
1975                                           I40E_TXD_FLTR_QW0_PCTYPE_MASK);
1976
1977         if (filter->input.flow_ext.is_vf)
1978                 vsi_id = pf->vfs[filter->input.flow_ext.dst_id].vsi->vsi_id;
1979         else
1980                 /* Use LAN VSI Id by default */
1981                 vsi_id = pf->main_vsi->vsi_id;
1982         fdirdp->qindex_flex_ptype_vsi |=
1983                 rte_cpu_to_le_32(((uint32_t)vsi_id <<
1984                                   I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
1985                                   I40E_TXD_FLTR_QW0_DEST_VSI_MASK);
1986
1987         fdirdp->dtype_cmd_cntindex =
1988                         rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG);
1989
1990         if (add)
1991                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1992                                 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1993                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1994         else
1995                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1996                                 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1997                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1998
1999         if (fdir_action->behavior == I40E_FDIR_REJECT)
2000                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
2001         else if (fdir_action->behavior == I40E_FDIR_ACCEPT)
2002                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX;
2003         else if (fdir_action->behavior == I40E_FDIR_PASSTHRU)
2004                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_OTHER;
2005         else {
2006                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter: unsupported fdir behavior.");
2007                 return -EINVAL;
2008         }
2009
2010         fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest <<
2011                                 I40E_TXD_FLTR_QW1_DEST_SHIFT) &
2012                                 I40E_TXD_FLTR_QW1_DEST_MASK);
2013
2014         fdirdp->dtype_cmd_cntindex |=
2015                 rte_cpu_to_le_32((fdir_action->report_status <<
2016                                 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
2017                                 I40E_TXD_FLTR_QW1_FD_STATUS_MASK);
2018
2019         fdirdp->dtype_cmd_cntindex |=
2020                         rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK);
2021         fdirdp->dtype_cmd_cntindex |=
2022                         rte_cpu_to_le_32(
2023                         ((uint32_t)pf->fdir.match_counter_index <<
2024                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
2025                         I40E_TXD_FLTR_QW1_CNTINDEX_MASK);
2026
2027         fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id);
2028
2029         PMD_DRV_LOG(INFO, "filling transmit descriptor.");
2030         txdp = &txq->tx_ring[txq->tx_tail + 1];
2031         txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
2032         td_cmd = I40E_TX_DESC_CMD_EOP |
2033                  I40E_TX_DESC_CMD_RS  |
2034                  I40E_TX_DESC_CMD_DUMMY;
2035
2036         txdp->cmd_type_offset_bsz =
2037                 i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0);
2038
2039         txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */
2040         if (txq->tx_tail >= txq->nb_tx_desc)
2041                 txq->tx_tail = 0;
2042         /* Update the tx tail register */
2043         rte_wmb();
2044         I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
2045         for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
2046                 if ((txdp->cmd_type_offset_bsz &
2047                                 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
2048                                 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
2049                         break;
2050                 rte_delay_us(1);
2051         }
2052         if (i >= I40E_FDIR_MAX_WAIT_US) {
2053                 PMD_DRV_LOG(ERR,
2054                     "Failed to program FDIR filter: time out to get DD on tx queue.");
2055                 return -ETIMEDOUT;
2056         }
2057         /* totally delay 10 ms to check programming status*/
2058         rte_delay_us(I40E_FDIR_MAX_WAIT_US);
2059         if (i40e_check_fdir_programming_status(rxq) < 0) {
2060                 PMD_DRV_LOG(ERR,
2061                     "Failed to program FDIR filter: programming status reported.");
2062                 return -ETIMEDOUT;
2063         }
2064
2065         return 0;
2066 }
2067
2068 /*
2069  * i40e_fdir_flush - clear all filters of Flow Director table
2070  * @pf: board private structure
2071  */
2072 int
2073 i40e_fdir_flush(struct rte_eth_dev *dev)
2074 {
2075         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2076         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2077         uint32_t reg;
2078         uint16_t guarant_cnt, best_cnt;
2079         uint16_t i;
2080
2081         I40E_WRITE_REG(hw, I40E_PFQF_CTL_1, I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
2082         I40E_WRITE_FLUSH(hw);
2083
2084         for (i = 0; i < I40E_FDIR_FLUSH_RETRY; i++) {
2085                 rte_delay_ms(I40E_FDIR_FLUSH_INTERVAL_MS);
2086                 reg = I40E_READ_REG(hw, I40E_PFQF_CTL_1);
2087                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
2088                         break;
2089         }
2090         if (i >= I40E_FDIR_FLUSH_RETRY) {
2091                 PMD_DRV_LOG(ERR, "FD table did not flush, may need more time.");
2092                 return -ETIMEDOUT;
2093         }
2094         guarant_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
2095                                 I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2096                                 I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2097         best_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
2098                                 I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2099                                 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2100         if (guarant_cnt != 0 || best_cnt != 0) {
2101                 PMD_DRV_LOG(ERR, "Failed to flush FD table.");
2102                 return -ENOSYS;
2103         } else
2104                 PMD_DRV_LOG(INFO, "FD table Flush success.");
2105         return 0;
2106 }
2107
2108 static inline void
2109 i40e_fdir_info_get_flex_set(struct i40e_pf *pf,
2110                         struct rte_eth_flex_payload_cfg *flex_set,
2111                         uint16_t *num)
2112 {
2113         struct i40e_fdir_flex_pit *flex_pit;
2114         struct rte_eth_flex_payload_cfg *ptr = flex_set;
2115         uint16_t src, dst, size, j, k;
2116         uint8_t i, layer_idx;
2117
2118         for (layer_idx = I40E_FLXPLD_L2_IDX;
2119              layer_idx <= I40E_FLXPLD_L4_IDX;
2120              layer_idx++) {
2121                 if (layer_idx == I40E_FLXPLD_L2_IDX)
2122                         ptr->type = RTE_ETH_L2_PAYLOAD;
2123                 else if (layer_idx == I40E_FLXPLD_L3_IDX)
2124                         ptr->type = RTE_ETH_L3_PAYLOAD;
2125                 else if (layer_idx == I40E_FLXPLD_L4_IDX)
2126                         ptr->type = RTE_ETH_L4_PAYLOAD;
2127
2128                 for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
2129                         flex_pit = &pf->fdir.flex_set[layer_idx *
2130                                 I40E_MAX_FLXPLD_FIED + i];
2131                         if (flex_pit->size == 0)
2132                                 continue;
2133                         src = flex_pit->src_offset * sizeof(uint16_t);
2134                         dst = flex_pit->dst_offset * sizeof(uint16_t);
2135                         size = flex_pit->size * sizeof(uint16_t);
2136                         for (j = src, k = dst; j < src + size; j++, k++)
2137                                 ptr->src_offset[k] = j;
2138                 }
2139                 (*num)++;
2140                 ptr++;
2141         }
2142 }
2143
2144 static inline void
2145 i40e_fdir_info_get_flex_mask(struct i40e_pf *pf,
2146                         struct rte_eth_fdir_flex_mask *flex_mask,
2147                         uint16_t *num)
2148 {
2149         struct i40e_fdir_flex_mask *mask;
2150         struct rte_eth_fdir_flex_mask *ptr = flex_mask;
2151         uint16_t flow_type;
2152         uint8_t i, j;
2153         uint16_t off_bytes, mask_tmp;
2154
2155         for (i = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
2156              i <= I40E_FILTER_PCTYPE_L2_PAYLOAD;
2157              i++) {
2158                 mask =  &pf->fdir.flex_mask[i];
2159                 flow_type = i40e_pctype_to_flowtype(pf->adapter,
2160                                                     (enum i40e_filter_pctype)i);
2161                 if (flow_type == RTE_ETH_FLOW_UNKNOWN)
2162                         continue;
2163
2164                 for (j = 0; j < I40E_FDIR_MAX_FLEXWORD_NUM; j++) {
2165                         if (mask->word_mask & I40E_FLEX_WORD_MASK(j)) {
2166                                 ptr->mask[j * sizeof(uint16_t)] = UINT8_MAX;
2167                                 ptr->mask[j * sizeof(uint16_t) + 1] = UINT8_MAX;
2168                         } else {
2169                                 ptr->mask[j * sizeof(uint16_t)] = 0x0;
2170                                 ptr->mask[j * sizeof(uint16_t) + 1] = 0x0;
2171                         }
2172                 }
2173                 for (j = 0; j < I40E_FDIR_BITMASK_NUM_WORD; j++) {
2174                         off_bytes = mask->bitmask[j].offset * sizeof(uint16_t);
2175                         mask_tmp = ~mask->bitmask[j].mask;
2176                         ptr->mask[off_bytes] &= I40E_HI_BYTE(mask_tmp);
2177                         ptr->mask[off_bytes + 1] &= I40E_LO_BYTE(mask_tmp);
2178                 }
2179                 ptr->flow_type = flow_type;
2180                 ptr++;
2181                 (*num)++;
2182         }
2183 }
2184
2185 /*
2186  * i40e_fdir_info_get - get information of Flow Director
2187  * @pf: ethernet device to get info from
2188  * @fdir: a pointer to a structure of type *rte_eth_fdir_info* to be filled with
2189  *    the flow director information.
2190  */
2191 void
2192 i40e_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir)
2193 {
2194         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2195         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2196         uint16_t num_flex_set = 0;
2197         uint16_t num_flex_mask = 0;
2198         uint16_t i;
2199
2200         if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT)
2201                 fdir->mode = RTE_FDIR_MODE_PERFECT;
2202         else
2203                 fdir->mode = RTE_FDIR_MODE_NONE;
2204
2205         fdir->guarant_spc =
2206                 (uint32_t)hw->func_caps.fd_filters_guaranteed;
2207         fdir->best_spc =
2208                 (uint32_t)hw->func_caps.fd_filters_best_effort;
2209         fdir->max_flexpayload = I40E_FDIR_MAX_FLEX_LEN;
2210         fdir->flow_types_mask[0] = I40E_FDIR_FLOWS;
2211         for (i = 1; i < RTE_FLOW_MASK_ARRAY_SIZE; i++)
2212                 fdir->flow_types_mask[i] = 0ULL;
2213         fdir->flex_payload_unit = sizeof(uint16_t);
2214         fdir->flex_bitmask_unit = sizeof(uint16_t);
2215         fdir->max_flex_payload_segment_num = I40E_MAX_FLXPLD_FIED;
2216         fdir->flex_payload_limit = I40E_MAX_FLX_SOURCE_OFF;
2217         fdir->max_flex_bitmask_num = I40E_FDIR_BITMASK_NUM_WORD;
2218
2219         i40e_fdir_info_get_flex_set(pf,
2220                                 fdir->flex_conf.flex_set,
2221                                 &num_flex_set);
2222         i40e_fdir_info_get_flex_mask(pf,
2223                                 fdir->flex_conf.flex_mask,
2224                                 &num_flex_mask);
2225
2226         fdir->flex_conf.nb_payloads = num_flex_set;
2227         fdir->flex_conf.nb_flexmasks = num_flex_mask;
2228 }
2229
2230 /*
2231  * i40e_fdir_stat_get - get statistics of Flow Director
2232  * @pf: ethernet device to get info from
2233  * @stat: a pointer to a structure of type *rte_eth_fdir_stats* to be filled with
2234  *    the flow director statistics.
2235  */
2236 void
2237 i40e_fdir_stats_get(struct rte_eth_dev *dev, struct rte_eth_fdir_stats *stat)
2238 {
2239         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2240         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2241         uint32_t fdstat;
2242
2243         fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT);
2244         stat->guarant_cnt =
2245                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2246                             I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2247         stat->best_cnt =
2248                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2249                             I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2250 }
2251
2252 static int
2253 i40e_fdir_filter_set(struct rte_eth_dev *dev,
2254                      struct rte_eth_fdir_filter_info *info)
2255 {
2256         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2257         int ret = 0;
2258
2259         if (!info) {
2260                 PMD_DRV_LOG(ERR, "Invalid pointer");
2261                 return -EFAULT;
2262         }
2263
2264         switch (info->info_type) {
2265         case RTE_ETH_FDIR_FILTER_INPUT_SET_SELECT:
2266                 ret = i40e_fdir_filter_inset_select(pf,
2267                                 &(info->info.input_set_conf));
2268                 break;
2269         default:
2270                 PMD_DRV_LOG(ERR, "FD filter info type (%d) not supported",
2271                             info->info_type);
2272                 return -EINVAL;
2273         }
2274
2275         return ret;
2276 }
2277
2278 /*
2279  * i40e_fdir_ctrl_func - deal with all operations on flow director.
2280  * @pf: board private structure
2281  * @filter_op:operation will be taken.
2282  * @arg: a pointer to specific structure corresponding to the filter_op
2283  */
2284 int
2285 i40e_fdir_ctrl_func(struct rte_eth_dev *dev,
2286                        enum rte_filter_op filter_op,
2287                        void *arg)
2288 {
2289         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2290         int ret = 0;
2291
2292         if ((pf->flags & I40E_FLAG_FDIR) == 0)
2293                 return -ENOTSUP;
2294
2295         if (filter_op == RTE_ETH_FILTER_NOP)
2296                 return 0;
2297
2298         if (arg == NULL && filter_op != RTE_ETH_FILTER_FLUSH)
2299                 return -EINVAL;
2300
2301         switch (filter_op) {
2302         case RTE_ETH_FILTER_ADD:
2303                 ret = i40e_add_del_fdir_filter(dev,
2304                         (struct rte_eth_fdir_filter *)arg,
2305                         TRUE);
2306                 break;
2307         case RTE_ETH_FILTER_DELETE:
2308                 ret = i40e_add_del_fdir_filter(dev,
2309                         (struct rte_eth_fdir_filter *)arg,
2310                         FALSE);
2311                 break;
2312         case RTE_ETH_FILTER_FLUSH:
2313                 ret = i40e_fdir_flush(dev);
2314                 break;
2315         case RTE_ETH_FILTER_INFO:
2316                 i40e_fdir_info_get(dev, (struct rte_eth_fdir_info *)arg);
2317                 break;
2318         case RTE_ETH_FILTER_SET:
2319                 ret = i40e_fdir_filter_set(dev,
2320                         (struct rte_eth_fdir_filter_info *)arg);
2321                 break;
2322         case RTE_ETH_FILTER_STATS:
2323                 i40e_fdir_stats_get(dev, (struct rte_eth_fdir_stats *)arg);
2324                 break;
2325         default:
2326                 PMD_DRV_LOG(ERR, "unknown operation %u.", filter_op);
2327                 ret = -EINVAL;
2328                 break;
2329         }
2330         return ret;
2331 }
2332
2333 /* Restore flow director filter */
2334 void
2335 i40e_fdir_filter_restore(struct i40e_pf *pf)
2336 {
2337         struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(pf->main_vsi);
2338         struct i40e_fdir_filter_list *fdir_list = &pf->fdir.fdir_list;
2339         struct i40e_fdir_filter *f;
2340         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2341         uint32_t fdstat;
2342         uint32_t guarant_cnt;  /**< Number of filters in guaranteed spaces. */
2343         uint32_t best_cnt;     /**< Number of filters in best effort spaces. */
2344
2345         TAILQ_FOREACH(f, fdir_list, rules)
2346                 i40e_flow_add_del_fdir_filter(dev, &f->fdir, TRUE);
2347
2348         fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT);
2349         guarant_cnt =
2350                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2351                            I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2352         best_cnt =
2353                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2354                            I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2355
2356         PMD_DRV_LOG(INFO, "FDIR: Guarant count: %d,  Best count: %d",
2357                     guarant_cnt, best_cnt);
2358 }