drivers/net/i40e/i40e_fdir.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_ether.h>
  14 #include <rte_ethdev_driver.h>
  15 #include <rte_log.h>
  16 #include <rte_memzone.h>
  17 #include <rte_malloc.h>
  18 #include <rte_arp.h>
  19 #include <rte_ip.h>
  20 #include <rte_udp.h>
  21 #include <rte_tcp.h>
  22 #include <rte_sctp.h>
  23 #include <rte_hash_crc.h>
  24
  25 #include "i40e_logs.h"
  26 #include "base/i40e_type.h"
  27 #include "base/i40e_prototype.h"
  28 #include "i40e_ethdev.h"
  29 #include "i40e_rxtx.h"
  30
  31 #define I40E_FDIR_MZ_NAME          "FDIR_MEMZONE"
  32 #ifndef IPV6_ADDR_LEN
  33 #define IPV6_ADDR_LEN              16
  34 #endif
  35
  36 #ifndef IPPROTO_L2TP
  37 #define IPPROTO_L2TP              115
  38 #endif
  39
  40 #define I40E_FDIR_PKT_LEN                   512
  41 #define I40E_FDIR_IP_DEFAULT_LEN            420
  42 #define I40E_FDIR_IP_DEFAULT_TTL            0x40
  43 #define I40E_FDIR_IP_DEFAULT_VERSION_IHL    0x45
  44 #define I40E_FDIR_TCP_DEFAULT_DATAOFF       0x50
  45 #define I40E_FDIR_IPv6_DEFAULT_VTC_FLOW     0x60000000
  46
  47 #define I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS   0xFF
  48 #define I40E_FDIR_IPv6_PAYLOAD_LEN          380
  49 #define I40E_FDIR_UDP_DEFAULT_LEN           400
  50 #define I40E_FDIR_GTP_DEFAULT_LEN           384
  51 #define I40E_FDIR_INNER_IP_DEFAULT_LEN      384
  52 #define I40E_FDIR_INNER_IPV6_DEFAULT_LEN    344
  53
  54 #define I40E_FDIR_GTPC_DST_PORT             2123
  55 #define I40E_FDIR_GTPU_DST_PORT             2152
  56 #define I40E_FDIR_GTP_VER_FLAG_0X30         0x30
  57 #define I40E_FDIR_GTP_VER_FLAG_0X32         0x32
  58 #define I40E_FDIR_GTP_MSG_TYPE_0X01         0x01
  59 #define I40E_FDIR_GTP_MSG_TYPE_0XFF         0xFF
  60
  61 #define I40E_FDIR_ESP_DST_PORT              4500
  62
  63 /* Wait time for fdir filter programming */
  64 #define I40E_FDIR_MAX_WAIT_US 10000
  65
  66 /* Wait count and interval for fdir filter flush */
  67 #define I40E_FDIR_FLUSH_RETRY       50
  68 #define I40E_FDIR_FLUSH_INTERVAL_MS 5
  69
  70 #define I40E_COUNTER_PF           2
  71 /* Statistic counter index for one pf */
  72 #define I40E_COUNTER_INDEX_FDIR(pf_id)   (0 + (pf_id) * I40E_COUNTER_PF)
  73
  74 #define I40E_FDIR_FLOWS ( \
  75         (1ULL << RTE_ETH_FLOW_FRAG_IPV4) | \
  76         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \
  77         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \
  78         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \
  79         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \
  80         (1ULL << RTE_ETH_FLOW_FRAG_IPV6) | \
  81         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \
  82         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \
  83         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \
  84         (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \
  85         (1ULL << RTE_ETH_FLOW_L2_PAYLOAD))
  86
  87 static int i40e_fdir_filter_programming(struct i40e_pf *pf,
  88                         enum i40e_filter_pctype pctype,
  89                         const struct rte_eth_fdir_filter *filter,
  90                         bool add);
  91 static int i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input,
  92                          struct i40e_fdir_filter *filter);
  93 static struct i40e_fdir_filter *
  94 i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info,
  95                         const struct i40e_fdir_input *input);
  96 static int i40e_sw_fdir_filter_insert(struct i40e_pf *pf,
  97                                    struct i40e_fdir_filter *filter);
  98 static int
  99 i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
 100                                   enum i40e_filter_pctype pctype,
 101                                   const struct i40e_fdir_filter_conf *filter,
 102                                   bool add);
 103
 104 static int
 105 i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq)
 106 {
 107         struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
 108         struct i40e_hmc_obj_rxq rx_ctx;
 109         int err = I40E_SUCCESS;
 110
 111         memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq));
 112         /* Init the RX queue in hardware */
 113         rx_ctx.dbuff = I40E_RXBUF_SZ_1024 >> I40E_RXQ_CTX_DBUFF_SHIFT;
 114         rx_ctx.hbuff = 0;
 115         rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
 116         rx_ctx.qlen = rxq->nb_rx_desc;
 117 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
 118         rx_ctx.dsize = 1;
 119 #endif
 120         rx_ctx.dtype = i40e_header_split_none;
 121         rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE;
 122         rx_ctx.rxmax = RTE_ETHER_MAX_LEN;
 123         rx_ctx.tphrdesc_ena = 1;
 124         rx_ctx.tphwdesc_ena = 1;
 125         rx_ctx.tphdata_ena = 1;
 126         rx_ctx.tphhead_ena = 1;
 127         rx_ctx.lrxqthresh = 2;
 128         rx_ctx.crcstrip = 0;
 129         rx_ctx.l2tsel = 1;
 130         rx_ctx.showiv = 0;
 131         rx_ctx.prefena = 1;
 132
 133         err = i40e_clear_lan_rx_queue_context(hw, rxq->reg_idx);
 134         if (err != I40E_SUCCESS) {
 135                 PMD_DRV_LOG(ERR, "Failed to clear FDIR RX queue context.");
 136                 return err;
 137         }
 138         err = i40e_set_lan_rx_queue_context(hw, rxq->reg_idx, &rx_ctx);
 139         if (err != I40E_SUCCESS) {
 140                 PMD_DRV_LOG(ERR, "Failed to set FDIR RX queue context.");
 141                 return err;
 142         }
 143         rxq->qrx_tail = hw->hw_addr +
 144                 I40E_QRX_TAIL(rxq->vsi->base_queue);
 145
 146         rte_wmb();
 147         /* Init the RX tail regieter. */
 148         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 149
 150         return err;
 151 }
 152
 153 /*
 154  * i40e_fdir_setup - reserve and initialize the Flow Director resources
 155  * @pf: board private structure
 156  */
 157 int
 158 i40e_fdir_setup(struct i40e_pf *pf)
 159 {
 160         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 161         struct i40e_vsi *vsi;
 162         int err = I40E_SUCCESS;
 163         char z_name[RTE_MEMZONE_NAMESIZE];
 164         const struct rte_memzone *mz = NULL;
 165         struct rte_eth_dev *eth_dev = pf->adapter->eth_dev;
 166
 167         if ((pf->flags & I40E_FLAG_FDIR) == 0) {
 168                 PMD_INIT_LOG(ERR, "HW doesn't support FDIR");
 169                 return I40E_NOT_SUPPORTED;
 170         }
 171
 172         PMD_DRV_LOG(INFO, "FDIR HW Capabilities: num_filters_guaranteed = %u,"
 173                         " num_filters_best_effort = %u.",
 174                         hw->func_caps.fd_filters_guaranteed,
 175                         hw->func_caps.fd_filters_best_effort);
 176
 177         vsi = pf->fdir.fdir_vsi;
 178         if (vsi) {
 179                 PMD_DRV_LOG(INFO, "FDIR initialization has been done.");
 180                 return I40E_SUCCESS;
 181         }
 182         /* make new FDIR VSI */
 183         vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->main_vsi, 0);
 184         if (!vsi) {
 185                 PMD_DRV_LOG(ERR, "Couldn't create FDIR VSI.");
 186                 return I40E_ERR_NO_AVAILABLE_VSI;
 187         }
 188         pf->fdir.fdir_vsi = vsi;
 189
 190         /*Fdir tx queue setup*/
 191         err = i40e_fdir_setup_tx_resources(pf);
 192         if (err) {
 193                 PMD_DRV_LOG(ERR, "Failed to setup FDIR TX resources.");
 194                 goto fail_setup_tx;
 195         }
 196
 197         /*Fdir rx queue setup*/
 198         err = i40e_fdir_setup_rx_resources(pf);
 199         if (err) {
 200                 PMD_DRV_LOG(ERR, "Failed to setup FDIR RX resources.");
 201                 goto fail_setup_rx;
 202         }
 203
 204         err = i40e_tx_queue_init(pf->fdir.txq);
 205         if (err) {
 206                 PMD_DRV_LOG(ERR, "Failed to do FDIR TX initialization.");
 207                 goto fail_mem;
 208         }
 209
 210         /* need switch on before dev start*/
 211         err = i40e_switch_tx_queue(hw, vsi->base_queue, TRUE);
 212         if (err) {
 213                 PMD_DRV_LOG(ERR, "Failed to do fdir TX switch on.");
 214                 goto fail_mem;
 215         }
 216
 217         /* Init the rx queue in hardware */
 218         err = i40e_fdir_rx_queue_init(pf->fdir.rxq);
 219         if (err) {
 220                 PMD_DRV_LOG(ERR, "Failed to do FDIR RX initialization.");
 221                 goto fail_mem;
 222         }
 223
 224         /* switch on rx queue */
 225         err = i40e_switch_rx_queue(hw, vsi->base_queue, TRUE);
 226         if (err) {
 227                 PMD_DRV_LOG(ERR, "Failed to do FDIR RX switch on.");
 228                 goto fail_mem;
 229         }
 230
 231         /* reserve memory for the fdir programming packet */
 232         snprintf(z_name, sizeof(z_name), "%s_%s_%d",
 233                         eth_dev->device->driver->name,
 234                         I40E_FDIR_MZ_NAME,
 235                         eth_dev->data->port_id);
 236         mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN, SOCKET_ID_ANY);
 237         if (!mz) {
 238                 PMD_DRV_LOG(ERR, "Cannot init memzone for "
 239                                  "flow director program packet.");
 240                 err = I40E_ERR_NO_MEMORY;
 241                 goto fail_mem;
 242         }
 243         pf->fdir.prg_pkt = mz->addr;
 244         pf->fdir.dma_addr = mz->iova;
 245
 246         pf->fdir.match_counter_index = I40E_COUNTER_INDEX_FDIR(hw->pf_id);
 247         PMD_DRV_LOG(INFO, "FDIR setup successfully, with programming queue %u.",
 248                     vsi->base_queue);
 249         return I40E_SUCCESS;
 250
 251 fail_mem:
 252         i40e_dev_rx_queue_release(pf->fdir.rxq);
 253         pf->fdir.rxq = NULL;
 254 fail_setup_rx:
 255         i40e_dev_tx_queue_release(pf->fdir.txq);
 256         pf->fdir.txq = NULL;
 257 fail_setup_tx:
 258         i40e_vsi_release(vsi);
 259         pf->fdir.fdir_vsi = NULL;
 260         return err;
 261 }
 262
 263 /*
 264  * i40e_fdir_teardown - release the Flow Director resources
 265  * @pf: board private structure
 266  */
 267 void
 268 i40e_fdir_teardown(struct i40e_pf *pf)
 269 {
 270         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 271         struct i40e_vsi *vsi;
 272         struct rte_eth_dev *dev = pf->adapter->eth_dev;
 273
 274         vsi = pf->fdir.fdir_vsi;
 275         if (!vsi)
 276                 return;
 277         int err = i40e_switch_tx_queue(hw, vsi->base_queue, FALSE);
 278         if (err)
 279                 PMD_DRV_LOG(DEBUG, "Failed to do FDIR TX switch off");
 280         err = i40e_switch_rx_queue(hw, vsi->base_queue, FALSE);
 281         if (err)
 282                 PMD_DRV_LOG(DEBUG, "Failed to do FDIR RX switch off");
 283         i40e_dev_rx_queue_release(pf->fdir.rxq);
 284         rte_eth_dma_zone_free(dev, "fdir_rx_ring", pf->fdir.rxq->queue_id);
 285         pf->fdir.rxq = NULL;
 286         i40e_dev_tx_queue_release(pf->fdir.txq);
 287         rte_eth_dma_zone_free(dev, "fdir_tx_ring", pf->fdir.txq->queue_id);
 288         pf->fdir.txq = NULL;
 289         i40e_vsi_release(vsi);
 290         pf->fdir.fdir_vsi = NULL;
 291 }
 292
 293 /* check whether the flow director table in empty */
 294 static inline int
 295 i40e_fdir_empty(struct i40e_hw *hw)
 296 {
 297         uint32_t guarant_cnt, best_cnt;
 298
 299         guarant_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
 300                                  I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
 301                                  I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
 302         best_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
 303                               I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
 304                               I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
 305         if (best_cnt + guarant_cnt > 0)
 306                 return -1;
 307
 308         return 0;
 309 }
 310
 311 /*
 312  * Initialize the configuration about bytes stream extracted as flexible payload
 313  * and mask setting
 314  */
 315 static inline void
 316 i40e_init_flx_pld(struct i40e_pf *pf)
 317 {
 318         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 319         uint8_t pctype;
 320         int i, index;
 321         uint16_t flow_type;
 322
 323         /*
 324          * Define the bytes stream extracted as flexible payload in
 325          * field vector. By default, select 8 words from the beginning
 326          * of payload as flexible payload.
 327          */
 328         for (i = I40E_FLXPLD_L2_IDX; i < I40E_MAX_FLXPLD_LAYER; i++) {
 329                 index = i * I40E_MAX_FLXPLD_FIED;
 330                 pf->fdir.flex_set[index].src_offset = 0;
 331                 pf->fdir.flex_set[index].size = I40E_FDIR_MAX_FLEXWORD_NUM;
 332                 pf->fdir.flex_set[index].dst_offset = 0;
 333                 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(index), 0x0000C900);
 334                 I40E_WRITE_REG(hw,
 335                         I40E_PRTQF_FLX_PIT(index + 1), 0x0000FC29);/*non-used*/
 336                 I40E_WRITE_REG(hw,
 337                         I40E_PRTQF_FLX_PIT(index + 2), 0x0000FC2A);/*non-used*/
 338         }
 339
 340         /* initialize the masks */
 341         for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
 342              pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) {
 343                 flow_type = i40e_pctype_to_flowtype(pf->adapter, pctype);
 344
 345                 if (flow_type == RTE_ETH_FLOW_UNKNOWN)
 346                         continue;
 347                 pf->fdir.flex_mask[pctype].word_mask = 0;
 348                 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), 0);
 349                 for (i = 0; i < I40E_FDIR_BITMASK_NUM_WORD; i++) {
 350                         pf->fdir.flex_mask[pctype].bitmask[i].offset = 0;
 351                         pf->fdir.flex_mask[pctype].bitmask[i].mask = 0;
 352                         i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), 0);
 353                 }
 354         }
 355 }
 356
 357 #define I40E_VALIDATE_FLEX_PIT(flex_pit1, flex_pit2) do { \
 358         if ((flex_pit2).src_offset < \
 359                 (flex_pit1).src_offset + (flex_pit1).size) { \
 360                 PMD_DRV_LOG(ERR, "src_offset should be not" \
 361                         " less than than previous offset" \
 362                         " + previous FSIZE."); \
 363                 return -EINVAL; \
 364         } \
 365 } while (0)
 366
 367 /*
 368  * i40e_srcoff_to_flx_pit - transform the src_offset into flex_pit structure,
 369  * and the flex_pit will be sorted by it's src_offset value
 370  */
 371 static inline uint16_t
 372 i40e_srcoff_to_flx_pit(const uint16_t *src_offset,
 373                         struct i40e_fdir_flex_pit *flex_pit)
 374 {
 375         uint16_t src_tmp, size, num = 0;
 376         uint16_t i, k, j = 0;
 377
 378         while (j < I40E_FDIR_MAX_FLEX_LEN) {
 379                 size = 1;
 380                 for (; j < I40E_FDIR_MAX_FLEX_LEN - 1; j++) {
 381                         if (src_offset[j + 1] == src_offset[j] + 1)
 382                                 size++;
 383                         else
 384                                 break;
 385                 }
 386                 src_tmp = src_offset[j] + 1 - size;
 387                 /* the flex_pit need to be sort by src_offset */
 388                 for (i = 0; i < num; i++) {
 389                         if (src_tmp < flex_pit[i].src_offset)
 390                                 break;
 391                 }
 392                 /* if insert required, move backward */
 393                 for (k = num; k > i; k--)
 394                         flex_pit[k] = flex_pit[k - 1];
 395                 /* insert */
 396                 flex_pit[i].dst_offset = j + 1 - size;
 397                 flex_pit[i].src_offset = src_tmp;
 398                 flex_pit[i].size = size;
 399                 j++;
 400                 num++;
 401         }
 402         return num;
 403 }
 404
 405 /* i40e_check_fdir_flex_payload -check flex payload configuration arguments */
 406 static inline int
 407 i40e_check_fdir_flex_payload(const struct rte_eth_flex_payload_cfg *flex_cfg)
 408 {
 409         struct i40e_fdir_flex_pit flex_pit[I40E_FDIR_MAX_FLEX_LEN];
 410         uint16_t num, i;
 411
 412         for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i++) {
 413                 if (flex_cfg->src_offset[i] >= I40E_MAX_FLX_SOURCE_OFF) {
 414                         PMD_DRV_LOG(ERR, "exceeds maxmial payload limit.");
 415                         return -EINVAL;
 416                 }
 417         }
 418
 419         memset(flex_pit, 0, sizeof(flex_pit));
 420         num = i40e_srcoff_to_flx_pit(flex_cfg->src_offset, flex_pit);
 421         if (num > I40E_MAX_FLXPLD_FIED) {
 422                 PMD_DRV_LOG(ERR, "exceeds maxmial number of flex fields.");
 423                 return -EINVAL;
 424         }
 425         for (i = 0; i < num; i++) {
 426                 if (flex_pit[i].size & 0x01 || flex_pit[i].dst_offset & 0x01 ||
 427                         flex_pit[i].src_offset & 0x01) {
 428                         PMD_DRV_LOG(ERR, "flexpayload should be measured"
 429                                 " in word");
 430                         return -EINVAL;
 431                 }
 432                 if (i != num - 1)
 433                         I40E_VALIDATE_FLEX_PIT(flex_pit[i], flex_pit[i + 1]);
 434         }
 435         return 0;
 436 }
 437
 438 /*
 439  * i40e_check_fdir_flex_conf -check if the flex payload and mask configuration
 440  * arguments are valid
 441  */
 442 static int
 443 i40e_check_fdir_flex_conf(const struct i40e_adapter *adapter,
 444                           const struct rte_eth_fdir_flex_conf *conf)
 445 {
 446         const struct rte_eth_flex_payload_cfg *flex_cfg;
 447         const struct rte_eth_fdir_flex_mask *flex_mask;
 448         uint16_t mask_tmp;
 449         uint8_t nb_bitmask;
 450         uint16_t i, j;
 451         int ret = 0;
 452         enum i40e_filter_pctype pctype;
 453
 454         if (conf == NULL) {
 455                 PMD_DRV_LOG(INFO, "NULL pointer.");
 456                 return -EINVAL;
 457         }
 458         /* check flexible payload setting configuration */
 459         if (conf->nb_payloads > RTE_ETH_L4_PAYLOAD) {
 460                 PMD_DRV_LOG(ERR, "invalid number of payload setting.");
 461                 return -EINVAL;
 462         }
 463         for (i = 0; i < conf->nb_payloads; i++) {
 464                 flex_cfg = &conf->flex_set[i];
 465                 if (flex_cfg->type > RTE_ETH_L4_PAYLOAD) {
 466                         PMD_DRV_LOG(ERR, "invalid payload type.");
 467                         return -EINVAL;
 468                 }
 469                 ret = i40e_check_fdir_flex_payload(flex_cfg);
 470                 if (ret < 0) {
 471                         PMD_DRV_LOG(ERR, "invalid flex payload arguments.");
 472                         return -EINVAL;
 473                 }
 474         }
 475
 476         /* check flex mask setting configuration */
 477         if (conf->nb_flexmasks >= RTE_ETH_FLOW_MAX) {
 478                 PMD_DRV_LOG(ERR, "invalid number of flex masks.");
 479                 return -EINVAL;
 480         }
 481         for (i = 0; i < conf->nb_flexmasks; i++) {
 482                 flex_mask = &conf->flex_mask[i];
 483                 pctype = i40e_flowtype_to_pctype(adapter, flex_mask->flow_type);
 484                 if (pctype == I40E_FILTER_PCTYPE_INVALID) {
 485                         PMD_DRV_LOG(WARNING, "invalid flow type.");
 486                         return -EINVAL;
 487                 }
 488                 nb_bitmask = 0;
 489                 for (j = 0; j < I40E_FDIR_MAX_FLEX_LEN; j += sizeof(uint16_t)) {
 490                         mask_tmp = I40E_WORD(flex_mask->mask[j],
 491                                              flex_mask->mask[j + 1]);
 492                         if (mask_tmp != 0x0 && mask_tmp != UINT16_MAX) {
 493                                 nb_bitmask++;
 494                                 if (nb_bitmask > I40E_FDIR_BITMASK_NUM_WORD) {
 495                                         PMD_DRV_LOG(ERR, " exceed maximal"
 496                                                 " number of bitmasks.");
 497                                         return -EINVAL;
 498                                 }
 499                         }
 500                 }
 501         }
 502         return 0;
 503 }
 504
 505 /*
 506  * i40e_set_flx_pld_cfg -configure the rule how bytes stream is extracted as flexible payload
 507  * @pf: board private structure
 508  * @cfg: the rule how bytes stream is extracted as flexible payload
 509  */
 510 static void
 511 i40e_set_flx_pld_cfg(struct i40e_pf *pf,
 512                          const struct rte_eth_flex_payload_cfg *cfg)
 513 {
 514         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 515         struct i40e_fdir_flex_pit flex_pit[I40E_MAX_FLXPLD_FIED];
 516         uint32_t flx_pit, flx_ort;
 517         uint16_t num, min_next_off;  /* in words */
 518         uint8_t field_idx = 0;
 519         uint8_t layer_idx = 0;
 520         uint16_t i;
 521
 522         if (cfg->type == RTE_ETH_L2_PAYLOAD)
 523                 layer_idx = I40E_FLXPLD_L2_IDX;
 524         else if (cfg->type == RTE_ETH_L3_PAYLOAD)
 525                 layer_idx = I40E_FLXPLD_L3_IDX;
 526         else if (cfg->type == RTE_ETH_L4_PAYLOAD)
 527                 layer_idx = I40E_FLXPLD_L4_IDX;
 528
 529         memset(flex_pit, 0, sizeof(flex_pit));
 530         num = RTE_MIN(i40e_srcoff_to_flx_pit(cfg->src_offset, flex_pit),
 531                       RTE_DIM(flex_pit));
 532
 533         if (num) {
 534                 flx_ort = (1 << I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT) |
 535                           (num << I40E_GLQF_ORT_FIELD_CNT_SHIFT) |
 536                           (layer_idx * I40E_MAX_FLXPLD_FIED);
 537                 I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33 + layer_idx), flx_ort);
 538         }
 539
 540         for (i = 0; i < num; i++) {
 541                 field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i;
 542                 /* record the info in fdir structure */
 543                 pf->fdir.flex_set[field_idx].src_offset =
 544                         flex_pit[i].src_offset / sizeof(uint16_t);
 545                 pf->fdir.flex_set[field_idx].size =
 546                         flex_pit[i].size / sizeof(uint16_t);
 547                 pf->fdir.flex_set[field_idx].dst_offset =
 548                         flex_pit[i].dst_offset / sizeof(uint16_t);
 549                 flx_pit = MK_FLX_PIT(pf->fdir.flex_set[field_idx].src_offset,
 550                                 pf->fdir.flex_set[field_idx].size,
 551                                 pf->fdir.flex_set[field_idx].dst_offset);
 552
 553                 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit);
 554         }
 555         min_next_off = pf->fdir.flex_set[field_idx].src_offset +
 556                                 pf->fdir.flex_set[field_idx].size;
 557
 558         for (; i < I40E_MAX_FLXPLD_FIED; i++) {
 559                 /* set the non-used register obeying register's constrain */
 560                 flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE,
 561                            NONUSE_FLX_PIT_DEST_OFF);
 562                 I40E_WRITE_REG(hw,
 563                         I40E_PRTQF_FLX_PIT(layer_idx * I40E_MAX_FLXPLD_FIED + i),
 564                         flx_pit);
 565                 min_next_off++;
 566         }
 567 }
 568
 569 /*
 570  * i40e_set_flex_mask_on_pctype - configure the mask on flexible payload
 571  * @pf: board private structure
 572  * @pctype: packet classify type
 573  * @flex_masks: mask for flexible payload
 574  */
 575 static void
 576 i40e_set_flex_mask_on_pctype(struct i40e_pf *pf,
 577                 enum i40e_filter_pctype pctype,
 578                 const struct rte_eth_fdir_flex_mask *mask_cfg)
 579 {
 580         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 581         struct i40e_fdir_flex_mask *flex_mask;
 582         uint32_t flxinset, fd_mask;
 583         uint16_t mask_tmp;
 584         uint8_t i, nb_bitmask = 0;
 585
 586         flex_mask = &pf->fdir.flex_mask[pctype];
 587         memset(flex_mask, 0, sizeof(struct i40e_fdir_flex_mask));
 588         for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i += sizeof(uint16_t)) {
 589                 mask_tmp = I40E_WORD(mask_cfg->mask[i], mask_cfg->mask[i + 1]);
 590                 if (mask_tmp != 0x0) {
 591                         flex_mask->word_mask |=
 592                                 I40E_FLEX_WORD_MASK(i / sizeof(uint16_t));
 593                         if (mask_tmp != UINT16_MAX) {
 594                                 /* set bit mask */
 595                                 flex_mask->bitmask[nb_bitmask].mask = ~mask_tmp;
 596                                 flex_mask->bitmask[nb_bitmask].offset =
 597                                         i / sizeof(uint16_t);
 598                                 nb_bitmask++;
 599                         }
 600                 }
 601         }
 602         /* write mask to hw */
 603         flxinset = (flex_mask->word_mask <<
 604                 I40E_PRTQF_FD_FLXINSET_INSET_SHIFT) &
 605                 I40E_PRTQF_FD_FLXINSET_INSET_MASK;
 606         i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), flxinset);
 607
 608         for (i = 0; i < nb_bitmask; i++) {
 609                 fd_mask = (flex_mask->bitmask[i].mask <<
 610                         I40E_PRTQF_FD_MSK_MASK_SHIFT) &
 611                         I40E_PRTQF_FD_MSK_MASK_MASK;
 612                 fd_mask |= ((flex_mask->bitmask[i].offset +
 613                         I40E_FLX_OFFSET_IN_FIELD_VECTOR) <<
 614                         I40E_PRTQF_FD_MSK_OFFSET_SHIFT) &
 615                         I40E_PRTQF_FD_MSK_OFFSET_MASK;
 616                 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), fd_mask);
 617         }
 618 }
 619
 620 /*
 621  * Enable/disable flow director RX processing in vector routines.
 622  */
 623 void
 624 i40e_fdir_rx_proc_enable(struct rte_eth_dev *dev, bool on)
 625 {
 626         int32_t i;
 627
 628         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 629                 struct i40e_rx_queue *rxq = dev->data->rx_queues[i];
 630                 if (!rxq)
 631                         continue;
 632                 rxq->fdir_enabled = on;
 633         }
 634         PMD_DRV_LOG(DEBUG, "Flow Director processing on RX set to %d", on);
 635 }
 636
 637 /*
 638  * Configure flow director related setting
 639  */
 640 int
 641 i40e_fdir_configure(struct rte_eth_dev *dev)
 642 {
 643         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 644         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 645         struct rte_eth_fdir_flex_conf *conf;
 646         enum i40e_filter_pctype pctype;
 647         uint32_t val;
 648         uint8_t i;
 649         int ret = 0;
 650
 651         /*
 652         * configuration need to be done before
 653         * flow director filters are added
 654         * If filters exist, flush them.
 655         */
 656         if (i40e_fdir_empty(hw) < 0) {
 657                 ret = i40e_fdir_flush(dev);
 658                 if (ret) {
 659                         PMD_DRV_LOG(ERR, "failed to flush fdir table.");
 660                         return ret;
 661                 }
 662         }
 663
 664         /* enable FDIR filter */
 665         val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
 666         val |= I40E_PFQF_CTL_0_FD_ENA_MASK;
 667         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, val);
 668
 669         i40e_init_flx_pld(pf); /* set flex config to default value */
 670
 671         conf = &dev->data->dev_conf.fdir_conf.flex_conf;
 672         ret = i40e_check_fdir_flex_conf(pf->adapter, conf);
 673         if (ret < 0) {
 674                 PMD_DRV_LOG(ERR, " invalid configuration arguments.");
 675                 return -EINVAL;
 676         }
 677
 678         if (!pf->support_multi_driver) {
 679                 /* configure flex payload */
 680                 for (i = 0; i < conf->nb_payloads; i++)
 681                         i40e_set_flx_pld_cfg(pf, &conf->flex_set[i]);
 682                 /* configure flex mask*/
 683                 for (i = 0; i < conf->nb_flexmasks; i++) {
 684                         if (hw->mac.type == I40E_MAC_X722) {
 685                                 /* get pctype value in fd pctype register */
 686                                 pctype = (enum i40e_filter_pctype)
 687                                           i40e_read_rx_ctl(hw,
 688                                                 I40E_GLQF_FD_PCTYPES(
 689                                                 (int)i40e_flowtype_to_pctype(
 690                                                 pf->adapter,
 691                                                 conf->flex_mask[i].flow_type)));
 692                         } else {
 693                                 pctype = i40e_flowtype_to_pctype(pf->adapter,
 694                                                   conf->flex_mask[i].flow_type);
 695                         }
 696
 697                         i40e_set_flex_mask_on_pctype(pf, pctype,
 698                                                      &conf->flex_mask[i]);
 699                 }
 700         } else {
 701                 PMD_DRV_LOG(ERR, "Not support flexible payload.");
 702         }
 703
 704         /* Enable FDIR processing in RX routines */
 705         i40e_fdir_rx_proc_enable(dev, 1);
 706
 707         return ret;
 708 }
 709
 710 static inline int
 711 i40e_fdir_fill_eth_ip_head(const struct rte_eth_fdir_input *fdir_input,
 712                            unsigned char *raw_pkt,
 713                            bool vlan)
 714 {
 715         static uint8_t vlan_frame[] = {0x81, 0, 0, 0};
 716         uint16_t *ether_type;
 717         uint8_t len = 2 * sizeof(struct rte_ether_addr);
 718         struct rte_ipv4_hdr *ip;
 719         struct rte_ipv6_hdr *ip6;
 720         static const uint8_t next_proto[] = {
 721                 [RTE_ETH_FLOW_FRAG_IPV4] = IPPROTO_IP,
 722                 [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = IPPROTO_TCP,
 723                 [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = IPPROTO_UDP,
 724                 [RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] = IPPROTO_SCTP,
 725                 [RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] = IPPROTO_IP,
 726                 [RTE_ETH_FLOW_FRAG_IPV6] = IPPROTO_NONE,
 727                 [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = IPPROTO_TCP,
 728                 [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = IPPROTO_UDP,
 729                 [RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] = IPPROTO_SCTP,
 730                 [RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] = IPPROTO_NONE,
 731         };
 732
 733         raw_pkt += 2 * sizeof(struct rte_ether_addr);
 734         if (vlan && fdir_input->flow_ext.vlan_tci) {
 735                 rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame));
 736                 rte_memcpy(raw_pkt + sizeof(uint16_t),
 737                            &fdir_input->flow_ext.vlan_tci,
 738                            sizeof(uint16_t));
 739                 raw_pkt += sizeof(vlan_frame);
 740                 len += sizeof(vlan_frame);
 741         }
 742         ether_type = (uint16_t *)raw_pkt;
 743         raw_pkt += sizeof(uint16_t);
 744         len += sizeof(uint16_t);
 745
 746         switch (fdir_input->flow_type) {
 747         case RTE_ETH_FLOW_L2_PAYLOAD:
 748                 *ether_type = fdir_input->flow.l2_flow.ether_type;
 749                 break;
 750         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 751         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 752         case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP:
 753         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 754         case RTE_ETH_FLOW_FRAG_IPV4:
 755                 ip = (struct rte_ipv4_hdr *)raw_pkt;
 756
 757                 *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
 758                 ip->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL;
 759                 /* set len to by default */
 760                 ip->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN);
 761                 ip->next_proto_id = fdir_input->flow.ip4_flow.proto ?
 762                                         fdir_input->flow.ip4_flow.proto :
 763                                         next_proto[fdir_input->flow_type];
 764                 ip->time_to_live = fdir_input->flow.ip4_flow.ttl ?
 765                                         fdir_input->flow.ip4_flow.ttl :
 766                                         I40E_FDIR_IP_DEFAULT_TTL;
 767                 ip->type_of_service = fdir_input->flow.ip4_flow.tos;
 768                 /*
 769                  * The source and destination fields in the transmitted packet
 770                  * need to be presented in a reversed order with respect
 771                  * to the expected received packets.
 772                  */
 773                 ip->src_addr = fdir_input->flow.ip4_flow.dst_ip;
 774                 ip->dst_addr = fdir_input->flow.ip4_flow.src_ip;
 775                 len += sizeof(struct rte_ipv4_hdr);
 776                 break;
 777         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 778         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 779         case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP:
 780         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 781         case RTE_ETH_FLOW_FRAG_IPV6:
 782                 ip6 = (struct rte_ipv6_hdr *)raw_pkt;
 783
 784                 *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
 785                 ip6->vtc_flow =
 786                         rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
 787                                          (fdir_input->flow.ipv6_flow.tc <<
 788                                           I40E_FDIR_IPv6_TC_OFFSET));
 789                 ip6->payload_len =
 790                         rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
 791                 ip6->proto = fdir_input->flow.ipv6_flow.proto ?
 792                                         fdir_input->flow.ipv6_flow.proto :
 793                                         next_proto[fdir_input->flow_type];
 794                 ip6->hop_limits = fdir_input->flow.ipv6_flow.hop_limits ?
 795                                         fdir_input->flow.ipv6_flow.hop_limits :
 796                                         I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
 797                 /*
 798                  * The source and destination fields in the transmitted packet
 799                  * need to be presented in a reversed order with respect
 800                  * to the expected received packets.
 801                  */
 802                 rte_memcpy(&(ip6->src_addr),
 803                            &(fdir_input->flow.ipv6_flow.dst_ip),
 804                            IPV6_ADDR_LEN);
 805                 rte_memcpy(&(ip6->dst_addr),
 806                            &(fdir_input->flow.ipv6_flow.src_ip),
 807                            IPV6_ADDR_LEN);
 808                 len += sizeof(struct rte_ipv6_hdr);
 809                 break;
 810         default:
 811                 PMD_DRV_LOG(ERR, "unknown flow type %u.",
 812                             fdir_input->flow_type);
 813                 return -1;
 814         }
 815         return len;
 816 }
 817
 818
 819 /*
 820  * i40e_fdir_construct_pkt - construct packet based on fields in input
 821  * @pf: board private structure
 822  * @fdir_input: input set of the flow director entry
 823  * @raw_pkt: a packet to be constructed
 824  */
 825 static int
 826 i40e_fdir_construct_pkt(struct i40e_pf *pf,
 827                              const struct rte_eth_fdir_input *fdir_input,
 828                              unsigned char *raw_pkt)
 829 {
 830         unsigned char *payload, *ptr;
 831         struct rte_udp_hdr *udp;
 832         struct rte_tcp_hdr *tcp;
 833         struct rte_sctp_hdr *sctp;
 834         uint8_t size, dst = 0;
 835         uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/
 836         int len;
 837
 838         /* fill the ethernet and IP head */
 839         len = i40e_fdir_fill_eth_ip_head(fdir_input, raw_pkt,
 840                                          !!fdir_input->flow_ext.vlan_tci);
 841         if (len < 0)
 842                 return -EINVAL;
 843
 844         /* fill the L4 head */
 845         switch (fdir_input->flow_type) {
 846         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 847                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
 848                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
 849                 /*
 850                  * The source and destination fields in the transmitted packet
 851                  * need to be presented in a reversed order with respect
 852                  * to the expected received packets.
 853                  */
 854                 udp->src_port = fdir_input->flow.udp4_flow.dst_port;
 855                 udp->dst_port = fdir_input->flow.udp4_flow.src_port;
 856                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
 857                 break;
 858
 859         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 860                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
 861                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
 862                 /*
 863                  * The source and destination fields in the transmitted packet
 864                  * need to be presented in a reversed order with respect
 865                  * to the expected received packets.
 866                  */
 867                 tcp->src_port = fdir_input->flow.tcp4_flow.dst_port;
 868                 tcp->dst_port = fdir_input->flow.tcp4_flow.src_port;
 869                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
 870                 break;
 871
 872         case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP:
 873                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
 874                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
 875                 /*
 876                  * The source and destination fields in the transmitted packet
 877                  * need to be presented in a reversed order with respect
 878                  * to the expected received packets.
 879                  */
 880                 sctp->src_port = fdir_input->flow.sctp4_flow.dst_port;
 881                 sctp->dst_port = fdir_input->flow.sctp4_flow.src_port;
 882                 sctp->tag = fdir_input->flow.sctp4_flow.verify_tag;
 883                 break;
 884
 885         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 886         case RTE_ETH_FLOW_FRAG_IPV4:
 887                 payload = raw_pkt + len;
 888                 set_idx = I40E_FLXPLD_L3_IDX;
 889                 break;
 890
 891         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 892                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
 893                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
 894                 /*
 895                  * The source and destination fields in the transmitted packet
 896                  * need to be presented in a reversed order with respect
 897                  * to the expected received packets.
 898                  */
 899                 udp->src_port = fdir_input->flow.udp6_flow.dst_port;
 900                 udp->dst_port = fdir_input->flow.udp6_flow.src_port;
 901                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
 902                 break;
 903
 904         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 905                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
 906                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
 907                 /*
 908                  * The source and destination fields in the transmitted packet
 909                  * need to be presented in a reversed order with respect
 910                  * to the expected received packets.
 911                  */
 912                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
 913                 tcp->src_port = fdir_input->flow.udp6_flow.dst_port;
 914                 tcp->dst_port = fdir_input->flow.udp6_flow.src_port;
 915                 break;
 916
 917         case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP:
 918                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
 919                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
 920                 /*
 921                  * The source and destination fields in the transmitted packet
 922                  * need to be presented in a reversed order with respect
 923                  * to the expected received packets.
 924                  */
 925                 sctp->src_port = fdir_input->flow.sctp6_flow.dst_port;
 926                 sctp->dst_port = fdir_input->flow.sctp6_flow.src_port;
 927                 sctp->tag = fdir_input->flow.sctp6_flow.verify_tag;
 928                 break;
 929
 930         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 931         case RTE_ETH_FLOW_FRAG_IPV6:
 932                 payload = raw_pkt + len;
 933                 set_idx = I40E_FLXPLD_L3_IDX;
 934                 break;
 935         case RTE_ETH_FLOW_L2_PAYLOAD:
 936                 payload = raw_pkt + len;
 937                 /*
 938                  * ARP packet is a special case on which the payload
 939                  * starts after the whole ARP header
 940                  */
 941                 if (fdir_input->flow.l2_flow.ether_type ==
 942                                 rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
 943                         payload += sizeof(struct rte_arp_hdr);
 944                 set_idx = I40E_FLXPLD_L2_IDX;
 945                 break;
 946         default:
 947                 PMD_DRV_LOG(ERR, "unknown flow type %u.", fdir_input->flow_type);
 948                 return -EINVAL;
 949         }
 950
 951         /* fill the flexbytes to payload */
 952         for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
 953                 pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i;
 954                 size = pf->fdir.flex_set[pit_idx].size;
 955                 if (size == 0)
 956                         continue;
 957                 dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t);
 958                 ptr = payload +
 959                         pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t);
 960                 rte_memcpy(ptr,
 961                                  &fdir_input->flow_ext.flexbytes[dst],
 962                                  size * sizeof(uint16_t));
 963         }
 964
 965         return 0;
 966 }
 967
 968 static struct i40e_customized_pctype *
 969 i40e_flow_fdir_find_customized_pctype(struct i40e_pf *pf, uint8_t pctype)
 970 {
 971         struct i40e_customized_pctype *cus_pctype;
 972         enum i40e_new_pctype i = I40E_CUSTOMIZED_GTPC;
 973
 974         for (; i < I40E_CUSTOMIZED_MAX; i++) {
 975                 cus_pctype = &pf->customized_pctype[i];
 976                 if (pctype == cus_pctype->pctype)
 977                         return cus_pctype;
 978         }
 979         return NULL;
 980 }
 981
 982 static inline int
 983 fill_ip6_head(const struct i40e_fdir_input *fdir_input, unsigned char *raw_pkt,
 984                 uint8_t next_proto, uint8_t len, uint16_t *ether_type)
 985 {
 986         struct rte_ipv6_hdr *ip6;
 987
 988         ip6 = (struct rte_ipv6_hdr *)raw_pkt;
 989
 990         *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
 991         ip6->vtc_flow = rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
 992                 (fdir_input->flow.ipv6_flow.tc << I40E_FDIR_IPv6_TC_OFFSET));
 993         ip6->payload_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
 994         ip6->proto = fdir_input->flow.ipv6_flow.proto ?
 995                 fdir_input->flow.ipv6_flow.proto : next_proto;
 996         ip6->hop_limits = fdir_input->flow.ipv6_flow.hop_limits ?
 997                 fdir_input->flow.ipv6_flow.hop_limits :
 998                 I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
 999         /**
1000          * The source and destination fields in the transmitted packet
1001          * need to be presented in a reversed order with respect
1002          * to the expected received packets.
1003          */
1004         rte_memcpy(&ip6->src_addr, &fdir_input->flow.ipv6_flow.dst_ip,
1005                 IPV6_ADDR_LEN);
1006         rte_memcpy(&ip6->dst_addr, &fdir_input->flow.ipv6_flow.src_ip,
1007                 IPV6_ADDR_LEN);
1008         len += sizeof(struct rte_ipv6_hdr);
1009
1010         return len;
1011 }
1012
1013 static inline int
1014 fill_ip4_head(const struct i40e_fdir_input *fdir_input, unsigned char *raw_pkt,
1015                 uint8_t next_proto, uint8_t len, uint16_t *ether_type)
1016 {
1017         struct rte_ipv4_hdr *ip4;
1018
1019         ip4 = (struct rte_ipv4_hdr *)raw_pkt;
1020
1021         *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1022         ip4->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL;
1023         /* set len to by default */
1024         ip4->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN);
1025         ip4->time_to_live = fdir_input->flow.ip4_flow.ttl ?
1026                 fdir_input->flow.ip4_flow.ttl :
1027                 I40E_FDIR_IP_DEFAULT_TTL;
1028         ip4->type_of_service = fdir_input->flow.ip4_flow.tos;
1029         ip4->next_proto_id = fdir_input->flow.ip4_flow.proto ?
1030                 fdir_input->flow.ip4_flow.proto : next_proto;
1031         /**
1032          * The source and destination fields in the transmitted packet
1033          * need to be presented in a reversed order with respect
1034          * to the expected received packets.
1035          */
1036         ip4->src_addr = fdir_input->flow.ip4_flow.dst_ip;
1037         ip4->dst_addr = fdir_input->flow.ip4_flow.src_ip;
1038         len += sizeof(struct rte_ipv4_hdr);
1039
1040         return len;
1041 }
1042
1043 static inline int
1044 i40e_flow_fdir_fill_eth_ip_head(struct i40e_pf *pf,
1045                                 const struct i40e_fdir_input *fdir_input,
1046                                 unsigned char *raw_pkt,
1047                                 bool vlan)
1048 {
1049         struct i40e_customized_pctype *cus_pctype = NULL;
1050         static uint8_t vlan_frame[] = {0x81, 0, 0, 0};
1051         uint16_t *ether_type;
1052         uint8_t len = 2 * sizeof(struct rte_ether_addr);
1053         uint8_t pctype = fdir_input->pctype;
1054         bool is_customized_pctype = fdir_input->flow_ext.customized_pctype;
1055         static const uint8_t next_proto[] = {
1056                 [I40E_FILTER_PCTYPE_FRAG_IPV4] = IPPROTO_IP,
1057                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = IPPROTO_TCP,
1058                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = IPPROTO_UDP,
1059                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = IPPROTO_SCTP,
1060                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = IPPROTO_IP,
1061                 [I40E_FILTER_PCTYPE_FRAG_IPV6] = IPPROTO_NONE,
1062                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = IPPROTO_TCP,
1063                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = IPPROTO_UDP,
1064                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = IPPROTO_SCTP,
1065                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = IPPROTO_NONE,
1066         };
1067
1068         rte_memcpy(raw_pkt, &fdir_input->flow.l2_flow.dst,
1069                 sizeof(struct rte_ether_addr));
1070         rte_memcpy(raw_pkt + sizeof(struct rte_ether_addr),
1071                 &fdir_input->flow.l2_flow.src,
1072                 sizeof(struct rte_ether_addr));
1073         raw_pkt += 2 * sizeof(struct rte_ether_addr);
1074
1075         if (vlan && fdir_input->flow_ext.vlan_tci) {
1076                 rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame));
1077                 rte_memcpy(raw_pkt + sizeof(uint16_t),
1078                            &fdir_input->flow_ext.vlan_tci,
1079                            sizeof(uint16_t));
1080                 raw_pkt += sizeof(vlan_frame);
1081                 len += sizeof(vlan_frame);
1082         }
1083         ether_type = (uint16_t *)raw_pkt;
1084         raw_pkt += sizeof(uint16_t);
1085         len += sizeof(uint16_t);
1086
1087         if (is_customized_pctype) {
1088                 cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype);
1089                 if (!cus_pctype) {
1090                         PMD_DRV_LOG(ERR, "unknown pctype %u.",
1091                                     fdir_input->pctype);
1092                         return -1;
1093                 }
1094         }
1095
1096         if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD)
1097                 *ether_type = fdir_input->flow.l2_flow.ether_type;
1098         else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP ||
1099                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP ||
1100                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP ||
1101                  pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1102                  pctype == I40E_FILTER_PCTYPE_FRAG_IPV4 ||
1103                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP ||
1104                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP ||
1105                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP ||
1106                  pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1107                  pctype == I40E_FILTER_PCTYPE_FRAG_IPV6 ||
1108                  is_customized_pctype) {
1109                 if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP ||
1110                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP ||
1111                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP ||
1112                         pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1113                         pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) {
1114                         len = fill_ip4_head(fdir_input, raw_pkt,
1115                                         next_proto[pctype], len, ether_type);
1116                 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP ||
1117                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP ||
1118                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP ||
1119                         pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1120                         pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) {
1121                         len = fill_ip6_head(fdir_input, raw_pkt,
1122                                         next_proto[pctype], len,
1123                                         ether_type);
1124                 } else if (cus_pctype->index == I40E_CUSTOMIZED_GTPC ||
1125                          cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 ||
1126                          cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 ||
1127                          cus_pctype->index == I40E_CUSTOMIZED_GTPU) {
1128                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1129                                         len, ether_type);
1130                 } else if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3) {
1131                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_L2TP,
1132                                         len, ether_type);
1133                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4) {
1134                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_ESP,
1135                                         len, ether_type);
1136                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1137                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1138                                         len, ether_type);
1139                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1140                         len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP,
1141                                         len, ether_type);
1142                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6)
1143                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_ESP,
1144                                         len, ether_type);
1145                 else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6_UDP)
1146                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_UDP,
1147                                         len, ether_type);
1148                 else if (cus_pctype->index == I40E_CUSTOMIZED_IPV6_L2TPV3)
1149                         len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_L2TP,
1150                                         len, ether_type);
1151         } else {
1152                 PMD_DRV_LOG(ERR, "unknown pctype %u.", fdir_input->pctype);
1153                 return -1;
1154         }
1155
1156         return len;
1157 }
1158
1159 /**
1160  * i40e_flow_fdir_construct_pkt - construct packet based on fields in input
1161  * @pf: board private structure
1162  * @fdir_input: input set of the flow director entry
1163  * @raw_pkt: a packet to be constructed
1164  */
1165 static int
1166 i40e_flow_fdir_construct_pkt(struct i40e_pf *pf,
1167                              const struct i40e_fdir_input *fdir_input,
1168                              unsigned char *raw_pkt)
1169 {
1170         unsigned char *payload = NULL;
1171         unsigned char *ptr;
1172         struct rte_udp_hdr *udp;
1173         struct rte_tcp_hdr *tcp;
1174         struct rte_sctp_hdr *sctp;
1175         struct rte_flow_item_gtp *gtp;
1176         struct rte_ipv4_hdr *gtp_ipv4;
1177         struct rte_ipv6_hdr *gtp_ipv6;
1178         struct rte_flow_item_l2tpv3oip *l2tpv3oip;
1179         struct rte_flow_item_esp *esp;
1180         struct rte_ipv4_hdr *esp_ipv4;
1181         struct rte_ipv6_hdr *esp_ipv6;
1182
1183         uint8_t size, dst = 0;
1184         uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/
1185         int len;
1186         uint8_t pctype = fdir_input->pctype;
1187         struct i40e_customized_pctype *cus_pctype;
1188
1189         /* raw pcket template - just copy contents of the raw packet */
1190         if (fdir_input->flow_ext.pkt_template) {
1191                 memcpy(raw_pkt, fdir_input->flow.raw_flow.packet,
1192                        fdir_input->flow.raw_flow.length);
1193                 return 0;
1194         }
1195
1196         /* fill the ethernet and IP head */
1197         len = i40e_flow_fdir_fill_eth_ip_head(pf, fdir_input, raw_pkt,
1198                                               !!fdir_input->flow_ext.vlan_tci);
1199         if (len < 0)
1200                 return -EINVAL;
1201
1202         /* fill the L4 head */
1203         if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP) {
1204                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
1205                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
1206                 /**
1207                  * The source and destination fields in the transmitted packet
1208                  * need to be presented in a reversed order with respect
1209                  * to the expected received packets.
1210                  */
1211                 udp->src_port = fdir_input->flow.udp4_flow.dst_port;
1212                 udp->dst_port = fdir_input->flow.udp4_flow.src_port;
1213                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
1214         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP) {
1215                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
1216                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
1217                 /**
1218                  * The source and destination fields in the transmitted packet
1219                  * need to be presented in a reversed order with respect
1220                  * to the expected received packets.
1221                  */
1222                 tcp->src_port = fdir_input->flow.tcp4_flow.dst_port;
1223                 tcp->dst_port = fdir_input->flow.tcp4_flow.src_port;
1224                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
1225         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) {
1226                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
1227                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
1228                 /**
1229                  * The source and destination fields in the transmitted packet
1230                  * need to be presented in a reversed order with respect
1231                  * to the expected received packets.
1232                  */
1233                 sctp->src_port = fdir_input->flow.sctp4_flow.dst_port;
1234                 sctp->dst_port = fdir_input->flow.sctp4_flow.src_port;
1235                 sctp->tag = fdir_input->flow.sctp4_flow.verify_tag;
1236         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER ||
1237                    pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) {
1238                 payload = raw_pkt + len;
1239                 set_idx = I40E_FLXPLD_L3_IDX;
1240         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP) {
1241                 udp = (struct rte_udp_hdr *)(raw_pkt + len);
1242                 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr);
1243                 /**
1244                  * The source and destination fields in the transmitted packet
1245                  * need to be presented in a reversed order with respect
1246                  * to the expected received packets.
1247                  */
1248                 udp->src_port = fdir_input->flow.udp6_flow.dst_port;
1249                 udp->dst_port = fdir_input->flow.udp6_flow.src_port;
1250                 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN);
1251         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP) {
1252                 tcp = (struct rte_tcp_hdr *)(raw_pkt + len);
1253                 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr);
1254                 /**
1255                  * The source and destination fields in the transmitted packet
1256                  * need to be presented in a reversed order with respect
1257                  * to the expected received packets.
1258                  */
1259                 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF;
1260                 tcp->src_port = fdir_input->flow.udp6_flow.dst_port;
1261                 tcp->dst_port = fdir_input->flow.udp6_flow.src_port;
1262         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) {
1263                 sctp = (struct rte_sctp_hdr *)(raw_pkt + len);
1264                 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr);
1265                 /**
1266                  * The source and destination fields in the transmitted packet
1267                  * need to be presented in a reversed order with respect
1268                  * to the expected received packets.
1269                  */
1270                 sctp->src_port = fdir_input->flow.sctp6_flow.dst_port;
1271                 sctp->dst_port = fdir_input->flow.sctp6_flow.src_port;
1272                 sctp->tag = fdir_input->flow.sctp6_flow.verify_tag;
1273         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER ||
1274                    pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) {
1275                 payload = raw_pkt + len;
1276                 set_idx = I40E_FLXPLD_L3_IDX;
1277         } else if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD) {
1278                 payload = raw_pkt + len;
1279                 /**
1280                  * ARP packet is a special case on which the payload
1281                  * starts after the whole ARP header
1282                  */
1283                 if (fdir_input->flow.l2_flow.ether_type ==
1284                                 rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
1285                         payload += sizeof(struct rte_arp_hdr);
1286                 set_idx = I40E_FLXPLD_L2_IDX;
1287         } else if (fdir_input->flow_ext.customized_pctype) {
1288                 /* If customized pctype is used */
1289                 cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype);
1290                 if (cus_pctype->index == I40E_CUSTOMIZED_GTPC ||
1291                     cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 ||
1292                     cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 ||
1293                     cus_pctype->index == I40E_CUSTOMIZED_GTPU) {
1294                         udp = (struct rte_udp_hdr *)(raw_pkt + len);
1295                         udp->dgram_len =
1296                                 rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN);
1297
1298                         gtp = (struct rte_flow_item_gtp *)
1299                                 ((unsigned char *)udp +
1300                                         sizeof(struct rte_udp_hdr));
1301                         gtp->msg_len =
1302                                 rte_cpu_to_be_16(I40E_FDIR_GTP_DEFAULT_LEN);
1303                         gtp->teid = fdir_input->flow.gtp_flow.teid;
1304                         gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0X01;
1305
1306                         /* GTP-C message type is not supported. */
1307                         if (cus_pctype->index == I40E_CUSTOMIZED_GTPC) {
1308                                 udp->dst_port =
1309                                       rte_cpu_to_be_16(I40E_FDIR_GTPC_DST_PORT);
1310                                 gtp->v_pt_rsv_flags =
1311                                         I40E_FDIR_GTP_VER_FLAG_0X32;
1312                         } else {
1313                                 udp->dst_port =
1314                                       rte_cpu_to_be_16(I40E_FDIR_GTPU_DST_PORT);
1315                                 gtp->v_pt_rsv_flags =
1316                                         I40E_FDIR_GTP_VER_FLAG_0X30;
1317                         }
1318
1319                         if (cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4) {
1320                                 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF;
1321                                 gtp_ipv4 = (struct rte_ipv4_hdr *)
1322                                         ((unsigned char *)gtp +
1323                                          sizeof(struct rte_flow_item_gtp));
1324                                 gtp_ipv4->version_ihl =
1325                                         I40E_FDIR_IP_DEFAULT_VERSION_IHL;
1326                                 gtp_ipv4->next_proto_id = IPPROTO_IP;
1327                                 gtp_ipv4->total_length =
1328                                         rte_cpu_to_be_16(
1329                                                 I40E_FDIR_INNER_IP_DEFAULT_LEN);
1330                                 payload = (unsigned char *)gtp_ipv4 +
1331                                         sizeof(struct rte_ipv4_hdr);
1332                         } else if (cus_pctype->index ==
1333                                    I40E_CUSTOMIZED_GTPU_IPV6) {
1334                                 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF;
1335                                 gtp_ipv6 = (struct rte_ipv6_hdr *)
1336                                         ((unsigned char *)gtp +
1337                                          sizeof(struct rte_flow_item_gtp));
1338                                 gtp_ipv6->vtc_flow =
1339                                         rte_cpu_to_be_32(
1340                                                I40E_FDIR_IPv6_DEFAULT_VTC_FLOW |
1341                                                (0 << I40E_FDIR_IPv6_TC_OFFSET));
1342                                 gtp_ipv6->proto = IPPROTO_NONE;
1343                                 gtp_ipv6->payload_len =
1344                                         rte_cpu_to_be_16(
1345                                               I40E_FDIR_INNER_IPV6_DEFAULT_LEN);
1346                                 gtp_ipv6->hop_limits =
1347                                         I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS;
1348                                 payload = (unsigned char *)gtp_ipv6 +
1349                                         sizeof(struct rte_ipv6_hdr);
1350                         } else
1351                                 payload = (unsigned char *)gtp +
1352                                         sizeof(struct rte_flow_item_gtp);
1353                 } else if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3 ||
1354                            cus_pctype->index == I40E_CUSTOMIZED_IPV6_L2TPV3) {
1355                         l2tpv3oip = (struct rte_flow_item_l2tpv3oip *)(raw_pkt
1356                                                                        + len);
1357
1358                         if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3)
1359                                 l2tpv3oip->session_id =
1360                                  fdir_input->flow.ip4_l2tpv3oip_flow.session_id;
1361                         else
1362                                 l2tpv3oip->session_id =
1363                                  fdir_input->flow.ip6_l2tpv3oip_flow.session_id;
1364                         payload = (unsigned char *)l2tpv3oip +
1365                                 sizeof(struct rte_flow_item_l2tpv3oip);
1366                 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4 ||
1367                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6 ||
1368                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP ||
1369                         cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6_UDP) {
1370                         if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4) {
1371                                 esp_ipv4 = (struct rte_ipv4_hdr *)
1372                                         (raw_pkt + len);
1373                                 esp = (struct rte_flow_item_esp *)esp_ipv4;
1374                                 esp->hdr.spi =
1375                                         fdir_input->flow.esp_ipv4_flow.spi;
1376                                 payload = (unsigned char *)esp +
1377                                         sizeof(struct rte_esp_hdr);
1378                                 len += sizeof(struct rte_esp_hdr);
1379                         } else if (cus_pctype->index ==
1380                                         I40E_CUSTOMIZED_ESP_IPV4_UDP) {
1381                                 esp_ipv4 = (struct rte_ipv4_hdr *)
1382                                         (raw_pkt + len);
1383                                 udp = (struct rte_udp_hdr *)esp_ipv4;
1384                                 udp->dst_port = rte_cpu_to_be_16
1385                                         (I40E_FDIR_ESP_DST_PORT);
1386
1387                                 udp->dgram_len = rte_cpu_to_be_16
1388                                                 (I40E_FDIR_UDP_DEFAULT_LEN);
1389                                 esp = (struct rte_flow_item_esp *)
1390                                         ((unsigned char *)esp_ipv4 +
1391                                                 sizeof(struct rte_udp_hdr));
1392                                 esp->hdr.spi =
1393                                         fdir_input->flow.esp_ipv4_udp_flow.spi;
1394                                 payload = (unsigned char *)esp +
1395                                         sizeof(struct rte_esp_hdr);
1396                                 len += sizeof(struct rte_udp_hdr) +
1397                                                 sizeof(struct rte_esp_hdr);
1398                         } else if (cus_pctype->index ==
1399                                         I40E_CUSTOMIZED_ESP_IPV6) {
1400                                 esp_ipv6 = (struct rte_ipv6_hdr *)
1401                                         (raw_pkt + len);
1402                                 esp = (struct rte_flow_item_esp *)esp_ipv6;
1403                                 esp->hdr.spi =
1404                                         fdir_input->flow.esp_ipv6_flow.spi;
1405                                 payload = (unsigned char *)esp +
1406                                         sizeof(struct rte_esp_hdr);
1407                                 len += sizeof(struct rte_esp_hdr);
1408                         } else if (cus_pctype->index ==
1409                                         I40E_CUSTOMIZED_ESP_IPV6_UDP) {
1410                                 esp_ipv6 = (struct rte_ipv6_hdr *)
1411                                         (raw_pkt + len);
1412                                 udp = (struct rte_udp_hdr *)esp_ipv6;
1413                                 udp->dst_port = rte_cpu_to_be_16
1414                                         (I40E_FDIR_ESP_DST_PORT);
1415
1416                                 udp->dgram_len = rte_cpu_to_be_16
1417                                         (I40E_FDIR_UDP_DEFAULT_LEN);
1418                                 esp = (struct rte_flow_item_esp *)
1419                                         ((unsigned char *)esp_ipv6 +
1420                                                 sizeof(struct rte_udp_hdr));
1421                                 esp->hdr.spi =
1422                                         fdir_input->flow.esp_ipv6_udp_flow.spi;
1423                                 payload = (unsigned char *)esp +
1424                                         sizeof(struct rte_esp_hdr);
1425                                 len += sizeof(struct rte_udp_hdr) +
1426                                                 sizeof(struct rte_esp_hdr);
1427                         }
1428                 }
1429         } else {
1430                 PMD_DRV_LOG(ERR, "unknown pctype %u.", fdir_input->pctype);
1431                 return -1;
1432         }
1433
1434         /* fill the flexbytes to payload */
1435         for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
1436                 pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i;
1437                 size = pf->fdir.flex_set[pit_idx].size;
1438                 if (size == 0)
1439                         continue;
1440                 dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t);
1441                 ptr = payload +
1442                       pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t);
1443                 (void)rte_memcpy(ptr,
1444                                  &fdir_input->flow_ext.flexbytes[dst],
1445                                  size * sizeof(uint16_t));
1446         }
1447
1448         return 0;
1449 }
1450
1451 /* Construct the tx flags */
1452 static inline uint64_t
1453 i40e_build_ctob(uint32_t td_cmd,
1454                 uint32_t td_offset,
1455                 unsigned int size,
1456                 uint32_t td_tag)
1457 {
1458         return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA |
1459                         ((uint64_t)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
1460                         ((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
1461                         ((uint64_t)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
1462                         ((uint64_t)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
1463 }
1464
1465 /*
1466  * check the programming status descriptor in rx queue.
1467  * done after Programming Flow Director is programmed on
1468  * tx queue
1469  */
1470 static inline int
1471 i40e_check_fdir_programming_status(struct i40e_rx_queue *rxq)
1472 {
1473         volatile union i40e_rx_desc *rxdp;
1474         uint64_t qword1;
1475         uint32_t rx_status;
1476         uint32_t len, id;
1477         uint32_t error;
1478         int ret = 0;
1479
1480         rxdp = &rxq->rx_ring[rxq->rx_tail];
1481         qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
1482         rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK)
1483                         >> I40E_RXD_QW1_STATUS_SHIFT;
1484
1485         if (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1486                 len = qword1 >> I40E_RX_PROG_STATUS_DESC_LENGTH_SHIFT;
1487                 id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
1488                             I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
1489
1490                 if (len  == I40E_RX_PROG_STATUS_DESC_LENGTH &&
1491                     id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) {
1492                         error = (qword1 &
1493                                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
1494                                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
1495                         if (error == (0x1 <<
1496                                 I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
1497                                 PMD_DRV_LOG(ERR, "Failed to add FDIR filter"
1498                                             " (FD_ID %u): programming status"
1499                                             " reported.",
1500                                             rxdp->wb.qword0.hi_dword.fd_id);
1501                                 ret = -1;
1502                         } else if (error == (0x1 <<
1503                                 I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
1504                                 PMD_DRV_LOG(ERR, "Failed to delete FDIR filter"
1505                                             " (FD_ID %u): programming status"
1506                                             " reported.",
1507                                             rxdp->wb.qword0.hi_dword.fd_id);
1508                                 ret = -1;
1509                         } else
1510                                 PMD_DRV_LOG(ERR, "invalid programming status"
1511                                             " reported, error = %u.", error);
1512                 } else
1513                         PMD_DRV_LOG(INFO, "unknown programming status"
1514                                     " reported, len = %d, id = %u.", len, id);
1515                 rxdp->wb.qword1.status_error_len = 0;
1516                 rxq->rx_tail++;
1517                 if (unlikely(rxq->rx_tail == rxq->nb_rx_desc))
1518                         rxq->rx_tail = 0;
1519                 if (rxq->rx_tail == 0)
1520                         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1521                 else
1522                         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_tail - 1);
1523         }
1524
1525         return ret;
1526 }
1527
1528 static int
1529 i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input,
1530                          struct i40e_fdir_filter *filter)
1531 {
1532         rte_memcpy(&filter->fdir, input, sizeof(struct i40e_fdir_filter_conf));
1533         if (input->input.flow_ext.pkt_template) {
1534                 filter->fdir.input.flow.raw_flow.packet = NULL;
1535                 filter->fdir.input.flow.raw_flow.length =
1536                         rte_hash_crc(input->input.flow.raw_flow.packet,
1537                                      input->input.flow.raw_flow.length,
1538                                      input->input.flow.raw_flow.pctype);
1539         }
1540         return 0;
1541 }
1542
1543 /* Check if there exists the flow director filter */
1544 static struct i40e_fdir_filter *
1545 i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info,
1546                         const struct i40e_fdir_input *input)
1547 {
1548         int ret;
1549
1550         if (input->flow_ext.pkt_template)
1551                 ret = rte_hash_lookup_with_hash(fdir_info->hash_table,
1552                                                 (const void *)input,
1553                                                 input->flow.raw_flow.length);
1554         else
1555                 ret = rte_hash_lookup(fdir_info->hash_table,
1556                                       (const void *)input);
1557         if (ret < 0)
1558                 return NULL;
1559
1560         return fdir_info->hash_map[ret];
1561 }
1562
1563 /* Add a flow director filter into the SW list */
1564 static int
1565 i40e_sw_fdir_filter_insert(struct i40e_pf *pf, struct i40e_fdir_filter *filter)
1566 {
1567         struct i40e_fdir_info *fdir_info = &pf->fdir;
1568         int ret;
1569
1570         if (filter->fdir.input.flow_ext.pkt_template)
1571                 ret = rte_hash_add_key_with_hash(fdir_info->hash_table,
1572                                  &filter->fdir.input,
1573                                  filter->fdir.input.flow.raw_flow.length);
1574         else
1575                 ret = rte_hash_add_key(fdir_info->hash_table,
1576                                        &filter->fdir.input);
1577         if (ret < 0) {
1578                 PMD_DRV_LOG(ERR,
1579                             "Failed to insert fdir filter to hash table %d!",
1580                             ret);
1581                 return ret;
1582         }
1583         fdir_info->hash_map[ret] = filter;
1584
1585         TAILQ_INSERT_TAIL(&fdir_info->fdir_list, filter, rules);
1586
1587         return 0;
1588 }
1589
1590 /* Delete a flow director filter from the SW list */
1591 int
1592 i40e_sw_fdir_filter_del(struct i40e_pf *pf, struct i40e_fdir_input *input)
1593 {
1594         struct i40e_fdir_info *fdir_info = &pf->fdir;
1595         struct i40e_fdir_filter *filter;
1596         int ret;
1597
1598         if (input->flow_ext.pkt_template)
1599                 ret = rte_hash_del_key_with_hash(fdir_info->hash_table,
1600                                                  input,
1601                                                  input->flow.raw_flow.length);
1602         else
1603                 ret = rte_hash_del_key(fdir_info->hash_table, input);
1604         if (ret < 0) {
1605                 PMD_DRV_LOG(ERR,
1606                             "Failed to delete fdir filter to hash table %d!",
1607                             ret);
1608                 return ret;
1609         }
1610         filter = fdir_info->hash_map[ret];
1611         fdir_info->hash_map[ret] = NULL;
1612
1613         TAILQ_REMOVE(&fdir_info->fdir_list, filter, rules);
1614         rte_free(filter);
1615
1616         return 0;
1617 }
1618
1619 /*
1620  * i40e_add_del_fdir_filter - add or remove a flow director filter.
1621  * @pf: board private structure
1622  * @filter: fdir filter entry
1623  * @add: 0 - delete, 1 - add
1624  */
1625 int
1626 i40e_add_del_fdir_filter(struct rte_eth_dev *dev,
1627                          const struct rte_eth_fdir_filter *filter,
1628                          bool add)
1629 {
1630         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1631         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1632         unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
1633         enum i40e_filter_pctype pctype;
1634         int ret = 0;
1635
1636         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_PERFECT) {
1637                 PMD_DRV_LOG(ERR, "FDIR is not enabled, please"
1638                         " check the mode in fdir_conf.");
1639                 return -ENOTSUP;
1640         }
1641
1642         pctype = i40e_flowtype_to_pctype(pf->adapter, filter->input.flow_type);
1643         if (pctype == I40E_FILTER_PCTYPE_INVALID) {
1644                 PMD_DRV_LOG(ERR, "invalid flow_type input.");
1645                 return -EINVAL;
1646         }
1647         if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) {
1648                 PMD_DRV_LOG(ERR, "Invalid queue ID");
1649                 return -EINVAL;
1650         }
1651         if (filter->input.flow_ext.is_vf &&
1652                 filter->input.flow_ext.dst_id >= pf->vf_num) {
1653                 PMD_DRV_LOG(ERR, "Invalid VF ID");
1654                 return -EINVAL;
1655         }
1656
1657         memset(pkt, 0, I40E_FDIR_PKT_LEN);
1658
1659         ret = i40e_fdir_construct_pkt(pf, &filter->input, pkt);
1660         if (ret < 0) {
1661                 PMD_DRV_LOG(ERR, "construct packet for fdir fails.");
1662                 return ret;
1663         }
1664
1665         if (hw->mac.type == I40E_MAC_X722) {
1666                 /* get translated pctype value in fd pctype register */
1667                 pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(
1668                         hw, I40E_GLQF_FD_PCTYPES((int)pctype));
1669         }
1670
1671         ret = i40e_fdir_filter_programming(pf, pctype, filter, add);
1672         if (ret < 0) {
1673                 PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).",
1674                             pctype);
1675                 return ret;
1676         }
1677
1678         return ret;
1679 }
1680
1681 /**
1682  * i40e_flow_add_del_fdir_filter - add or remove a flow director filter.
1683  * @pf: board private structure
1684  * @filter: fdir filter entry
1685  * @add: 0 - delete, 1 - add
1686  */
1687 int
1688 i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev,
1689                               const struct i40e_fdir_filter_conf *filter,
1690                               bool add)
1691 {
1692         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1693         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1694         unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
1695         enum i40e_filter_pctype pctype;
1696         struct i40e_fdir_info *fdir_info = &pf->fdir;
1697         struct i40e_fdir_filter *fdir_filter, *node;
1698         struct i40e_fdir_filter check_filter; /* Check if the filter exists */
1699         int ret = 0;
1700
1701         if (pf->fdir.fdir_vsi == NULL) {
1702                 PMD_DRV_LOG(ERR, "FDIR is not enabled");
1703                 return -ENOTSUP;
1704         }
1705
1706         if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) {
1707                 PMD_DRV_LOG(ERR, "Invalid queue ID");
1708                 return -EINVAL;
1709         }
1710         if (filter->input.flow_ext.is_vf &&
1711             filter->input.flow_ext.dst_id >= pf->vf_num) {
1712                 PMD_DRV_LOG(ERR, "Invalid VF ID");
1713                 return -EINVAL;
1714         }
1715         if (filter->input.flow_ext.pkt_template) {
1716                 if (filter->input.flow.raw_flow.length > I40E_FDIR_PKT_LEN ||
1717                     !filter->input.flow.raw_flow.packet) {
1718                         PMD_DRV_LOG(ERR, "Invalid raw packet template"
1719                                 " flow filter parameters!");
1720                         return -EINVAL;
1721                 }
1722                 pctype = filter->input.flow.raw_flow.pctype;
1723         } else {
1724                 pctype = filter->input.pctype;
1725         }
1726
1727         /* Check if there is the filter in SW list */
1728         memset(&check_filter, 0, sizeof(check_filter));
1729         i40e_fdir_filter_convert(filter, &check_filter);
1730         node = i40e_sw_fdir_filter_lookup(fdir_info, &check_filter.fdir.input);
1731         if (add && node) {
1732                 PMD_DRV_LOG(ERR,
1733                             "Conflict with existing flow director rules!");
1734                 return -EINVAL;
1735         }
1736
1737         if (!add && !node) {
1738                 PMD_DRV_LOG(ERR,
1739                             "There's no corresponding flow firector filter!");
1740                 return -EINVAL;
1741         }
1742
1743         memset(pkt, 0, I40E_FDIR_PKT_LEN);
1744
1745         ret = i40e_flow_fdir_construct_pkt(pf, &filter->input, pkt);
1746         if (ret < 0) {
1747                 PMD_DRV_LOG(ERR, "construct packet for fdir fails.");
1748                 return ret;
1749         }
1750
1751         if (hw->mac.type == I40E_MAC_X722) {
1752                 /* get translated pctype value in fd pctype register */
1753                 pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(
1754                         hw, I40E_GLQF_FD_PCTYPES((int)pctype));
1755         }
1756
1757         ret = i40e_flow_fdir_filter_programming(pf, pctype, filter, add);
1758         if (ret < 0) {
1759                 PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).",
1760                             pctype);
1761                 return ret;
1762         }
1763
1764         if (add) {
1765                 fdir_filter = rte_zmalloc("fdir_filter",
1766                                           sizeof(*fdir_filter), 0);
1767                 if (fdir_filter == NULL) {
1768                         PMD_DRV_LOG(ERR, "Failed to alloc memory.");
1769                         return -ENOMEM;
1770                 }
1771
1772                 rte_memcpy(fdir_filter, &check_filter, sizeof(check_filter));
1773                 ret = i40e_sw_fdir_filter_insert(pf, fdir_filter);
1774                 if (ret < 0)
1775                         rte_free(fdir_filter);
1776         } else {
1777                 ret = i40e_sw_fdir_filter_del(pf, &node->fdir.input);
1778         }
1779
1780         return ret;
1781 }
1782
1783 /*
1784  * i40e_fdir_filter_programming - Program a flow director filter rule.
1785  * Is done by Flow Director Programming Descriptor followed by packet
1786  * structure that contains the filter fields need to match.
1787  * @pf: board private structure
1788  * @pctype: pctype
1789  * @filter: fdir filter entry
1790  * @add: 0 - delete, 1 - add
1791  */
1792 static int
1793 i40e_fdir_filter_programming(struct i40e_pf *pf,
1794                         enum i40e_filter_pctype pctype,
1795                         const struct rte_eth_fdir_filter *filter,
1796                         bool add)
1797 {
1798         struct i40e_tx_queue *txq = pf->fdir.txq;
1799         struct i40e_rx_queue *rxq = pf->fdir.rxq;
1800         const struct rte_eth_fdir_action *fdir_action = &filter->action;
1801         volatile struct i40e_tx_desc *txdp;
1802         volatile struct i40e_filter_program_desc *fdirdp;
1803         uint32_t td_cmd;
1804         uint16_t vsi_id, i;
1805         uint8_t dest;
1806
1807         PMD_DRV_LOG(INFO, "filling filter programming descriptor.");
1808         fdirdp = (volatile struct i40e_filter_program_desc *)
1809                         (&(txq->tx_ring[txq->tx_tail]));
1810
1811         fdirdp->qindex_flex_ptype_vsi =
1812                         rte_cpu_to_le_32((fdir_action->rx_queue <<
1813                                           I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1814                                           I40E_TXD_FLTR_QW0_QINDEX_MASK);
1815
1816         fdirdp->qindex_flex_ptype_vsi |=
1817                         rte_cpu_to_le_32((fdir_action->flex_off <<
1818                                           I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
1819                                           I40E_TXD_FLTR_QW0_FLEXOFF_MASK);
1820
1821         fdirdp->qindex_flex_ptype_vsi |=
1822                         rte_cpu_to_le_32((pctype <<
1823                                           I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
1824                                           I40E_TXD_FLTR_QW0_PCTYPE_MASK);
1825
1826         if (filter->input.flow_ext.is_vf)
1827                 vsi_id = pf->vfs[filter->input.flow_ext.dst_id].vsi->vsi_id;
1828         else
1829                 /* Use LAN VSI Id by default */
1830                 vsi_id = pf->main_vsi->vsi_id;
1831         fdirdp->qindex_flex_ptype_vsi |=
1832                 rte_cpu_to_le_32(((uint32_t)vsi_id <<
1833                                   I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
1834                                   I40E_TXD_FLTR_QW0_DEST_VSI_MASK);
1835
1836         fdirdp->dtype_cmd_cntindex =
1837                         rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG);
1838
1839         if (add)
1840                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1841                                 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1842                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1843         else
1844                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1845                                 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1846                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1847
1848         if (fdir_action->behavior == RTE_ETH_FDIR_REJECT)
1849                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
1850         else if (fdir_action->behavior == RTE_ETH_FDIR_ACCEPT)
1851                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX;
1852         else if (fdir_action->behavior == RTE_ETH_FDIR_PASSTHRU)
1853                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_OTHER;
1854         else {
1855                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter:"
1856                             " unsupported fdir behavior.");
1857                 return -EINVAL;
1858         }
1859
1860         fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest <<
1861                                 I40E_TXD_FLTR_QW1_DEST_SHIFT) &
1862                                 I40E_TXD_FLTR_QW1_DEST_MASK);
1863
1864         fdirdp->dtype_cmd_cntindex |=
1865                 rte_cpu_to_le_32((fdir_action->report_status<<
1866                                 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
1867                                 I40E_TXD_FLTR_QW1_FD_STATUS_MASK);
1868
1869         fdirdp->dtype_cmd_cntindex |=
1870                         rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK);
1871         fdirdp->dtype_cmd_cntindex |=
1872                         rte_cpu_to_le_32(
1873                         ((uint32_t)pf->fdir.match_counter_index <<
1874                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
1875                         I40E_TXD_FLTR_QW1_CNTINDEX_MASK);
1876
1877         fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id);
1878
1879         PMD_DRV_LOG(INFO, "filling transmit descriptor.");
1880         txdp = &(txq->tx_ring[txq->tx_tail + 1]);
1881         txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
1882         td_cmd = I40E_TX_DESC_CMD_EOP |
1883                  I40E_TX_DESC_CMD_RS  |
1884                  I40E_TX_DESC_CMD_DUMMY;
1885
1886         txdp->cmd_type_offset_bsz =
1887                 i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0);
1888
1889         txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */
1890         if (txq->tx_tail >= txq->nb_tx_desc)
1891                 txq->tx_tail = 0;
1892         /* Update the tx tail register */
1893         rte_wmb();
1894         I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
1895         for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
1896                 if ((txdp->cmd_type_offset_bsz &
1897                                 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
1898                                 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
1899                         break;
1900                 rte_delay_us(1);
1901         }
1902         if (i >= I40E_FDIR_MAX_WAIT_US) {
1903                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter:"
1904                             " time out to get DD on tx queue.");
1905                 return -ETIMEDOUT;
1906         }
1907         /* totally delay 10 ms to check programming status*/
1908         for (; i < I40E_FDIR_MAX_WAIT_US; i++) {
1909                 if (i40e_check_fdir_programming_status(rxq) >= 0)
1910                         return 0;
1911                 rte_delay_us(1);
1912         }
1913         PMD_DRV_LOG(ERR,
1914                 "Failed to program FDIR filter: programming status reported.");
1915         return -ETIMEDOUT;
1916 }
1917
1918 /*
1919  * i40e_flow_fdir_filter_programming - Program a flow director filter rule.
1920  * Is done by Flow Director Programming Descriptor followed by packet
1921  * structure that contains the filter fields need to match.
1922  * @pf: board private structure
1923  * @pctype: pctype
1924  * @filter: fdir filter entry
1925  * @add: 0 - delete, 1 - add
1926  */
1927 static int
1928 i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
1929                                   enum i40e_filter_pctype pctype,
1930                                   const struct i40e_fdir_filter_conf *filter,
1931                                   bool add)
1932 {
1933         struct i40e_tx_queue *txq = pf->fdir.txq;
1934         struct i40e_rx_queue *rxq = pf->fdir.rxq;
1935         const struct i40e_fdir_action *fdir_action = &filter->action;
1936         volatile struct i40e_tx_desc *txdp;
1937         volatile struct i40e_filter_program_desc *fdirdp;
1938         uint32_t td_cmd;
1939         uint16_t vsi_id, i;
1940         uint8_t dest;
1941
1942         PMD_DRV_LOG(INFO, "filling filter programming descriptor.");
1943         fdirdp = (volatile struct i40e_filter_program_desc *)
1944                                 (&txq->tx_ring[txq->tx_tail]);
1945
1946         fdirdp->qindex_flex_ptype_vsi =
1947                         rte_cpu_to_le_32((fdir_action->rx_queue <<
1948                                           I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1949                                           I40E_TXD_FLTR_QW0_QINDEX_MASK);
1950
1951         fdirdp->qindex_flex_ptype_vsi |=
1952                         rte_cpu_to_le_32((fdir_action->flex_off <<
1953                                           I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
1954                                           I40E_TXD_FLTR_QW0_FLEXOFF_MASK);
1955
1956         fdirdp->qindex_flex_ptype_vsi |=
1957                         rte_cpu_to_le_32((pctype <<
1958                                           I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
1959                                           I40E_TXD_FLTR_QW0_PCTYPE_MASK);
1960
1961         if (filter->input.flow_ext.is_vf)
1962                 vsi_id = pf->vfs[filter->input.flow_ext.dst_id].vsi->vsi_id;
1963         else
1964                 /* Use LAN VSI Id by default */
1965                 vsi_id = pf->main_vsi->vsi_id;
1966         fdirdp->qindex_flex_ptype_vsi |=
1967                 rte_cpu_to_le_32(((uint32_t)vsi_id <<
1968                                   I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
1969                                   I40E_TXD_FLTR_QW0_DEST_VSI_MASK);
1970
1971         fdirdp->dtype_cmd_cntindex =
1972                         rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG);
1973
1974         if (add)
1975                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1976                                 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1977                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1978         else
1979                 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32(
1980                                 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1981                                 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1982
1983         if (fdir_action->behavior == I40E_FDIR_REJECT)
1984                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
1985         else if (fdir_action->behavior == I40E_FDIR_ACCEPT)
1986                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX;
1987         else if (fdir_action->behavior == I40E_FDIR_PASSTHRU)
1988                 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_OTHER;
1989         else {
1990                 PMD_DRV_LOG(ERR, "Failed to program FDIR filter: unsupported fdir behavior.");
1991                 return -EINVAL;
1992         }
1993
1994         fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest <<
1995                                 I40E_TXD_FLTR_QW1_DEST_SHIFT) &
1996                                 I40E_TXD_FLTR_QW1_DEST_MASK);
1997
1998         fdirdp->dtype_cmd_cntindex |=
1999                 rte_cpu_to_le_32((fdir_action->report_status <<
2000                                 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
2001                                 I40E_TXD_FLTR_QW1_FD_STATUS_MASK);
2002
2003         fdirdp->dtype_cmd_cntindex |=
2004                         rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK);
2005         fdirdp->dtype_cmd_cntindex |=
2006                         rte_cpu_to_le_32(
2007                         ((uint32_t)pf->fdir.match_counter_index <<
2008                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
2009                         I40E_TXD_FLTR_QW1_CNTINDEX_MASK);
2010
2011         fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id);
2012
2013         PMD_DRV_LOG(INFO, "filling transmit descriptor.");
2014         txdp = &txq->tx_ring[txq->tx_tail + 1];
2015         txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
2016         td_cmd = I40E_TX_DESC_CMD_EOP |
2017                  I40E_TX_DESC_CMD_RS  |
2018                  I40E_TX_DESC_CMD_DUMMY;
2019
2020         txdp->cmd_type_offset_bsz =
2021                 i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0);
2022
2023         txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */
2024         if (txq->tx_tail >= txq->nb_tx_desc)
2025                 txq->tx_tail = 0;
2026         /* Update the tx tail register */
2027         rte_wmb();
2028         I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
2029         for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
2030                 if ((txdp->cmd_type_offset_bsz &
2031                                 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
2032                                 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
2033                         break;
2034                 rte_delay_us(1);
2035         }
2036         if (i >= I40E_FDIR_MAX_WAIT_US) {
2037                 PMD_DRV_LOG(ERR,
2038                     "Failed to program FDIR filter: time out to get DD on tx queue.");
2039                 return -ETIMEDOUT;
2040         }
2041         /* totally delay 10 ms to check programming status*/
2042         rte_delay_us(I40E_FDIR_MAX_WAIT_US);
2043         if (i40e_check_fdir_programming_status(rxq) < 0) {
2044                 PMD_DRV_LOG(ERR,
2045                     "Failed to program FDIR filter: programming status reported.");
2046                 return -ETIMEDOUT;
2047         }
2048
2049         return 0;
2050 }
2051
2052 /*
2053  * i40e_fdir_flush - clear all filters of Flow Director table
2054  * @pf: board private structure
2055  */
2056 int
2057 i40e_fdir_flush(struct rte_eth_dev *dev)
2058 {
2059         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2060         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2061         uint32_t reg;
2062         uint16_t guarant_cnt, best_cnt;
2063         uint16_t i;
2064
2065         I40E_WRITE_REG(hw, I40E_PFQF_CTL_1, I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
2066         I40E_WRITE_FLUSH(hw);
2067
2068         for (i = 0; i < I40E_FDIR_FLUSH_RETRY; i++) {
2069                 rte_delay_ms(I40E_FDIR_FLUSH_INTERVAL_MS);
2070                 reg = I40E_READ_REG(hw, I40E_PFQF_CTL_1);
2071                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
2072                         break;
2073         }
2074         if (i >= I40E_FDIR_FLUSH_RETRY) {
2075                 PMD_DRV_LOG(ERR, "FD table did not flush, may need more time.");
2076                 return -ETIMEDOUT;
2077         }
2078         guarant_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
2079                                 I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2080                                 I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2081         best_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) &
2082                                 I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2083                                 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2084         if (guarant_cnt != 0 || best_cnt != 0) {
2085                 PMD_DRV_LOG(ERR, "Failed to flush FD table.");
2086                 return -ENOSYS;
2087         } else
2088                 PMD_DRV_LOG(INFO, "FD table Flush success.");
2089         return 0;
2090 }
2091
2092 static inline void
2093 i40e_fdir_info_get_flex_set(struct i40e_pf *pf,
2094                         struct rte_eth_flex_payload_cfg *flex_set,
2095                         uint16_t *num)
2096 {
2097         struct i40e_fdir_flex_pit *flex_pit;
2098         struct rte_eth_flex_payload_cfg *ptr = flex_set;
2099         uint16_t src, dst, size, j, k;
2100         uint8_t i, layer_idx;
2101
2102         for (layer_idx = I40E_FLXPLD_L2_IDX;
2103              layer_idx <= I40E_FLXPLD_L4_IDX;
2104              layer_idx++) {
2105                 if (layer_idx == I40E_FLXPLD_L2_IDX)
2106                         ptr->type = RTE_ETH_L2_PAYLOAD;
2107                 else if (layer_idx == I40E_FLXPLD_L3_IDX)
2108                         ptr->type = RTE_ETH_L3_PAYLOAD;
2109                 else if (layer_idx == I40E_FLXPLD_L4_IDX)
2110                         ptr->type = RTE_ETH_L4_PAYLOAD;
2111
2112                 for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) {
2113                         flex_pit = &pf->fdir.flex_set[layer_idx *
2114                                 I40E_MAX_FLXPLD_FIED + i];
2115                         if (flex_pit->size == 0)
2116                                 continue;
2117                         src = flex_pit->src_offset * sizeof(uint16_t);
2118                         dst = flex_pit->dst_offset * sizeof(uint16_t);
2119                         size = flex_pit->size * sizeof(uint16_t);
2120                         for (j = src, k = dst; j < src + size; j++, k++)
2121                                 ptr->src_offset[k] = j;
2122                 }
2123                 (*num)++;
2124                 ptr++;
2125         }
2126 }
2127
2128 static inline void
2129 i40e_fdir_info_get_flex_mask(struct i40e_pf *pf,
2130                         struct rte_eth_fdir_flex_mask *flex_mask,
2131                         uint16_t *num)
2132 {
2133         struct i40e_fdir_flex_mask *mask;
2134         struct rte_eth_fdir_flex_mask *ptr = flex_mask;
2135         uint16_t flow_type;
2136         uint8_t i, j;
2137         uint16_t off_bytes, mask_tmp;
2138
2139         for (i = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
2140              i <= I40E_FILTER_PCTYPE_L2_PAYLOAD;
2141              i++) {
2142                 mask =  &pf->fdir.flex_mask[i];
2143                 flow_type = i40e_pctype_to_flowtype(pf->adapter,
2144                                                     (enum i40e_filter_pctype)i);
2145                 if (flow_type == RTE_ETH_FLOW_UNKNOWN)
2146                         continue;
2147
2148                 for (j = 0; j < I40E_FDIR_MAX_FLEXWORD_NUM; j++) {
2149                         if (mask->word_mask & I40E_FLEX_WORD_MASK(j)) {
2150                                 ptr->mask[j * sizeof(uint16_t)] = UINT8_MAX;
2151                                 ptr->mask[j * sizeof(uint16_t) + 1] = UINT8_MAX;
2152                         } else {
2153                                 ptr->mask[j * sizeof(uint16_t)] = 0x0;
2154                                 ptr->mask[j * sizeof(uint16_t) + 1] = 0x0;
2155                         }
2156                 }
2157                 for (j = 0; j < I40E_FDIR_BITMASK_NUM_WORD; j++) {
2158                         off_bytes = mask->bitmask[j].offset * sizeof(uint16_t);
2159                         mask_tmp = ~mask->bitmask[j].mask;
2160                         ptr->mask[off_bytes] &= I40E_HI_BYTE(mask_tmp);
2161                         ptr->mask[off_bytes + 1] &= I40E_LO_BYTE(mask_tmp);
2162                 }
2163                 ptr->flow_type = flow_type;
2164                 ptr++;
2165                 (*num)++;
2166         }
2167 }
2168
2169 /*
2170  * i40e_fdir_info_get - get information of Flow Director
2171  * @pf: ethernet device to get info from
2172  * @fdir: a pointer to a structure of type *rte_eth_fdir_info* to be filled with
2173  *    the flow director information.
2174  */
2175 void
2176 i40e_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir)
2177 {
2178         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2179         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2180         uint16_t num_flex_set = 0;
2181         uint16_t num_flex_mask = 0;
2182         uint16_t i;
2183
2184         if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT)
2185                 fdir->mode = RTE_FDIR_MODE_PERFECT;
2186         else
2187                 fdir->mode = RTE_FDIR_MODE_NONE;
2188
2189         fdir->guarant_spc =
2190                 (uint32_t)hw->func_caps.fd_filters_guaranteed;
2191         fdir->best_spc =
2192                 (uint32_t)hw->func_caps.fd_filters_best_effort;
2193         fdir->max_flexpayload = I40E_FDIR_MAX_FLEX_LEN;
2194         fdir->flow_types_mask[0] = I40E_FDIR_FLOWS;
2195         for (i = 1; i < RTE_FLOW_MASK_ARRAY_SIZE; i++)
2196                 fdir->flow_types_mask[i] = 0ULL;
2197         fdir->flex_payload_unit = sizeof(uint16_t);
2198         fdir->flex_bitmask_unit = sizeof(uint16_t);
2199         fdir->max_flex_payload_segment_num = I40E_MAX_FLXPLD_FIED;
2200         fdir->flex_payload_limit = I40E_MAX_FLX_SOURCE_OFF;
2201         fdir->max_flex_bitmask_num = I40E_FDIR_BITMASK_NUM_WORD;
2202
2203         i40e_fdir_info_get_flex_set(pf,
2204                                 fdir->flex_conf.flex_set,
2205                                 &num_flex_set);
2206         i40e_fdir_info_get_flex_mask(pf,
2207                                 fdir->flex_conf.flex_mask,
2208                                 &num_flex_mask);
2209
2210         fdir->flex_conf.nb_payloads = num_flex_set;
2211         fdir->flex_conf.nb_flexmasks = num_flex_mask;
2212 }
2213
2214 /*
2215  * i40e_fdir_stat_get - get statistics of Flow Director
2216  * @pf: ethernet device to get info from
2217  * @stat: a pointer to a structure of type *rte_eth_fdir_stats* to be filled with
2218  *    the flow director statistics.
2219  */
2220 void
2221 i40e_fdir_stats_get(struct rte_eth_dev *dev, struct rte_eth_fdir_stats *stat)
2222 {
2223         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2224         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2225         uint32_t fdstat;
2226
2227         fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT);
2228         stat->guarant_cnt =
2229                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2230                             I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2231         stat->best_cnt =
2232                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2233                             I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2234 }
2235
2236 static int
2237 i40e_fdir_filter_set(struct rte_eth_dev *dev,
2238                      struct rte_eth_fdir_filter_info *info)
2239 {
2240         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2241         int ret = 0;
2242
2243         if (!info) {
2244                 PMD_DRV_LOG(ERR, "Invalid pointer");
2245                 return -EFAULT;
2246         }
2247
2248         switch (info->info_type) {
2249         case RTE_ETH_FDIR_FILTER_INPUT_SET_SELECT:
2250                 ret = i40e_fdir_filter_inset_select(pf,
2251                                 &(info->info.input_set_conf));
2252                 break;
2253         default:
2254                 PMD_DRV_LOG(ERR, "FD filter info type (%d) not supported",
2255                             info->info_type);
2256                 return -EINVAL;
2257         }
2258
2259         return ret;
2260 }
2261
2262 /*
2263  * i40e_fdir_ctrl_func - deal with all operations on flow director.
2264  * @pf: board private structure
2265  * @filter_op:operation will be taken.
2266  * @arg: a pointer to specific structure corresponding to the filter_op
2267  */
2268 int
2269 i40e_fdir_ctrl_func(struct rte_eth_dev *dev,
2270                        enum rte_filter_op filter_op,
2271                        void *arg)
2272 {
2273         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2274         int ret = 0;
2275
2276         if ((pf->flags & I40E_FLAG_FDIR) == 0)
2277                 return -ENOTSUP;
2278
2279         if (filter_op == RTE_ETH_FILTER_NOP)
2280                 return 0;
2281
2282         if (arg == NULL && filter_op != RTE_ETH_FILTER_FLUSH)
2283                 return -EINVAL;
2284
2285         switch (filter_op) {
2286         case RTE_ETH_FILTER_ADD:
2287                 ret = i40e_add_del_fdir_filter(dev,
2288                         (struct rte_eth_fdir_filter *)arg,
2289                         TRUE);
2290                 break;
2291         case RTE_ETH_FILTER_DELETE:
2292                 ret = i40e_add_del_fdir_filter(dev,
2293                         (struct rte_eth_fdir_filter *)arg,
2294                         FALSE);
2295                 break;
2296         case RTE_ETH_FILTER_FLUSH:
2297                 ret = i40e_fdir_flush(dev);
2298                 break;
2299         case RTE_ETH_FILTER_INFO:
2300                 i40e_fdir_info_get(dev, (struct rte_eth_fdir_info *)arg);
2301                 break;
2302         case RTE_ETH_FILTER_SET:
2303                 ret = i40e_fdir_filter_set(dev,
2304                         (struct rte_eth_fdir_filter_info *)arg);
2305                 break;
2306         case RTE_ETH_FILTER_STATS:
2307                 i40e_fdir_stats_get(dev, (struct rte_eth_fdir_stats *)arg);
2308                 break;
2309         default:
2310                 PMD_DRV_LOG(ERR, "unknown operation %u.", filter_op);
2311                 ret = -EINVAL;
2312                 break;
2313         }
2314         return ret;
2315 }
2316
2317 /* Restore flow director filter */
2318 void
2319 i40e_fdir_filter_restore(struct i40e_pf *pf)
2320 {
2321         struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(pf->main_vsi);
2322         struct i40e_fdir_filter_list *fdir_list = &pf->fdir.fdir_list;
2323         struct i40e_fdir_filter *f;
2324         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2325         uint32_t fdstat;
2326         uint32_t guarant_cnt;  /**< Number of filters in guaranteed spaces. */
2327         uint32_t best_cnt;     /**< Number of filters in best effort spaces. */
2328
2329         TAILQ_FOREACH(f, fdir_list, rules)
2330                 i40e_flow_add_del_fdir_filter(dev, &f->fdir, TRUE);
2331
2332         fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT);
2333         guarant_cnt =
2334                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >>
2335                            I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT);
2336         best_cnt =
2337                 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
2338                            I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
2339
2340         PMD_DRV_LOG(INFO, "FDIR: Guarant count: %d,  Best count: %d",
2341                     guarant_cnt, best_cnt);
2342 }