drivers/net/ice/ice_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_pci.h>
   6
   7 #include "base/ice_sched.h"
   8 #include "ice_ethdev.h"
   9 #include "ice_rxtx.h"
  10
  11 #define ICE_MAX_QP_NUM "max_queue_pair_num"
  12 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
  13
  14 int ice_logtype_init;
  15 int ice_logtype_driver;
  16
  17 static int ice_dev_configure(struct rte_eth_dev *dev);
  18 static int ice_dev_start(struct rte_eth_dev *dev);
  19 static void ice_dev_stop(struct rte_eth_dev *dev);
  20 static void ice_dev_close(struct rte_eth_dev *dev);
  21 static int ice_dev_reset(struct rte_eth_dev *dev);
  22 static void ice_dev_info_get(struct rte_eth_dev *dev,
  23                              struct rte_eth_dev_info *dev_info);
  24 static int ice_link_update(struct rte_eth_dev *dev,
  25                            int wait_to_complete);
  26 static int ice_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
  27 static int ice_macaddr_set(struct rte_eth_dev *dev,
  28                            struct ether_addr *mac_addr);
  29 static int ice_macaddr_add(struct rte_eth_dev *dev,
  30                            struct ether_addr *mac_addr,
  31                            __rte_unused uint32_t index,
  32                            uint32_t pool);
  33 static void ice_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
  34
  35 static const struct rte_pci_id pci_id_ice_map[] = {
  36         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810C_BACKPLANE) },
  37         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810C_QSFP) },
  38         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810C_SFP) },
  39         { .vendor_id = 0, /* sentinel */ },
  40 };
  41
  42 static const struct eth_dev_ops ice_eth_dev_ops = {
  43         .dev_configure                = ice_dev_configure,
  44         .dev_start                    = ice_dev_start,
  45         .dev_stop                     = ice_dev_stop,
  46         .dev_close                    = ice_dev_close,
  47         .dev_reset                    = ice_dev_reset,
  48         .rx_queue_start               = ice_rx_queue_start,
  49         .rx_queue_stop                = ice_rx_queue_stop,
  50         .tx_queue_start               = ice_tx_queue_start,
  51         .tx_queue_stop                = ice_tx_queue_stop,
  52         .rx_queue_setup               = ice_rx_queue_setup,
  53         .rx_queue_release             = ice_rx_queue_release,
  54         .tx_queue_setup               = ice_tx_queue_setup,
  55         .tx_queue_release             = ice_tx_queue_release,
  56         .dev_infos_get                = ice_dev_info_get,
  57         .dev_supported_ptypes_get     = ice_dev_supported_ptypes_get,
  58         .link_update                  = ice_link_update,
  59         .mtu_set                      = ice_mtu_set,
  60         .mac_addr_set                 = ice_macaddr_set,
  61         .mac_addr_add                 = ice_macaddr_add,
  62         .mac_addr_remove              = ice_macaddr_remove,
  63         .rxq_info_get                 = ice_rxq_info_get,
  64         .txq_info_get                 = ice_txq_info_get,
  65         .rx_queue_count               = ice_rx_queue_count,
  66 };
  67
  68 static void
  69 ice_init_controlq_parameter(struct ice_hw *hw)
  70 {
  71         /* fields for adminq */
  72         hw->adminq.num_rq_entries = ICE_ADMINQ_LEN;
  73         hw->adminq.num_sq_entries = ICE_ADMINQ_LEN;
  74         hw->adminq.rq_buf_size = ICE_ADMINQ_BUF_SZ;
  75         hw->adminq.sq_buf_size = ICE_ADMINQ_BUF_SZ;
  76
  77         /* fields for mailboxq, DPDK used as PF host */
  78         hw->mailboxq.num_rq_entries = ICE_MAILBOXQ_LEN;
  79         hw->mailboxq.num_sq_entries = ICE_MAILBOXQ_LEN;
  80         hw->mailboxq.rq_buf_size = ICE_MAILBOXQ_BUF_SZ;
  81         hw->mailboxq.sq_buf_size = ICE_MAILBOXQ_BUF_SZ;
  82 }
  83
  84 static int
  85 ice_check_qp_num(const char *key, const char *qp_value,
  86                  __rte_unused void *opaque)
  87 {
  88         char *end = NULL;
  89         int num = 0;
  90
  91         while (isblank(*qp_value))
  92                 qp_value++;
  93
  94         num = strtoul(qp_value, &end, 10);
  95
  96         if (!num || (*end == '-') || errno) {
  97                 PMD_DRV_LOG(WARNING, "invalid value:\"%s\" for key:\"%s\", "
  98                             "value must be > 0",
  99                             qp_value, key);
 100                 return -1;
 101         }
 102
 103         return num;
 104 }
 105
 106 static int
 107 ice_config_max_queue_pair_num(struct rte_devargs *devargs)
 108 {
 109         struct rte_kvargs *kvlist;
 110         const char *queue_num_key = ICE_MAX_QP_NUM;
 111         int ret;
 112
 113         if (!devargs)
 114                 return 0;
 115
 116         kvlist = rte_kvargs_parse(devargs->args, NULL);
 117         if (!kvlist)
 118                 return 0;
 119
 120         if (!rte_kvargs_count(kvlist, queue_num_key)) {
 121                 rte_kvargs_free(kvlist);
 122                 return 0;
 123         }
 124
 125         if (rte_kvargs_process(kvlist, queue_num_key,
 126                                ice_check_qp_num, NULL) < 0) {
 127                 rte_kvargs_free(kvlist);
 128                 return 0;
 129         }
 130         ret = rte_kvargs_process(kvlist, queue_num_key,
 131                                  ice_check_qp_num, NULL);
 132         rte_kvargs_free(kvlist);
 133
 134         return ret;
 135 }
 136
 137 static int
 138 ice_res_pool_init(struct ice_res_pool_info *pool, uint32_t base,
 139                   uint32_t num)
 140 {
 141         struct pool_entry *entry;
 142
 143         if (!pool || !num)
 144                 return -EINVAL;
 145
 146         entry = rte_zmalloc(NULL, sizeof(*entry), 0);
 147         if (!entry) {
 148                 PMD_INIT_LOG(ERR,
 149                              "Failed to allocate memory for resource pool");
 150                 return -ENOMEM;
 151         }
 152
 153         /* queue heap initialize */
 154         pool->num_free = num;
 155         pool->num_alloc = 0;
 156         pool->base = base;
 157         LIST_INIT(&pool->alloc_list);
 158         LIST_INIT(&pool->free_list);
 159
 160         /* Initialize element  */
 161         entry->base = 0;
 162         entry->len = num;
 163
 164         LIST_INSERT_HEAD(&pool->free_list, entry, next);
 165         return 0;
 166 }
 167
 168 static int
 169 ice_res_pool_alloc(struct ice_res_pool_info *pool,
 170                    uint16_t num)
 171 {
 172         struct pool_entry *entry, *valid_entry;
 173
 174         if (!pool || !num) {
 175                 PMD_INIT_LOG(ERR, "Invalid parameter");
 176                 return -EINVAL;
 177         }
 178
 179         if (pool->num_free < num) {
 180                 PMD_INIT_LOG(ERR, "No resource. ask:%u, available:%u",
 181                              num, pool->num_free);
 182                 return -ENOMEM;
 183         }
 184
 185         valid_entry = NULL;
 186         /* Lookup  in free list and find most fit one */
 187         LIST_FOREACH(entry, &pool->free_list, next) {
 188                 if (entry->len >= num) {
 189                         /* Find best one */
 190                         if (entry->len == num) {
 191                                 valid_entry = entry;
 192                                 break;
 193                         }
 194                         if (!valid_entry ||
 195                             valid_entry->len > entry->len)
 196                                 valid_entry = entry;
 197                 }
 198         }
 199
 200         /* Not find one to satisfy the request, return */
 201         if (!valid_entry) {
 202                 PMD_INIT_LOG(ERR, "No valid entry found");
 203                 return -ENOMEM;
 204         }
 205         /**
 206          * The entry have equal queue number as requested,
 207          * remove it from alloc_list.
 208          */
 209         if (valid_entry->len == num) {
 210                 LIST_REMOVE(valid_entry, next);
 211         } else {
 212                 /**
 213                  * The entry have more numbers than requested,
 214                  * create a new entry for alloc_list and minus its
 215                  * queue base and number in free_list.
 216                  */
 217                 entry = rte_zmalloc(NULL, sizeof(*entry), 0);
 218                 if (!entry) {
 219                         PMD_INIT_LOG(ERR,
 220                                      "Failed to allocate memory for "
 221                                      "resource pool");
 222                         return -ENOMEM;
 223                 }
 224                 entry->base = valid_entry->base;
 225                 entry->len = num;
 226                 valid_entry->base += num;
 227                 valid_entry->len -= num;
 228                 valid_entry = entry;
 229         }
 230
 231         /* Insert it into alloc list, not sorted */
 232         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
 233
 234         pool->num_free -= valid_entry->len;
 235         pool->num_alloc += valid_entry->len;
 236
 237         return valid_entry->base + pool->base;
 238 }
 239
 240 static void
 241 ice_res_pool_destroy(struct ice_res_pool_info *pool)
 242 {
 243         struct pool_entry *entry, *next_entry;
 244
 245         if (!pool)
 246                 return;
 247
 248         for (entry = LIST_FIRST(&pool->alloc_list);
 249              entry && (next_entry = LIST_NEXT(entry, next), 1);
 250              entry = next_entry) {
 251                 LIST_REMOVE(entry, next);
 252                 rte_free(entry);
 253         }
 254
 255         for (entry = LIST_FIRST(&pool->free_list);
 256              entry && (next_entry = LIST_NEXT(entry, next), 1);
 257              entry = next_entry) {
 258                 LIST_REMOVE(entry, next);
 259                 rte_free(entry);
 260         }
 261
 262         pool->num_free = 0;
 263         pool->num_alloc = 0;
 264         pool->base = 0;
 265         LIST_INIT(&pool->alloc_list);
 266         LIST_INIT(&pool->free_list);
 267 }
 268
 269 static void
 270 ice_vsi_config_default_rss(struct ice_aqc_vsi_props *info)
 271 {
 272         /* Set VSI LUT selection */
 273         info->q_opt_rss = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI &
 274                           ICE_AQ_VSI_Q_OPT_RSS_LUT_M;
 275         /* Set Hash scheme */
 276         info->q_opt_rss |= ICE_AQ_VSI_Q_OPT_RSS_TPLZ &
 277                            ICE_AQ_VSI_Q_OPT_RSS_HASH_M;
 278         /* enable TC */
 279         info->q_opt_tc = ICE_AQ_VSI_Q_OPT_TC_OVR_M;
 280 }
 281
 282 static enum ice_status
 283 ice_vsi_config_tc_queue_mapping(struct ice_vsi *vsi,
 284                                 struct ice_aqc_vsi_props *info,
 285                                 uint8_t enabled_tcmap)
 286 {
 287         uint16_t bsf, qp_idx;
 288
 289         /* default tc 0 now. Multi-TC supporting need to be done later.
 290          * Configure TC and queue mapping parameters, for enabled TC,
 291          * allocate qpnum_per_tc queues to this traffic.
 292          */
 293         if (enabled_tcmap != 0x01) {
 294                 PMD_INIT_LOG(ERR, "only TC0 is supported");
 295                 return -ENOTSUP;
 296         }
 297
 298         vsi->nb_qps = RTE_MIN(vsi->nb_qps, ICE_MAX_Q_PER_TC);
 299         bsf = rte_bsf32(vsi->nb_qps);
 300         /* Adjust the queue number to actual queues that can be applied */
 301         vsi->nb_qps = 0x1 << bsf;
 302
 303         qp_idx = 0;
 304         /* Set tc and queue mapping with VSI */
 305         info->tc_mapping[0] = rte_cpu_to_le_16((qp_idx <<
 306                                                 ICE_AQ_VSI_TC_Q_OFFSET_S) |
 307                                                (bsf << ICE_AQ_VSI_TC_Q_NUM_S));
 308
 309         /* Associate queue number with VSI */
 310         info->mapping_flags |= rte_cpu_to_le_16(ICE_AQ_VSI_Q_MAP_CONTIG);
 311         info->q_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
 312         info->q_mapping[1] = rte_cpu_to_le_16(vsi->nb_qps);
 313         info->valid_sections |=
 314                 rte_cpu_to_le_16(ICE_AQ_VSI_PROP_RXQ_MAP_VALID);
 315         /* Set the info.ingress_table and info.egress_table
 316          * for UP translate table. Now just set it to 1:1 map by default
 317          * -- 0b 111 110 101 100 011 010 001 000 == 0xFAC688
 318          */
 319 #define ICE_TC_QUEUE_TABLE_DFLT 0x00FAC688
 320         info->ingress_table  = rte_cpu_to_le_32(ICE_TC_QUEUE_TABLE_DFLT);
 321         info->egress_table   = rte_cpu_to_le_32(ICE_TC_QUEUE_TABLE_DFLT);
 322         info->outer_up_table = rte_cpu_to_le_32(ICE_TC_QUEUE_TABLE_DFLT);
 323         return 0;
 324 }
 325
 326 static int
 327 ice_init_mac_address(struct rte_eth_dev *dev)
 328 {
 329         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 330
 331         if (!is_unicast_ether_addr
 332                 ((struct ether_addr *)hw->port_info[0].mac.lan_addr)) {
 333                 PMD_INIT_LOG(ERR, "Invalid MAC address");
 334                 return -EINVAL;
 335         }
 336
 337         ether_addr_copy((struct ether_addr *)hw->port_info[0].mac.lan_addr,
 338                         (struct ether_addr *)hw->port_info[0].mac.perm_addr);
 339
 340         dev->data->mac_addrs = rte_zmalloc(NULL, sizeof(struct ether_addr), 0);
 341         if (!dev->data->mac_addrs) {
 342                 PMD_INIT_LOG(ERR,
 343                              "Failed to allocate memory to store mac address");
 344                 return -ENOMEM;
 345         }
 346         /* store it to dev data */
 347         ether_addr_copy((struct ether_addr *)hw->port_info[0].mac.perm_addr,
 348                         &dev->data->mac_addrs[0]);
 349         return 0;
 350 }
 351
 352 /* Find out specific MAC filter */
 353 static struct ice_mac_filter *
 354 ice_find_mac_filter(struct ice_vsi *vsi, struct ether_addr *macaddr)
 355 {
 356         struct ice_mac_filter *f;
 357
 358         TAILQ_FOREACH(f, &vsi->mac_list, next) {
 359                 if (is_same_ether_addr(macaddr, &f->mac_info.mac_addr))
 360                         return f;
 361         }
 362
 363         return NULL;
 364 }
 365
 366 static int
 367 ice_add_mac_filter(struct ice_vsi *vsi, struct ether_addr *mac_addr)
 368 {
 369         struct ice_fltr_list_entry *m_list_itr = NULL;
 370         struct ice_mac_filter *f;
 371         struct LIST_HEAD_TYPE list_head;
 372         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
 373         int ret = 0;
 374
 375         /* If it's added and configured, return */
 376         f = ice_find_mac_filter(vsi, mac_addr);
 377         if (f) {
 378                 PMD_DRV_LOG(INFO, "This MAC filter already exists.");
 379                 return 0;
 380         }
 381
 382         INIT_LIST_HEAD(&list_head);
 383
 384         m_list_itr = (struct ice_fltr_list_entry *)
 385                 ice_malloc(hw, sizeof(*m_list_itr));
 386         if (!m_list_itr) {
 387                 ret = -ENOMEM;
 388                 goto DONE;
 389         }
 390         ice_memcpy(m_list_itr->fltr_info.l_data.mac.mac_addr,
 391                    mac_addr, ETH_ALEN, ICE_NONDMA_TO_NONDMA);
 392         m_list_itr->fltr_info.src_id = ICE_SRC_ID_VSI;
 393         m_list_itr->fltr_info.fltr_act = ICE_FWD_TO_VSI;
 394         m_list_itr->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
 395         m_list_itr->fltr_info.flag = ICE_FLTR_TX;
 396         m_list_itr->fltr_info.vsi_handle = vsi->idx;
 397
 398         LIST_ADD(&m_list_itr->list_entry, &list_head);
 399
 400         /* Add the mac */
 401         ret = ice_add_mac(hw, &list_head);
 402         if (ret != ICE_SUCCESS) {
 403                 PMD_DRV_LOG(ERR, "Failed to add MAC filter");
 404                 ret = -EINVAL;
 405                 goto DONE;
 406         }
 407         /* Add the mac addr into mac list */
 408         f = rte_zmalloc(NULL, sizeof(*f), 0);
 409         if (!f) {
 410                 PMD_DRV_LOG(ERR, "failed to allocate memory");
 411                 ret = -ENOMEM;
 412                 goto DONE;
 413         }
 414         rte_memcpy(&f->mac_info.mac_addr, mac_addr, ETH_ADDR_LEN);
 415         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
 416         vsi->mac_num++;
 417
 418         ret = 0;
 419
 420 DONE:
 421         rte_free(m_list_itr);
 422         return ret;
 423 }
 424
 425 static int
 426 ice_remove_mac_filter(struct ice_vsi *vsi, struct ether_addr *mac_addr)
 427 {
 428         struct ice_fltr_list_entry *m_list_itr = NULL;
 429         struct ice_mac_filter *f;
 430         struct LIST_HEAD_TYPE list_head;
 431         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
 432         int ret = 0;
 433
 434         /* Can't find it, return an error */
 435         f = ice_find_mac_filter(vsi, mac_addr);
 436         if (!f)
 437                 return -EINVAL;
 438
 439         INIT_LIST_HEAD(&list_head);
 440
 441         m_list_itr = (struct ice_fltr_list_entry *)
 442                 ice_malloc(hw, sizeof(*m_list_itr));
 443         if (!m_list_itr) {
 444                 ret = -ENOMEM;
 445                 goto DONE;
 446         }
 447         ice_memcpy(m_list_itr->fltr_info.l_data.mac.mac_addr,
 448                    mac_addr, ETH_ALEN, ICE_NONDMA_TO_NONDMA);
 449         m_list_itr->fltr_info.src_id = ICE_SRC_ID_VSI;
 450         m_list_itr->fltr_info.fltr_act = ICE_FWD_TO_VSI;
 451         m_list_itr->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
 452         m_list_itr->fltr_info.flag = ICE_FLTR_TX;
 453         m_list_itr->fltr_info.vsi_handle = vsi->idx;
 454
 455         LIST_ADD(&m_list_itr->list_entry, &list_head);
 456
 457         /* remove the mac filter */
 458         ret = ice_remove_mac(hw, &list_head);
 459         if (ret != ICE_SUCCESS) {
 460                 PMD_DRV_LOG(ERR, "Failed to remove MAC filter");
 461                 ret = -EINVAL;
 462                 goto DONE;
 463         }
 464
 465         /* Remove the mac addr from mac list */
 466         TAILQ_REMOVE(&vsi->mac_list, f, next);
 467         rte_free(f);
 468         vsi->mac_num--;
 469
 470         ret = 0;
 471 DONE:
 472         rte_free(m_list_itr);
 473         return ret;
 474 }
 475
 476 /* Enable IRQ0 */
 477 static void
 478 ice_pf_enable_irq0(struct ice_hw *hw)
 479 {
 480         /* reset the registers */
 481         ICE_WRITE_REG(hw, PFINT_OICR_ENA, 0);
 482         ICE_READ_REG(hw, PFINT_OICR);
 483
 484 #ifdef ICE_LSE_SPT
 485         ICE_WRITE_REG(hw, PFINT_OICR_ENA,
 486                       (uint32_t)(PFINT_OICR_ENA_INT_ENA_M &
 487                                  (~PFINT_OICR_LINK_STAT_CHANGE_M)));
 488
 489         ICE_WRITE_REG(hw, PFINT_OICR_CTL,
 490                       (0 & PFINT_OICR_CTL_MSIX_INDX_M) |
 491                       ((0 << PFINT_OICR_CTL_ITR_INDX_S) &
 492                        PFINT_OICR_CTL_ITR_INDX_M) |
 493                       PFINT_OICR_CTL_CAUSE_ENA_M);
 494
 495         ICE_WRITE_REG(hw, PFINT_FW_CTL,
 496                       (0 & PFINT_FW_CTL_MSIX_INDX_M) |
 497                       ((0 << PFINT_FW_CTL_ITR_INDX_S) &
 498                        PFINT_FW_CTL_ITR_INDX_M) |
 499                       PFINT_FW_CTL_CAUSE_ENA_M);
 500 #else
 501         ICE_WRITE_REG(hw, PFINT_OICR_ENA, PFINT_OICR_ENA_INT_ENA_M);
 502 #endif
 503
 504         ICE_WRITE_REG(hw, GLINT_DYN_CTL(0),
 505                       GLINT_DYN_CTL_INTENA_M |
 506                       GLINT_DYN_CTL_CLEARPBA_M |
 507                       GLINT_DYN_CTL_ITR_INDX_M);
 508
 509         ice_flush(hw);
 510 }
 511
 512 /* Disable IRQ0 */
 513 static void
 514 ice_pf_disable_irq0(struct ice_hw *hw)
 515 {
 516         /* Disable all interrupt types */
 517         ICE_WRITE_REG(hw, GLINT_DYN_CTL(0), GLINT_DYN_CTL_WB_ON_ITR_M);
 518         ice_flush(hw);
 519 }
 520
 521 #ifdef ICE_LSE_SPT
 522 static void
 523 ice_handle_aq_msg(struct rte_eth_dev *dev)
 524 {
 525         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 526         struct ice_ctl_q_info *cq = &hw->adminq;
 527         struct ice_rq_event_info event;
 528         uint16_t pending, opcode;
 529         int ret;
 530
 531         event.buf_len = ICE_AQ_MAX_BUF_LEN;
 532         event.msg_buf = rte_zmalloc(NULL, event.buf_len, 0);
 533         if (!event.msg_buf) {
 534                 PMD_DRV_LOG(ERR, "Failed to allocate mem");
 535                 return;
 536         }
 537
 538         pending = 1;
 539         while (pending) {
 540                 ret = ice_clean_rq_elem(hw, cq, &event, &pending);
 541
 542                 if (ret != ICE_SUCCESS) {
 543                         PMD_DRV_LOG(INFO,
 544                                     "Failed to read msg from AdminQ, "
 545                                     "adminq_err: %u",
 546                                     hw->adminq.sq_last_status);
 547                         break;
 548                 }
 549                 opcode = rte_le_to_cpu_16(event.desc.opcode);
 550
 551                 switch (opcode) {
 552                 case ice_aqc_opc_get_link_status:
 553                         ret = ice_link_update(dev, 0);
 554                         if (!ret)
 555                                 _rte_eth_dev_callback_process
 556                                         (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 557                         break;
 558                 default:
 559                         PMD_DRV_LOG(DEBUG, "Request %u is not supported yet",
 560                                     opcode);
 561                         break;
 562                 }
 563         }
 564         rte_free(event.msg_buf);
 565 }
 566 #endif
 567
 568 /**
 569  * Interrupt handler triggered by NIC for handling
 570  * specific interrupt.
 571  *
 572  * @param handle
 573  *  Pointer to interrupt handle.
 574  * @param param
 575  *  The address of parameter (struct rte_eth_dev *) regsitered before.
 576  *
 577  * @return
 578  *  void
 579  */
 580 static void
 581 ice_interrupt_handler(void *param)
 582 {
 583         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 584         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 585         uint32_t oicr;
 586         uint32_t reg;
 587         uint8_t pf_num;
 588         uint8_t event;
 589         uint16_t queue;
 590 #ifdef ICE_LSE_SPT
 591         uint32_t int_fw_ctl;
 592 #endif
 593
 594         /* Disable interrupt */
 595         ice_pf_disable_irq0(hw);
 596
 597         /* read out interrupt causes */
 598         oicr = ICE_READ_REG(hw, PFINT_OICR);
 599 #ifdef ICE_LSE_SPT
 600         int_fw_ctl = ICE_READ_REG(hw, PFINT_FW_CTL);
 601 #endif
 602
 603         /* No interrupt event indicated */
 604         if (!(oicr & PFINT_OICR_INTEVENT_M)) {
 605                 PMD_DRV_LOG(INFO, "No interrupt event");
 606                 goto done;
 607         }
 608
 609 #ifdef ICE_LSE_SPT
 610         if (int_fw_ctl & PFINT_FW_CTL_INTEVENT_M) {
 611                 PMD_DRV_LOG(INFO, "FW_CTL: link state change event");
 612                 ice_handle_aq_msg(dev);
 613         }
 614 #else
 615         if (oicr & PFINT_OICR_LINK_STAT_CHANGE_M) {
 616                 PMD_DRV_LOG(INFO, "OICR: link state change event");
 617                 ice_link_update(dev, 0);
 618         }
 619 #endif
 620
 621         if (oicr & PFINT_OICR_MAL_DETECT_M) {
 622                 PMD_DRV_LOG(WARNING, "OICR: MDD event");
 623                 reg = ICE_READ_REG(hw, GL_MDET_TX_PQM);
 624                 if (reg & GL_MDET_TX_PQM_VALID_M) {
 625                         pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >>
 626                                  GL_MDET_TX_PQM_PF_NUM_S;
 627                         event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >>
 628                                 GL_MDET_TX_PQM_MAL_TYPE_S;
 629                         queue = (reg & GL_MDET_TX_PQM_QNUM_M) >>
 630                                 GL_MDET_TX_PQM_QNUM_S;
 631
 632                         PMD_DRV_LOG(WARNING, "Malicious Driver Detection event "
 633                                     "%d by PQM on TX queue %d PF# %d",
 634                                     event, queue, pf_num);
 635                 }
 636
 637                 reg = ICE_READ_REG(hw, GL_MDET_TX_TCLAN);
 638                 if (reg & GL_MDET_TX_TCLAN_VALID_M) {
 639                         pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >>
 640                                  GL_MDET_TX_TCLAN_PF_NUM_S;
 641                         event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >>
 642                                 GL_MDET_TX_TCLAN_MAL_TYPE_S;
 643                         queue = (reg & GL_MDET_TX_TCLAN_QNUM_M) >>
 644                                 GL_MDET_TX_TCLAN_QNUM_S;
 645
 646                         PMD_DRV_LOG(WARNING, "Malicious Driver Detection event "
 647                                     "%d by TCLAN on TX queue %d PF# %d",
 648                                     event, queue, pf_num);
 649                 }
 650         }
 651 done:
 652         /* Enable interrupt */
 653         ice_pf_enable_irq0(hw);
 654         rte_intr_enable(dev->intr_handle);
 655 }
 656
 657 /*  Initialize SW parameters of PF */
 658 static int
 659 ice_pf_sw_init(struct rte_eth_dev *dev)
 660 {
 661         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 662         struct ice_hw *hw = ICE_PF_TO_HW(pf);
 663
 664         if (ice_config_max_queue_pair_num(dev->device->devargs) > 0)
 665                 pf->lan_nb_qp_max =
 666                         ice_config_max_queue_pair_num(dev->device->devargs);
 667         else
 668                 pf->lan_nb_qp_max =
 669                         (uint16_t)RTE_MIN(hw->func_caps.common_cap.num_txq,
 670                                           hw->func_caps.common_cap.num_rxq);
 671
 672         pf->lan_nb_qps = pf->lan_nb_qp_max;
 673
 674         return 0;
 675 }
 676
 677 static struct ice_vsi *
 678 ice_setup_vsi(struct ice_pf *pf, enum ice_vsi_type type)
 679 {
 680         struct ice_hw *hw = ICE_PF_TO_HW(pf);
 681         struct ice_vsi *vsi = NULL;
 682         struct ice_vsi_ctx vsi_ctx;
 683         int ret;
 684         struct ether_addr broadcast = {
 685                 .addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff} };
 686         struct ether_addr mac_addr;
 687         uint16_t max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
 688         uint8_t tc_bitmap = 0x1;
 689
 690         /* hw->num_lports = 1 in NIC mode */
 691         vsi = rte_zmalloc(NULL, sizeof(struct ice_vsi), 0);
 692         if (!vsi)
 693                 return NULL;
 694
 695         vsi->idx = pf->next_vsi_idx;
 696         pf->next_vsi_idx++;
 697         vsi->type = type;
 698         vsi->adapter = ICE_PF_TO_ADAPTER(pf);
 699         vsi->max_macaddrs = ICE_NUM_MACADDR_MAX;
 700         vsi->vlan_anti_spoof_on = 0;
 701         vsi->vlan_filter_on = 1;
 702         TAILQ_INIT(&vsi->mac_list);
 703         TAILQ_INIT(&vsi->vlan_list);
 704
 705         memset(&vsi_ctx, 0, sizeof(vsi_ctx));
 706         /* base_queue in used in queue mapping of VSI add/update command.
 707          * Suppose vsi->base_queue is 0 now, don't consider SRIOV, VMDQ
 708          * cases in the first stage. Only Main VSI.
 709          */
 710         vsi->base_queue = 0;
 711         switch (type) {
 712         case ICE_VSI_PF:
 713                 vsi->nb_qps = pf->lan_nb_qps;
 714                 ice_vsi_config_default_rss(&vsi_ctx.info);
 715                 vsi_ctx.alloc_from_pool = true;
 716                 vsi_ctx.flags = ICE_AQ_VSI_TYPE_PF;
 717                 /* switch_id is queried by get_switch_config aq, which is done
 718                  * by ice_init_hw
 719                  */
 720                 vsi_ctx.info.sw_id = hw->port_info->sw_id;
 721                 vsi_ctx.info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA;
 722                 /* Allow all untagged or tagged packets */
 723                 vsi_ctx.info.vlan_flags = ICE_AQ_VSI_VLAN_MODE_ALL;
 724                 vsi_ctx.info.vlan_flags |= ICE_AQ_VSI_VLAN_EMOD_NOTHING;
 725                 vsi_ctx.info.q_opt_rss = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF |
 726                                          ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
 727                 /* Enable VLAN/UP trip */
 728                 ret = ice_vsi_config_tc_queue_mapping(vsi,
 729                                                       &vsi_ctx.info,
 730                                                       ICE_DEFAULT_TCMAP);
 731                 if (ret) {
 732                         PMD_INIT_LOG(ERR,
 733                                      "tc queue mapping with vsi failed, "
 734                                      "err = %d",
 735                                      ret);
 736                         goto fail_mem;
 737                 }
 738
 739                 break;
 740         default:
 741                 /* for other types of VSI */
 742                 PMD_INIT_LOG(ERR, "other types of VSI not supported");
 743                 goto fail_mem;
 744         }
 745
 746         /* VF has MSIX interrupt in VF range, don't allocate here */
 747         if (type == ICE_VSI_PF) {
 748                 ret = ice_res_pool_alloc(&pf->msix_pool,
 749                                          RTE_MIN(vsi->nb_qps,
 750                                                  RTE_MAX_RXTX_INTR_VEC_ID));
 751                 if (ret < 0) {
 752                         PMD_INIT_LOG(ERR, "VSI MAIN %d get heap failed %d",
 753                                      vsi->vsi_id, ret);
 754                 }
 755                 vsi->msix_intr = ret;
 756                 vsi->nb_msix = RTE_MIN(vsi->nb_qps, RTE_MAX_RXTX_INTR_VEC_ID);
 757         } else {
 758                 vsi->msix_intr = 0;
 759                 vsi->nb_msix = 0;
 760         }
 761         ret = ice_add_vsi(hw, vsi->idx, &vsi_ctx, NULL);
 762         if (ret != ICE_SUCCESS) {
 763                 PMD_INIT_LOG(ERR, "add vsi failed, err = %d", ret);
 764                 goto fail_mem;
 765         }
 766         /* store vsi information is SW structure */
 767         vsi->vsi_id = vsi_ctx.vsi_num;
 768         vsi->info = vsi_ctx.info;
 769         pf->vsis_allocated = vsi_ctx.vsis_allocd;
 770         pf->vsis_unallocated = vsi_ctx.vsis_unallocated;
 771
 772         /* MAC configuration */
 773         rte_memcpy(pf->dev_addr.addr_bytes,
 774                    hw->port_info->mac.perm_addr,
 775                    ETH_ADDR_LEN);
 776
 777         rte_memcpy(&mac_addr, &pf->dev_addr, ETHER_ADDR_LEN);
 778         ret = ice_add_mac_filter(vsi, &mac_addr);
 779         if (ret != ICE_SUCCESS)
 780                 PMD_INIT_LOG(ERR, "Failed to add dflt MAC filter");
 781
 782         rte_memcpy(&mac_addr, &broadcast, ETHER_ADDR_LEN);
 783         ret = ice_add_mac_filter(vsi, &mac_addr);
 784         if (ret != ICE_SUCCESS)
 785                 PMD_INIT_LOG(ERR, "Failed to add MAC filter");
 786
 787         /* At the beginning, only TC0. */
 788         /* What we need here is the maximam number of the TX queues.
 789          * Currently vsi->nb_qps means it.
 790          * Correct it if any change.
 791          */
 792         max_txqs[0] = vsi->nb_qps;
 793         ret = ice_cfg_vsi_lan(hw->port_info, vsi->idx,
 794                               tc_bitmap, max_txqs);
 795         if (ret != ICE_SUCCESS)
 796                 PMD_INIT_LOG(ERR, "Failed to config vsi sched");
 797
 798         return vsi;
 799 fail_mem:
 800         rte_free(vsi);
 801         pf->next_vsi_idx--;
 802         return NULL;
 803 }
 804
 805 static int
 806 ice_pf_setup(struct ice_pf *pf)
 807 {
 808         struct ice_vsi *vsi;
 809
 810         /* Clear all stats counters */
 811         pf->offset_loaded = FALSE;
 812         memset(&pf->stats, 0, sizeof(struct ice_hw_port_stats));
 813         memset(&pf->stats_offset, 0, sizeof(struct ice_hw_port_stats));
 814         memset(&pf->internal_stats, 0, sizeof(struct ice_eth_stats));
 815         memset(&pf->internal_stats_offset, 0, sizeof(struct ice_eth_stats));
 816
 817         vsi = ice_setup_vsi(pf, ICE_VSI_PF);
 818         if (!vsi) {
 819                 PMD_INIT_LOG(ERR, "Failed to add vsi for PF");
 820                 return -EINVAL;
 821         }
 822
 823         pf->main_vsi = vsi;
 824
 825         return 0;
 826 }
 827
 828 static int
 829 ice_dev_init(struct rte_eth_dev *dev)
 830 {
 831         struct rte_pci_device *pci_dev;
 832         struct rte_intr_handle *intr_handle;
 833         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 834         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 835         int ret;
 836
 837         dev->dev_ops = &ice_eth_dev_ops;
 838         dev->rx_pkt_burst = ice_recv_pkts;
 839         dev->tx_pkt_burst = ice_xmit_pkts;
 840         dev->tx_pkt_prepare = ice_prep_pkts;
 841
 842         ice_set_default_ptype_table(dev);
 843         pci_dev = RTE_DEV_TO_PCI(dev->device);
 844         intr_handle = &pci_dev->intr_handle;
 845
 846         pf->adapter = ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 847         pf->adapter->eth_dev = dev;
 848         pf->dev_data = dev->data;
 849         hw->back = pf->adapter;
 850         hw->hw_addr = (uint8_t *)pci_dev->mem_resource[0].addr;
 851         hw->vendor_id = pci_dev->id.vendor_id;
 852         hw->device_id = pci_dev->id.device_id;
 853         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 854         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 855         hw->bus.device = pci_dev->addr.devid;
 856         hw->bus.func = pci_dev->addr.function;
 857
 858         ice_init_controlq_parameter(hw);
 859
 860         ret = ice_init_hw(hw);
 861         if (ret) {
 862                 PMD_INIT_LOG(ERR, "Failed to initialize HW");
 863                 return -EINVAL;
 864         }
 865
 866         PMD_INIT_LOG(INFO, "FW %d.%d.%05d API %d.%d",
 867                      hw->fw_maj_ver, hw->fw_min_ver, hw->fw_build,
 868                      hw->api_maj_ver, hw->api_min_ver);
 869
 870         ice_pf_sw_init(dev);
 871         ret = ice_init_mac_address(dev);
 872         if (ret) {
 873                 PMD_INIT_LOG(ERR, "Failed to initialize mac address");
 874                 goto err_init_mac;
 875         }
 876
 877         ret = ice_res_pool_init(&pf->msix_pool, 1,
 878                                 hw->func_caps.common_cap.num_msix_vectors - 1);
 879         if (ret) {
 880                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
 881                 goto err_msix_pool_init;
 882         }
 883
 884         ret = ice_pf_setup(pf);
 885         if (ret) {
 886                 PMD_INIT_LOG(ERR, "Failed to setup PF");
 887                 goto err_pf_setup;
 888         }
 889
 890         /* register callback func to eal lib */
 891         rte_intr_callback_register(intr_handle,
 892                                    ice_interrupt_handler, dev);
 893
 894         ice_pf_enable_irq0(hw);
 895
 896         /* enable uio intr after callback register */
 897         rte_intr_enable(intr_handle);
 898
 899         return 0;
 900
 901 err_pf_setup:
 902         ice_res_pool_destroy(&pf->msix_pool);
 903 err_msix_pool_init:
 904         rte_free(dev->data->mac_addrs);
 905 err_init_mac:
 906         ice_sched_cleanup_all(hw);
 907         rte_free(hw->port_info);
 908         ice_shutdown_all_ctrlq(hw);
 909
 910         return ret;
 911 }
 912
 913 static int
 914 ice_release_vsi(struct ice_vsi *vsi)
 915 {
 916         struct ice_hw *hw;
 917         struct ice_vsi_ctx vsi_ctx;
 918         enum ice_status ret;
 919
 920         if (!vsi)
 921                 return 0;
 922
 923         hw = ICE_VSI_TO_HW(vsi);
 924
 925         memset(&vsi_ctx, 0, sizeof(vsi_ctx));
 926
 927         vsi_ctx.vsi_num = vsi->vsi_id;
 928         vsi_ctx.info = vsi->info;
 929         ret = ice_free_vsi(hw, vsi->idx, &vsi_ctx, false, NULL);
 930         if (ret != ICE_SUCCESS) {
 931                 PMD_INIT_LOG(ERR, "Failed to free vsi by aq, %u", vsi->vsi_id);
 932                 rte_free(vsi);
 933                 return -1;
 934         }
 935
 936         rte_free(vsi);
 937         return 0;
 938 }
 939
 940 static void
 941 ice_dev_stop(struct rte_eth_dev *dev)
 942 {
 943         struct rte_eth_dev_data *data = dev->data;
 944         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 945         struct rte_pci_device *pci_dev = ICE_DEV_TO_PCI(dev);
 946         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 947         uint16_t i;
 948
 949         /* avoid stopping again */
 950         if (pf->adapter_stopped)
 951                 return;
 952
 953         /* stop and clear all Rx queues */
 954         for (i = 0; i < data->nb_rx_queues; i++)
 955                 ice_rx_queue_stop(dev, i);
 956
 957         /* stop and clear all Tx queues */
 958         for (i = 0; i < data->nb_tx_queues; i++)
 959                 ice_tx_queue_stop(dev, i);
 960
 961         /* Clear all queues and release mbufs */
 962         ice_clear_queues(dev);
 963
 964         /* Clean datapath event and queue/vec mapping */
 965         rte_intr_efd_disable(intr_handle);
 966         if (intr_handle->intr_vec) {
 967                 rte_free(intr_handle->intr_vec);
 968                 intr_handle->intr_vec = NULL;
 969         }
 970
 971         pf->adapter_stopped = true;
 972 }
 973
 974 static void
 975 ice_dev_close(struct rte_eth_dev *dev)
 976 {
 977         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 978         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 979
 980         ice_dev_stop(dev);
 981
 982         /* release all queue resource */
 983         ice_free_queues(dev);
 984
 985         ice_res_pool_destroy(&pf->msix_pool);
 986         ice_release_vsi(pf->main_vsi);
 987
 988         ice_shutdown_all_ctrlq(hw);
 989 }
 990
 991 static int
 992 ice_dev_uninit(struct rte_eth_dev *dev)
 993 {
 994         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 995         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 996         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 997         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 998
 999         ice_dev_close(dev);
1000
1001         dev->dev_ops = NULL;
1002         dev->rx_pkt_burst = NULL;
1003         dev->tx_pkt_burst = NULL;
1004
1005         rte_free(dev->data->mac_addrs);
1006         dev->data->mac_addrs = NULL;
1007
1008         /* disable uio intr before callback unregister */
1009         rte_intr_disable(intr_handle);
1010
1011         /* register callback func to eal lib */
1012         rte_intr_callback_unregister(intr_handle,
1013                                      ice_interrupt_handler, dev);
1014
1015         ice_release_vsi(pf->main_vsi);
1016         ice_sched_cleanup_all(hw);
1017         rte_free(hw->port_info);
1018         ice_shutdown_all_ctrlq(hw);
1019
1020         return 0;
1021 }
1022
1023 static int
1024 ice_dev_configure(__rte_unused struct rte_eth_dev *dev)
1025 {
1026         struct ice_adapter *ad =
1027                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1028
1029         /* Initialize to TRUE. If any of Rx queues doesn't meet the
1030          * bulk allocation or vector Rx preconditions we will reset it.
1031          */
1032         ad->rx_bulk_alloc_allowed = true;
1033         ad->tx_simple_allowed = true;
1034
1035         return 0;
1036 }
1037
1038 static int ice_init_rss(struct ice_pf *pf)
1039 {
1040         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1041         struct ice_vsi *vsi = pf->main_vsi;
1042         struct rte_eth_dev *dev = pf->adapter->eth_dev;
1043         struct rte_eth_rss_conf *rss_conf;
1044         struct ice_aqc_get_set_rss_keys key;
1045         uint16_t i, nb_q;
1046         int ret = 0;
1047
1048         rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1049         nb_q = dev->data->nb_rx_queues;
1050         vsi->rss_key_size = ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE;
1051         vsi->rss_lut_size = hw->func_caps.common_cap.rss_table_size;
1052
1053         if (!vsi->rss_key)
1054                 vsi->rss_key = rte_zmalloc(NULL,
1055                                            vsi->rss_key_size, 0);
1056         if (!vsi->rss_lut)
1057                 vsi->rss_lut = rte_zmalloc(NULL,
1058                                            vsi->rss_lut_size, 0);
1059
1060         /* configure RSS key */
1061         if (!rss_conf->rss_key) {
1062                 /* Calculate the default hash key */
1063                 for (i = 0; i <= vsi->rss_key_size; i++)
1064                         vsi->rss_key[i] = (uint8_t)rte_rand();
1065         } else {
1066                 rte_memcpy(vsi->rss_key, rss_conf->rss_key,
1067                            RTE_MIN(rss_conf->rss_key_len,
1068                                    vsi->rss_key_size));
1069         }
1070         rte_memcpy(key.standard_rss_key, vsi->rss_key, vsi->rss_key_size);
1071         ret = ice_aq_set_rss_key(hw, vsi->idx, &key);
1072         if (ret)
1073                 return -EINVAL;
1074
1075         /* init RSS LUT table */
1076         for (i = 0; i < vsi->rss_lut_size; i++)
1077                 vsi->rss_lut[i] = i % nb_q;
1078
1079         ret = ice_aq_set_rss_lut(hw, vsi->idx,
1080                                  ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF,
1081                                  vsi->rss_lut, vsi->rss_lut_size);
1082         if (ret)
1083                 return -EINVAL;
1084
1085         return 0;
1086 }
1087
1088 static int
1089 ice_dev_start(struct rte_eth_dev *dev)
1090 {
1091         struct rte_eth_dev_data *data = dev->data;
1092         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1094         uint16_t nb_rxq = 0;
1095         uint16_t nb_txq, i;
1096         int ret;
1097
1098         /* program Tx queues' context in hardware */
1099         for (nb_txq = 0; nb_txq < data->nb_tx_queues; nb_txq++) {
1100                 ret = ice_tx_queue_start(dev, nb_txq);
1101                 if (ret) {
1102                         PMD_DRV_LOG(ERR, "fail to start Tx queue %u", nb_txq);
1103                         goto tx_err;
1104                 }
1105         }
1106
1107         /* program Rx queues' context in hardware*/
1108         for (nb_rxq = 0; nb_rxq < data->nb_rx_queues; nb_rxq++) {
1109                 ret = ice_rx_queue_start(dev, nb_rxq);
1110                 if (ret) {
1111                         PMD_DRV_LOG(ERR, "fail to start Rx queue %u", nb_rxq);
1112                         goto rx_err;
1113                 }
1114         }
1115
1116         ret = ice_init_rss(pf);
1117         if (ret) {
1118                 PMD_DRV_LOG(ERR, "Failed to enable rss for PF");
1119                 goto rx_err;
1120         }
1121
1122         ice_set_rx_function(dev);
1123
1124         ret = ice_aq_set_event_mask(hw, hw->port_info->lport,
1125                                     ((u16)(ICE_AQ_LINK_EVENT_LINK_FAULT |
1126                                      ICE_AQ_LINK_EVENT_PHY_TEMP_ALARM |
1127                                      ICE_AQ_LINK_EVENT_EXCESSIVE_ERRORS |
1128                                      ICE_AQ_LINK_EVENT_SIGNAL_DETECT |
1129                                      ICE_AQ_LINK_EVENT_AN_COMPLETED |
1130                                      ICE_AQ_LINK_EVENT_PORT_TX_SUSPENDED)),
1131                                      NULL);
1132         if (ret != ICE_SUCCESS)
1133                 PMD_DRV_LOG(WARNING, "Fail to set phy mask");
1134
1135         /* Call get_link_info aq commond to enable/disable LSE */
1136         ice_link_update(dev, 0);
1137
1138         pf->adapter_stopped = false;
1139
1140         return 0;
1141
1142         /* stop the started queues if failed to start all queues */
1143 rx_err:
1144         for (i = 0; i < nb_rxq; i++)
1145                 ice_rx_queue_stop(dev, i);
1146 tx_err:
1147         for (i = 0; i < nb_txq; i++)
1148                 ice_tx_queue_stop(dev, i);
1149
1150         return -EIO;
1151 }
1152
1153 static int
1154 ice_dev_reset(struct rte_eth_dev *dev)
1155 {
1156         int ret;
1157
1158         if (dev->data->sriov.active)
1159                 return -ENOTSUP;
1160
1161         ret = ice_dev_uninit(dev);
1162         if (ret) {
1163                 PMD_INIT_LOG(ERR, "failed to uninit device, status = %d", ret);
1164                 return -ENXIO;
1165         }
1166
1167         ret = ice_dev_init(dev);
1168         if (ret) {
1169                 PMD_INIT_LOG(ERR, "failed to init device, status = %d", ret);
1170                 return -ENXIO;
1171         }
1172
1173         return 0;
1174 }
1175
1176 static void
1177 ice_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1178 {
1179         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1180         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1181         struct ice_vsi *vsi = pf->main_vsi;
1182         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1183
1184         dev_info->min_rx_bufsize = ICE_BUF_SIZE_MIN;
1185         dev_info->max_rx_pktlen = ICE_FRAME_SIZE_MAX;
1186         dev_info->max_rx_queues = vsi->nb_qps;
1187         dev_info->max_tx_queues = vsi->nb_qps;
1188         dev_info->max_mac_addrs = vsi->max_macaddrs;
1189         dev_info->max_vfs = pci_dev->max_vfs;
1190
1191         dev_info->rx_offload_capa =
1192                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1193                 DEV_RX_OFFLOAD_UDP_CKSUM |
1194                 DEV_RX_OFFLOAD_TCP_CKSUM |
1195                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
1196                 DEV_RX_OFFLOAD_JUMBO_FRAME |
1197                 DEV_RX_OFFLOAD_KEEP_CRC;
1198         dev_info->tx_offload_capa =
1199                 DEV_TX_OFFLOAD_IPV4_CKSUM |
1200                 DEV_TX_OFFLOAD_UDP_CKSUM |
1201                 DEV_TX_OFFLOAD_TCP_CKSUM |
1202                 DEV_TX_OFFLOAD_SCTP_CKSUM |
1203                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1204                 DEV_TX_OFFLOAD_TCP_TSO |
1205                 DEV_TX_OFFLOAD_MULTI_SEGS;
1206         dev_info->rx_queue_offload_capa = 0;
1207         dev_info->tx_queue_offload_capa = 0;
1208
1209         dev_info->reta_size = hw->func_caps.common_cap.rss_table_size;
1210         dev_info->hash_key_size = (VSIQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
1211
1212         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1213                 .rx_thresh = {
1214                         .pthresh = ICE_DEFAULT_RX_PTHRESH,
1215                         .hthresh = ICE_DEFAULT_RX_HTHRESH,
1216                         .wthresh = ICE_DEFAULT_RX_WTHRESH,
1217                 },
1218                 .rx_free_thresh = ICE_DEFAULT_RX_FREE_THRESH,
1219                 .rx_drop_en = 0,
1220                 .offloads = 0,
1221         };
1222
1223         dev_info->default_txconf = (struct rte_eth_txconf) {
1224                 .tx_thresh = {
1225                         .pthresh = ICE_DEFAULT_TX_PTHRESH,
1226                         .hthresh = ICE_DEFAULT_TX_HTHRESH,
1227                         .wthresh = ICE_DEFAULT_TX_WTHRESH,
1228                 },
1229                 .tx_free_thresh = ICE_DEFAULT_TX_FREE_THRESH,
1230                 .tx_rs_thresh = ICE_DEFAULT_TX_RSBIT_THRESH,
1231                 .offloads = 0,
1232         };
1233
1234         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1235                 .nb_max = ICE_MAX_RING_DESC,
1236                 .nb_min = ICE_MIN_RING_DESC,
1237                 .nb_align = ICE_ALIGN_RING_DESC,
1238         };
1239
1240         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1241                 .nb_max = ICE_MAX_RING_DESC,
1242                 .nb_min = ICE_MIN_RING_DESC,
1243                 .nb_align = ICE_ALIGN_RING_DESC,
1244         };
1245
1246         dev_info->speed_capa = ETH_LINK_SPEED_10M |
1247                                ETH_LINK_SPEED_100M |
1248                                ETH_LINK_SPEED_1G |
1249                                ETH_LINK_SPEED_2_5G |
1250                                ETH_LINK_SPEED_5G |
1251                                ETH_LINK_SPEED_10G |
1252                                ETH_LINK_SPEED_20G |
1253                                ETH_LINK_SPEED_25G |
1254                                ETH_LINK_SPEED_40G;
1255
1256         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
1257         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
1258
1259         dev_info->default_rxportconf.burst_size = ICE_RX_MAX_BURST;
1260         dev_info->default_txportconf.burst_size = ICE_TX_MAX_BURST;
1261         dev_info->default_rxportconf.nb_queues = 1;
1262         dev_info->default_txportconf.nb_queues = 1;
1263         dev_info->default_rxportconf.ring_size = ICE_BUF_SIZE_MIN;
1264         dev_info->default_txportconf.ring_size = ICE_BUF_SIZE_MIN;
1265 }
1266
1267 static inline int
1268 ice_atomic_read_link_status(struct rte_eth_dev *dev,
1269                             struct rte_eth_link *link)
1270 {
1271         struct rte_eth_link *dst = link;
1272         struct rte_eth_link *src = &dev->data->dev_link;
1273
1274         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1275                                 *(uint64_t *)src) == 0)
1276                 return -1;
1277
1278         return 0;
1279 }
1280
1281 static inline int
1282 ice_atomic_write_link_status(struct rte_eth_dev *dev,
1283                              struct rte_eth_link *link)
1284 {
1285         struct rte_eth_link *dst = &dev->data->dev_link;
1286         struct rte_eth_link *src = link;
1287
1288         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1289                                 *(uint64_t *)src) == 0)
1290                 return -1;
1291
1292         return 0;
1293 }
1294
1295 static int
1296 ice_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
1297 {
1298 #define CHECK_INTERVAL 100  /* 100ms */
1299 #define MAX_REPEAT_TIME 10  /* 1s (10 * 100ms) in total */
1300         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1301         struct ice_link_status link_status;
1302         struct rte_eth_link link, old;
1303         int status;
1304         unsigned int rep_cnt = MAX_REPEAT_TIME;
1305         bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
1306
1307         memset(&link, 0, sizeof(link));
1308         memset(&old, 0, sizeof(old));
1309         memset(&link_status, 0, sizeof(link_status));
1310         ice_atomic_read_link_status(dev, &old);
1311
1312         do {
1313                 /* Get link status information from hardware */
1314                 status = ice_aq_get_link_info(hw->port_info, enable_lse,
1315                                               &link_status, NULL);
1316                 if (status != ICE_SUCCESS) {
1317                         link.link_speed = ETH_SPEED_NUM_100M;
1318                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1319                         PMD_DRV_LOG(ERR, "Failed to get link info");
1320                         goto out;
1321                 }
1322
1323                 link.link_status = link_status.link_info & ICE_AQ_LINK_UP;
1324                 if (!wait_to_complete || link.link_status)
1325                         break;
1326
1327                 rte_delay_ms(CHECK_INTERVAL);
1328         } while (--rep_cnt);
1329
1330         if (!link.link_status)
1331                 goto out;
1332
1333         /* Full-duplex operation at all supported speeds */
1334         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1335
1336         /* Parse the link status */
1337         switch (link_status.link_speed) {
1338         case ICE_AQ_LINK_SPEED_10MB:
1339                 link.link_speed = ETH_SPEED_NUM_10M;
1340                 break;
1341         case ICE_AQ_LINK_SPEED_100MB:
1342                 link.link_speed = ETH_SPEED_NUM_100M;
1343                 break;
1344         case ICE_AQ_LINK_SPEED_1000MB:
1345                 link.link_speed = ETH_SPEED_NUM_1G;
1346                 break;
1347         case ICE_AQ_LINK_SPEED_2500MB:
1348                 link.link_speed = ETH_SPEED_NUM_2_5G;
1349                 break;
1350         case ICE_AQ_LINK_SPEED_5GB:
1351                 link.link_speed = ETH_SPEED_NUM_5G;
1352                 break;
1353         case ICE_AQ_LINK_SPEED_10GB:
1354                 link.link_speed = ETH_SPEED_NUM_10G;
1355                 break;
1356         case ICE_AQ_LINK_SPEED_20GB:
1357                 link.link_speed = ETH_SPEED_NUM_20G;
1358                 break;
1359         case ICE_AQ_LINK_SPEED_25GB:
1360                 link.link_speed = ETH_SPEED_NUM_25G;
1361                 break;
1362         case ICE_AQ_LINK_SPEED_40GB:
1363                 link.link_speed = ETH_SPEED_NUM_40G;
1364                 break;
1365         case ICE_AQ_LINK_SPEED_UNKNOWN:
1366         default:
1367                 PMD_DRV_LOG(ERR, "Unknown link speed");
1368                 link.link_speed = ETH_SPEED_NUM_NONE;
1369                 break;
1370         }
1371
1372         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
1373                               ETH_LINK_SPEED_FIXED);
1374
1375 out:
1376         ice_atomic_write_link_status(dev, &link);
1377         if (link.link_status == old.link_status)
1378                 return -1;
1379
1380         return 0;
1381 }
1382
1383 static int
1384 ice_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1385 {
1386         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1387         struct rte_eth_dev_data *dev_data = pf->dev_data;
1388         uint32_t frame_size = mtu + ETHER_HDR_LEN
1389                               + ETHER_CRC_LEN + ICE_VLAN_TAG_SIZE;
1390
1391         /* check if mtu is within the allowed range */
1392         if (mtu < ETHER_MIN_MTU || frame_size > ICE_FRAME_SIZE_MAX)
1393                 return -EINVAL;
1394
1395         /* mtu setting is forbidden if port is start */
1396         if (dev_data->dev_started) {
1397                 PMD_DRV_LOG(ERR,
1398                             "port %d must be stopped before configuration",
1399                             dev_data->port_id);
1400                 return -EBUSY;
1401         }
1402
1403         if (frame_size > ETHER_MAX_LEN)
1404                 dev_data->dev_conf.rxmode.offloads |=
1405                         DEV_RX_OFFLOAD_JUMBO_FRAME;
1406         else
1407                 dev_data->dev_conf.rxmode.offloads &=
1408                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1409
1410         dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1411
1412         return 0;
1413 }
1414
1415 static int ice_macaddr_set(struct rte_eth_dev *dev,
1416                            struct ether_addr *mac_addr)
1417 {
1418         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1419         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1420         struct ice_vsi *vsi = pf->main_vsi;
1421         struct ice_mac_filter *f;
1422         uint8_t flags = 0;
1423         int ret;
1424
1425         if (!is_valid_assigned_ether_addr(mac_addr)) {
1426                 PMD_DRV_LOG(ERR, "Tried to set invalid MAC address.");
1427                 return -EINVAL;
1428         }
1429
1430         TAILQ_FOREACH(f, &vsi->mac_list, next) {
1431                 if (is_same_ether_addr(&pf->dev_addr, &f->mac_info.mac_addr))
1432                         break;
1433         }
1434
1435         if (!f) {
1436                 PMD_DRV_LOG(ERR, "Failed to find filter for default mac");
1437                 return -EIO;
1438         }
1439
1440         ret = ice_remove_mac_filter(vsi, &f->mac_info.mac_addr);
1441         if (ret != ICE_SUCCESS) {
1442                 PMD_DRV_LOG(ERR, "Failed to delete mac filter");
1443                 return -EIO;
1444         }
1445         ret = ice_add_mac_filter(vsi, mac_addr);
1446         if (ret != ICE_SUCCESS) {
1447                 PMD_DRV_LOG(ERR, "Failed to add mac filter");
1448                 return -EIO;
1449         }
1450         memcpy(&pf->dev_addr, mac_addr, ETH_ADDR_LEN);
1451
1452         flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
1453         ret = ice_aq_manage_mac_write(hw, mac_addr->addr_bytes, flags, NULL);
1454         if (ret != ICE_SUCCESS)
1455                 PMD_DRV_LOG(ERR, "Failed to set manage mac");
1456
1457         return 0;
1458 }
1459
1460 /* Add a MAC address, and update filters */
1461 static int
1462 ice_macaddr_add(struct rte_eth_dev *dev,
1463                 struct ether_addr *mac_addr,
1464                 __rte_unused uint32_t index,
1465                 __rte_unused uint32_t pool)
1466 {
1467         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1468         struct ice_vsi *vsi = pf->main_vsi;
1469         int ret;
1470
1471         ret = ice_add_mac_filter(vsi, mac_addr);
1472         if (ret != ICE_SUCCESS) {
1473                 PMD_DRV_LOG(ERR, "Failed to add MAC filter");
1474                 return -EINVAL;
1475         }
1476
1477         return ICE_SUCCESS;
1478 }
1479
1480 /* Remove a MAC address, and update filters */
1481 static void
1482 ice_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1483 {
1484         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1485         struct ice_vsi *vsi = pf->main_vsi;
1486         struct rte_eth_dev_data *data = dev->data;
1487         struct ether_addr *macaddr;
1488         int ret;
1489
1490         macaddr = &data->mac_addrs[index];
1491         ret = ice_remove_mac_filter(vsi, macaddr);
1492         if (ret) {
1493                 PMD_DRV_LOG(ERR, "Failed to remove MAC filter");
1494                 return;
1495         }
1496 }
1497
1498 static int
1499 ice_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1500               struct rte_pci_device *pci_dev)
1501 {
1502         return rte_eth_dev_pci_generic_probe(pci_dev,
1503                                              sizeof(struct ice_adapter),
1504                                              ice_dev_init);
1505 }
1506
1507 static int
1508 ice_pci_remove(struct rte_pci_device *pci_dev)
1509 {
1510         return rte_eth_dev_pci_generic_remove(pci_dev, ice_dev_uninit);
1511 }
1512
1513 static struct rte_pci_driver rte_ice_pmd = {
1514         .id_table = pci_id_ice_map,
1515         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
1516                      RTE_PCI_DRV_IOVA_AS_VA,
1517         .probe = ice_pci_probe,
1518         .remove = ice_pci_remove,
1519 };
1520
1521 /**
1522  * Driver initialization routine.
1523  * Invoked once at EAL init time.
1524  * Register itself as the [Poll Mode] Driver of PCI devices.
1525  */
1526 RTE_PMD_REGISTER_PCI(net_ice, rte_ice_pmd);
1527 RTE_PMD_REGISTER_PCI_TABLE(net_ice, pci_id_ice_map);
1528 RTE_PMD_REGISTER_KMOD_DEP(net_ice, "* igb_uio | uio_pci_generic | vfio-pci");
1529 RTE_PMD_REGISTER_PARAM_STRING(net_ice,
1530                               ICE_MAX_QP_NUM "=<int>");
1531
1532 RTE_INIT(ice_init_log)
1533 {
1534         ice_logtype_init = rte_log_register("pmd.net.ice.init");
1535         if (ice_logtype_init >= 0)
1536                 rte_log_set_level(ice_logtype_init, RTE_LOG_NOTICE);
1537         ice_logtype_driver = rte_log_register("pmd.net.ice.driver");
1538         if (ice_logtype_driver >= 0)
1539                 rte_log_set_level(ice_logtype_driver, RTE_LOG_NOTICE);
1540 }