drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11
  12 #include <rte_pci.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_ethdev_driver.h>
  19
  20 #include "enic_compat.h"
  21 #include "enic.h"
  22 #include "wq_enet_desc.h"
  23 #include "rq_enet_desc.h"
  24 #include "cq_enet_desc.h"
  25 #include "vnic_enet.h"
  26 #include "vnic_dev.h"
  27 #include "vnic_wq.h"
  28 #include "vnic_rq.h"
  29 #include "vnic_cq.h"
  30 #include "vnic_intr.h"
  31 #include "vnic_nic.h"
  32
  33 static inline int enic_is_sriov_vf(struct enic *enic)
  34 {
  35         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  36 }
  37
  38 static int is_zero_addr(uint8_t *addr)
  39 {
  40         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  41 }
  42
  43 static int is_mcast_addr(uint8_t *addr)
  44 {
  45         return addr[0] & 1;
  46 }
  47
  48 static int is_eth_addr_valid(uint8_t *addr)
  49 {
  50         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  51 }
  52
  53 static void
  54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  55 {
  56         uint16_t i;
  57
  58         if (!rq || !rq->mbuf_ring) {
  59                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  60                 return;
  61         }
  62
  63         for (i = 0; i < rq->ring.desc_count; i++) {
  64                 if (rq->mbuf_ring[i]) {
  65                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  66                         rq->mbuf_ring[i] = NULL;
  67                 }
  68         }
  69 }
  70
  71 static void enic_free_wq_buf(struct rte_mbuf **buf)
  72 {
  73         struct rte_mbuf *mbuf = *buf;
  74
  75         rte_pktmbuf_free_seg(mbuf);
  76         *buf = NULL;
  77 }
  78
  79 static void enic_log_q_error(struct enic *enic)
  80 {
  81         unsigned int i;
  82         u32 error_status;
  83
  84         for (i = 0; i < enic->wq_count; i++) {
  85                 error_status = vnic_wq_error_status(&enic->wq[i]);
  86                 if (error_status)
  87                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  88                                 error_status);
  89         }
  90
  91         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  92                 if (!enic->rq[i].in_use)
  93                         continue;
  94                 error_status = vnic_rq_error_status(&enic->rq[i]);
  95                 if (error_status)
  96                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  97                                 error_status);
  98         }
  99 }
 100
 101 static void enic_clear_soft_stats(struct enic *enic)
 102 {
 103         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 104         rte_atomic64_clear(&soft_stats->rx_nombuf);
 105         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 106         rte_atomic64_clear(&soft_stats->tx_oversized);
 107 }
 108
 109 static void enic_init_soft_stats(struct enic *enic)
 110 {
 111         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 112         rte_atomic64_init(&soft_stats->rx_nombuf);
 113         rte_atomic64_init(&soft_stats->rx_packet_errors);
 114         rte_atomic64_init(&soft_stats->tx_oversized);
 115         enic_clear_soft_stats(enic);
 116 }
 117
 118 int enic_dev_stats_clear(struct enic *enic)
 119 {
 120         int ret;
 121
 122         ret = vnic_dev_stats_clear(enic->vdev);
 123         if (ret != 0) {
 124                 dev_err(enic, "Error in clearing stats\n");
 125                 return ret;
 126         }
 127         enic_clear_soft_stats(enic);
 128
 129         return 0;
 130 }
 131
 132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 133 {
 134         struct vnic_stats *stats;
 135         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 136         int64_t rx_truncated;
 137         uint64_t rx_packet_errors;
 138         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 139
 140         if (ret) {
 141                 dev_err(enic, "Error in getting stats\n");
 142                 return ret;
 143         }
 144
 145         /* The number of truncated packets can only be calculated by
 146          * subtracting a hardware counter from error packets received by
 147          * the driver. Note: this causes transient inaccuracies in the
 148          * ipackets count. Also, the length of truncated packets are
 149          * counted in ibytes even though truncated packets are dropped
 150          * which can make ibytes be slightly higher than it should be.
 151          */
 152         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 153         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 154
 155         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 156         r_stats->opackets = stats->tx.tx_frames_ok;
 157
 158         r_stats->ibytes = stats->rx.rx_bytes_ok;
 159         r_stats->obytes = stats->tx.tx_bytes_ok;
 160
 161         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 162         r_stats->oerrors = stats->tx.tx_errors
 163                            + rte_atomic64_read(&soft_stats->tx_oversized);
 164
 165         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 166
 167         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 168         return 0;
 169 }
 170
 171 int enic_del_mac_address(struct enic *enic, int mac_index)
 172 {
 173         struct rte_eth_dev *eth_dev = enic->rte_dev;
 174         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 175
 176         return vnic_dev_del_addr(enic->vdev, mac_addr);
 177 }
 178
 179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 180 {
 181         int err;
 182
 183         if (!is_eth_addr_valid(mac_addr)) {
 184                 dev_err(enic, "invalid mac address\n");
 185                 return -EINVAL;
 186         }
 187
 188         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 189         if (err)
 190                 dev_err(enic, "add mac addr failed\n");
 191         return err;
 192 }
 193
 194 static void
 195 enic_free_rq_buf(struct rte_mbuf **mbuf)
 196 {
 197         if (*mbuf == NULL)
 198                 return;
 199
 200         rte_pktmbuf_free(*mbuf);
 201         *mbuf = NULL;
 202 }
 203
 204 void enic_init_vnic_resources(struct enic *enic)
 205 {
 206         unsigned int error_interrupt_enable = 1;
 207         unsigned int error_interrupt_offset = 0;
 208         unsigned int rxq_interrupt_enable = 0;
 209         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 210         unsigned int index = 0;
 211         unsigned int cq_idx;
 212         struct vnic_rq *data_rq;
 213
 214         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 215                 rxq_interrupt_enable = 1;
 216
 217         for (index = 0; index < enic->rq_count; index++) {
 218                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 219
 220                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 221                         cq_idx,
 222                         error_interrupt_enable,
 223                         error_interrupt_offset);
 224
 225                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 226                 if (data_rq->in_use)
 227                         vnic_rq_init(data_rq,
 228                                      cq_idx,
 229                                      error_interrupt_enable,
 230                                      error_interrupt_offset);
 231
 232                 vnic_cq_init(&enic->cq[cq_idx],
 233                         0 /* flow_control_enable */,
 234                         1 /* color_enable */,
 235                         0 /* cq_head */,
 236                         0 /* cq_tail */,
 237                         1 /* cq_tail_color */,
 238                         rxq_interrupt_enable,
 239                         1 /* cq_entry_enable */,
 240                         0 /* cq_message_enable */,
 241                         rxq_interrupt_offset,
 242                         0 /* cq_message_addr */);
 243                 if (rxq_interrupt_enable)
 244                         rxq_interrupt_offset++;
 245         }
 246
 247         for (index = 0; index < enic->wq_count; index++) {
 248                 vnic_wq_init(&enic->wq[index],
 249                         enic_cq_wq(enic, index),
 250                         error_interrupt_enable,
 251                         error_interrupt_offset);
 252                 /* Compute unsupported ol flags for enic_prep_pkts() */
 253                 enic->wq[index].tx_offload_notsup_mask =
 254                         PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 255
 256                 cq_idx = enic_cq_wq(enic, index);
 257                 vnic_cq_init(&enic->cq[cq_idx],
 258                         0 /* flow_control_enable */,
 259                         1 /* color_enable */,
 260                         0 /* cq_head */,
 261                         0 /* cq_tail */,
 262                         1 /* cq_tail_color */,
 263                         0 /* interrupt_enable */,
 264                         0 /* cq_entry_enable */,
 265                         1 /* cq_message_enable */,
 266                         0 /* interrupt offset */,
 267                         (u64)enic->wq[index].cqmsg_rz->iova);
 268         }
 269
 270         for (index = 0; index < enic->intr_count; index++) {
 271                 vnic_intr_init(&enic->intr[index],
 272                                enic->config.intr_timer_usec,
 273                                enic->config.intr_timer_type,
 274                                /*mask_on_assertion*/1);
 275         }
 276 }
 277
 278
 279 static int
 280 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 281 {
 282         struct rte_mbuf *mb;
 283         struct rq_enet_desc *rqd = rq->ring.descs;
 284         unsigned i;
 285         dma_addr_t dma_addr;
 286         uint32_t max_rx_pkt_len;
 287         uint16_t rq_buf_len;
 288
 289         if (!rq->in_use)
 290                 return 0;
 291
 292         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 293                   rq->ring.desc_count);
 294
 295         /*
 296          * If *not* using scatter and the mbuf size is greater than the
 297          * requested max packet size (max_rx_pkt_len), then reduce the
 298          * posted buffer size to max_rx_pkt_len. HW still receives packets
 299          * larger than max_rx_pkt_len, but they will be truncated, which we
 300          * drop in the rx handler. Not ideal, but better than returning
 301          * large packets when the user is not expecting them.
 302          */
 303         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 304         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 305         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 306                 rq_buf_len = max_rx_pkt_len;
 307         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 308                 mb = rte_mbuf_raw_alloc(rq->mp);
 309                 if (mb == NULL) {
 310                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 311                         (unsigned)rq->index);
 312                         return -ENOMEM;
 313                 }
 314
 315                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 316                 dma_addr = (dma_addr_t)(mb->buf_iova
 317                            + RTE_PKTMBUF_HEADROOM);
 318                 rq_enet_desc_enc(rqd, dma_addr,
 319                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 320                                 : RQ_ENET_TYPE_NOT_SOP),
 321                                 rq_buf_len);
 322                 rq->mbuf_ring[i] = mb;
 323         }
 324         /*
 325          * Do not post the buffers to the NIC until we enable the RQ via
 326          * enic_start_rq().
 327          */
 328         rq->need_initial_post = true;
 329         /* Initialize fetch index while RQ is disabled */
 330         iowrite32(0, &rq->ctrl->fetch_index);
 331         return 0;
 332 }
 333
 334 /*
 335  * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
 336  * allocated the buffers and filled the RQ descriptor ring. Just need to push
 337  * the post index to the NIC.
 338  */
 339 static void
 340 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 341 {
 342         if (!rq->in_use || !rq->need_initial_post)
 343                 return;
 344
 345         /* make sure all prior writes are complete before doing the PIO write */
 346         rte_rmb();
 347
 348         /* Post all but the last buffer to VIC. */
 349         rq->posted_index = rq->ring.desc_count - 1;
 350
 351         rq->rx_nb_hold = 0;
 352
 353         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 354                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 355         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 356         rte_rmb();
 357         rq->need_initial_post = false;
 358 }
 359
 360 static void *
 361 enic_alloc_consistent(void *priv, size_t size,
 362         dma_addr_t *dma_handle, u8 *name)
 363 {
 364         void *vaddr;
 365         const struct rte_memzone *rz;
 366         *dma_handle = 0;
 367         struct enic *enic = (struct enic *)priv;
 368         struct enic_memzone_entry *mze;
 369
 370         rz = rte_memzone_reserve_aligned((const char *)name, size,
 371                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 372         if (!rz) {
 373                 pr_err("%s : Failed to allocate memory requested for %s\n",
 374                         __func__, name);
 375                 return NULL;
 376         }
 377
 378         vaddr = rz->addr;
 379         *dma_handle = (dma_addr_t)rz->iova;
 380
 381         mze = rte_malloc("enic memzone entry",
 382                          sizeof(struct enic_memzone_entry), 0);
 383
 384         if (!mze) {
 385                 pr_err("%s : Failed to allocate memory for memzone list\n",
 386                        __func__);
 387                 rte_memzone_free(rz);
 388                 return NULL;
 389         }
 390
 391         mze->rz = rz;
 392
 393         rte_spinlock_lock(&enic->memzone_list_lock);
 394         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 395         rte_spinlock_unlock(&enic->memzone_list_lock);
 396
 397         return vaddr;
 398 }
 399
 400 static void
 401 enic_free_consistent(void *priv,
 402                      __rte_unused size_t size,
 403                      void *vaddr,
 404                      dma_addr_t dma_handle)
 405 {
 406         struct enic_memzone_entry *mze;
 407         struct enic *enic = (struct enic *)priv;
 408
 409         rte_spinlock_lock(&enic->memzone_list_lock);
 410         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 411                 if (mze->rz->addr == vaddr &&
 412                     mze->rz->iova == dma_handle)
 413                         break;
 414         }
 415         if (mze == NULL) {
 416                 rte_spinlock_unlock(&enic->memzone_list_lock);
 417                 dev_warning(enic,
 418                             "Tried to free memory, but couldn't find it in the memzone list\n");
 419                 return;
 420         }
 421         LIST_REMOVE(mze, entries);
 422         rte_spinlock_unlock(&enic->memzone_list_lock);
 423         rte_memzone_free(mze->rz);
 424         rte_free(mze);
 425 }
 426
 427 int enic_link_update(struct enic *enic)
 428 {
 429         struct rte_eth_dev *eth_dev = enic->rte_dev;
 430         struct rte_eth_link link;
 431
 432         memset(&link, 0, sizeof(link));
 433         link.link_status = enic_get_link_status(enic);
 434         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 435         link.link_speed = vnic_dev_port_speed(enic->vdev);
 436
 437         return rte_eth_linkstatus_set(eth_dev, &link);
 438 }
 439
 440 static void
 441 enic_intr_handler(void *arg)
 442 {
 443         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 444         struct enic *enic = pmd_priv(dev);
 445
 446         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 447
 448         enic_link_update(enic);
 449         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 450         enic_log_q_error(enic);
 451 }
 452
 453 static int enic_rxq_intr_init(struct enic *enic)
 454 {
 455         struct rte_intr_handle *intr_handle;
 456         uint32_t rxq_intr_count, i;
 457         int err;
 458
 459         intr_handle = enic->rte_dev->intr_handle;
 460         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 461                 return 0;
 462         /*
 463          * Rx queue interrupts only work when we have MSI-X interrupts,
 464          * one per queue. Sharing one interrupt is technically
 465          * possible with VIC, but it is not worth the complications it brings.
 466          */
 467         if (!rte_intr_cap_multiple(intr_handle)) {
 468                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 469                         " (vfio-pci driver)\n");
 470                 return -ENOTSUP;
 471         }
 472         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 473         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 474         if (err) {
 475                 dev_err(enic, "Failed to enable event fds for Rx queue"
 476                         " interrupts\n");
 477                 return err;
 478         }
 479         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 480                                             rxq_intr_count * sizeof(int), 0);
 481         if (intr_handle->intr_vec == NULL) {
 482                 dev_err(enic, "Failed to allocate intr_vec\n");
 483                 return -ENOMEM;
 484         }
 485         for (i = 0; i < rxq_intr_count; i++)
 486                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 487         return 0;
 488 }
 489
 490 static void enic_rxq_intr_deinit(struct enic *enic)
 491 {
 492         struct rte_intr_handle *intr_handle;
 493
 494         intr_handle = enic->rte_dev->intr_handle;
 495         rte_intr_efd_disable(intr_handle);
 496         if (intr_handle->intr_vec != NULL) {
 497                 rte_free(intr_handle->intr_vec);
 498                 intr_handle->intr_vec = NULL;
 499         }
 500 }
 501
 502 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 503 {
 504         struct wq_enet_desc *desc;
 505         struct vnic_wq *wq;
 506         unsigned int i;
 507
 508         /*
 509          * Fill WQ descriptor fields that never change. Every descriptor is
 510          * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
 511          * descriptors (i.e. request one completion update every 32 packets).
 512          */
 513         wq = &enic->wq[queue_idx];
 514         desc = (struct wq_enet_desc *)wq->ring.descs;
 515         for (i = 0; i < wq->ring.desc_count; i++, desc++) {
 516                 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
 517                 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
 518                         desc->header_length_flags |=
 519                                 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
 520         }
 521 }
 522
 523 /*
 524  * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
 525  * used when that file is not compiled.
 526  */
 527 __rte_weak bool
 528 enic_use_vector_rx_handler(__rte_unused struct enic *enic)
 529 {
 530         return false;
 531 }
 532
 533 static void pick_rx_handler(struct enic *enic)
 534 {
 535         struct rte_eth_dev *eth_dev;
 536
 537         /*
 538          * Preference order:
 539          * 1. The vectorized handler if possible and requested.
 540          * 2. The non-scatter, simplified handler if scatter Rx is not used.
 541          * 3. The default handler as a fallback.
 542          */
 543         eth_dev = enic->rte_dev;
 544         if (enic_use_vector_rx_handler(enic))
 545                 return;
 546         if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 547                 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
 548                 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
 549         } else {
 550                 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
 551                 eth_dev->rx_pkt_burst = &enic_recv_pkts;
 552         }
 553 }
 554
 555 int enic_enable(struct enic *enic)
 556 {
 557         unsigned int index;
 558         int err;
 559         struct rte_eth_dev *eth_dev = enic->rte_dev;
 560         uint64_t simple_tx_offloads;
 561         uintptr_t p;
 562
 563         if (enic->enable_avx2_rx) {
 564                 struct rte_mbuf mb_def = { .buf_addr = 0 };
 565
 566                 /*
 567                  * mbuf_initializer contains const-after-init fields of
 568                  * receive mbufs (i.e. 64 bits of fields from rearm_data).
 569                  * It is currently used by the vectorized handler.
 570                  */
 571                 mb_def.nb_segs = 1;
 572                 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 573                 mb_def.port = enic->port_id;
 574                 rte_mbuf_refcnt_set(&mb_def, 1);
 575                 rte_compiler_barrier();
 576                 p = (uintptr_t)&mb_def.rearm_data;
 577                 enic->mbuf_initializer = *(uint64_t *)p;
 578         }
 579
 580         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 581         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 582
 583         /* vnic notification of link status has already been turned on in
 584          * enic_dev_init() which is called during probe time.  Here we are
 585          * just turning on interrupt vector 0 if needed.
 586          */
 587         if (eth_dev->data->dev_conf.intr_conf.lsc)
 588                 vnic_dev_notify_set(enic->vdev, 0);
 589
 590         err = enic_rxq_intr_init(enic);
 591         if (err)
 592                 return err;
 593         if (enic_clsf_init(enic))
 594                 dev_warning(enic, "Init of hash table for clsf failed."\
 595                         "Flow director feature will not work\n");
 596
 597         for (index = 0; index < enic->rq_count; index++) {
 598                 err = enic_alloc_rx_queue_mbufs(enic,
 599                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 600                 if (err) {
 601                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 602                         return err;
 603                 }
 604                 err = enic_alloc_rx_queue_mbufs(enic,
 605                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 606                 if (err) {
 607                         /* release the allocated mbufs for the sop rq*/
 608                         enic_rxmbuf_queue_release(enic,
 609                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 610
 611                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 612                         return err;
 613                 }
 614         }
 615
 616         /*
 617          * Use the simple TX handler if possible. Only checksum offloads
 618          * and vlan insertion are supported.
 619          */
 620         simple_tx_offloads = enic->tx_offload_capa &
 621                 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 622                  DEV_TX_OFFLOAD_VLAN_INSERT |
 623                  DEV_TX_OFFLOAD_IPV4_CKSUM |
 624                  DEV_TX_OFFLOAD_UDP_CKSUM |
 625                  DEV_TX_OFFLOAD_TCP_CKSUM);
 626         if ((eth_dev->data->dev_conf.txmode.offloads &
 627              ~simple_tx_offloads) == 0) {
 628                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 629                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 630                 for (index = 0; index < enic->wq_count; index++)
 631                         enic_prep_wq_for_simple_tx(enic, index);
 632         } else {
 633                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 634                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 635         }
 636
 637         pick_rx_handler(enic);
 638
 639         for (index = 0; index < enic->wq_count; index++)
 640                 enic_start_wq(enic, index);
 641         for (index = 0; index < enic->rq_count; index++)
 642                 enic_start_rq(enic, index);
 643
 644         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 645
 646         vnic_dev_enable_wait(enic->vdev);
 647
 648         /* Register and enable error interrupt */
 649         rte_intr_callback_register(&(enic->pdev->intr_handle),
 650                 enic_intr_handler, (void *)enic->rte_dev);
 651
 652         rte_intr_enable(&(enic->pdev->intr_handle));
 653         /* Unmask LSC interrupt */
 654         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 655
 656         return 0;
 657 }
 658
 659 int enic_alloc_intr_resources(struct enic *enic)
 660 {
 661         int err;
 662         unsigned int i;
 663
 664         dev_info(enic, "vNIC resources used:  "\
 665                 "wq %d rq %d cq %d intr %d\n",
 666                 enic->wq_count, enic_vnic_rq_count(enic),
 667                 enic->cq_count, enic->intr_count);
 668
 669         for (i = 0; i < enic->intr_count; i++) {
 670                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 671                 if (err) {
 672                         enic_free_vnic_resources(enic);
 673                         return err;
 674                 }
 675         }
 676         return 0;
 677 }
 678
 679 void enic_free_rq(void *rxq)
 680 {
 681         struct vnic_rq *rq_sop, *rq_data;
 682         struct enic *enic;
 683
 684         if (rxq == NULL)
 685                 return;
 686
 687         rq_sop = (struct vnic_rq *)rxq;
 688         enic = vnic_dev_priv(rq_sop->vdev);
 689         rq_data = &enic->rq[rq_sop->data_queue_idx];
 690
 691         if (rq_sop->free_mbufs) {
 692                 struct rte_mbuf **mb;
 693                 int i;
 694
 695                 mb = rq_sop->free_mbufs;
 696                 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
 697                      i < ENIC_RX_BURST_MAX; i++)
 698                         rte_pktmbuf_free(mb[i]);
 699                 rte_free(rq_sop->free_mbufs);
 700                 rq_sop->free_mbufs = NULL;
 701                 rq_sop->num_free_mbufs = 0;
 702         }
 703
 704         enic_rxmbuf_queue_release(enic, rq_sop);
 705         if (rq_data->in_use)
 706                 enic_rxmbuf_queue_release(enic, rq_data);
 707
 708         rte_free(rq_sop->mbuf_ring);
 709         if (rq_data->in_use)
 710                 rte_free(rq_data->mbuf_ring);
 711
 712         rq_sop->mbuf_ring = NULL;
 713         rq_data->mbuf_ring = NULL;
 714
 715         vnic_rq_free(rq_sop);
 716         if (rq_data->in_use)
 717                 vnic_rq_free(rq_data);
 718
 719         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 720
 721         rq_sop->in_use = 0;
 722         rq_data->in_use = 0;
 723 }
 724
 725 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 726 {
 727         struct rte_eth_dev *eth_dev = enic->rte_dev;
 728         vnic_wq_enable(&enic->wq[queue_idx]);
 729         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 730 }
 731
 732 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 733 {
 734         struct rte_eth_dev *eth_dev = enic->rte_dev;
 735         int ret;
 736
 737         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 738         if (ret)
 739                 return ret;
 740
 741         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 742         return 0;
 743 }
 744
 745 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 746 {
 747         struct vnic_rq *rq_sop;
 748         struct vnic_rq *rq_data;
 749         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 750         rq_data = &enic->rq[rq_sop->data_queue_idx];
 751         struct rte_eth_dev *eth_dev = enic->rte_dev;
 752
 753         if (rq_data->in_use) {
 754                 vnic_rq_enable(rq_data);
 755                 enic_initial_post_rx(enic, rq_data);
 756         }
 757         rte_mb();
 758         vnic_rq_enable(rq_sop);
 759         enic_initial_post_rx(enic, rq_sop);
 760         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 761 }
 762
 763 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 764 {
 765         int ret1 = 0, ret2 = 0;
 766         struct rte_eth_dev *eth_dev = enic->rte_dev;
 767         struct vnic_rq *rq_sop;
 768         struct vnic_rq *rq_data;
 769         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 770         rq_data = &enic->rq[rq_sop->data_queue_idx];
 771
 772         ret2 = vnic_rq_disable(rq_sop);
 773         rte_mb();
 774         if (rq_data->in_use)
 775                 ret1 = vnic_rq_disable(rq_data);
 776
 777         if (ret2)
 778                 return ret2;
 779         else if (ret1)
 780                 return ret1;
 781
 782         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 783         return 0;
 784 }
 785
 786 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 787         unsigned int socket_id, struct rte_mempool *mp,
 788         uint16_t nb_desc, uint16_t free_thresh)
 789 {
 790         int rc;
 791         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 792         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 793         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 794         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 795         unsigned int mbuf_size, mbufs_per_pkt;
 796         unsigned int nb_sop_desc, nb_data_desc;
 797         uint16_t min_sop, max_sop, min_data, max_data;
 798         uint32_t max_rx_pkt_len;
 799
 800         rq_sop->is_sop = 1;
 801         rq_sop->data_queue_idx = data_queue_idx;
 802         rq_data->is_sop = 0;
 803         rq_data->data_queue_idx = 0;
 804         rq_sop->socket_id = socket_id;
 805         rq_sop->mp = mp;
 806         rq_data->socket_id = socket_id;
 807         rq_data->mp = mp;
 808         rq_sop->in_use = 1;
 809         rq_sop->rx_free_thresh = free_thresh;
 810         rq_data->rx_free_thresh = free_thresh;
 811         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 812                   free_thresh);
 813
 814         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 815                                RTE_PKTMBUF_HEADROOM);
 816         /* max_rx_pkt_len includes the ethernet header and CRC. */
 817         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 818
 819         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 820             DEV_RX_OFFLOAD_SCATTER) {
 821                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 822                 /* ceil((max pkt len)/mbuf_size) */
 823                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 824         } else {
 825                 dev_info(enic, "Scatter rx mode disabled\n");
 826                 mbufs_per_pkt = 1;
 827                 if (max_rx_pkt_len > mbuf_size) {
 828                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 829                                     " larger than the mbuf size (%u), and"
 830                                     " scatter is disabled. Larger packets will"
 831                                     " be truncated.\n",
 832                                     max_rx_pkt_len, mbuf_size);
 833                 }
 834         }
 835
 836         if (mbufs_per_pkt > 1) {
 837                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 838                 rq_sop->data_queue_enable = 1;
 839                 rq_data->in_use = 1;
 840                 /*
 841                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 842                  * receives packet sizes up to the "max" MTU.
 843                  * If not using scatter, we can achieve the effect of dropping
 844                  * larger packets by reducing the size of posted buffers.
 845                  * See enic_alloc_rx_queue_mbufs().
 846                  */
 847                 if (max_rx_pkt_len <
 848                     enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
 849                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 850                                     " when scatter rx mode is in use.\n");
 851                 }
 852         } else {
 853                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 854                          queue_idx);
 855                 rq_sop->data_queue_enable = 0;
 856                 rq_data->in_use = 0;
 857         }
 858
 859         /* number of descriptors have to be a multiple of 32 */
 860         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
 861         nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 862
 863         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 864         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 865
 866         if (mbufs_per_pkt > 1) {
 867                 min_sop = ENIC_RX_BURST_MAX;
 868                 max_sop = ((enic->config.rq_desc_count /
 869                             (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 870                 min_data = min_sop * (mbufs_per_pkt - 1);
 871                 max_data = enic->config.rq_desc_count;
 872         } else {
 873                 min_sop = ENIC_RX_BURST_MAX;
 874                 max_sop = enic->config.rq_desc_count;
 875                 min_data = 0;
 876                 max_data = 0;
 877         }
 878
 879         if (nb_desc < (min_sop + min_data)) {
 880                 dev_warning(enic,
 881                             "Number of rx descs too low, adjusting to minimum\n");
 882                 nb_sop_desc = min_sop;
 883                 nb_data_desc = min_data;
 884         } else if (nb_desc > (max_sop + max_data)) {
 885                 dev_warning(enic,
 886                             "Number of rx_descs too high, adjusting to maximum\n");
 887                 nb_sop_desc = max_sop;
 888                 nb_data_desc = max_data;
 889         }
 890         if (mbufs_per_pkt > 1) {
 891                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 892                          " rx descriptor range is %u to %u\n",
 893                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 894                          max_sop + max_data);
 895         }
 896         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 897                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 898
 899         /* Allocate sop queue resources */
 900         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 901                 nb_sop_desc, sizeof(struct rq_enet_desc));
 902         if (rc) {
 903                 dev_err(enic, "error in allocation of sop rq\n");
 904                 goto err_exit;
 905         }
 906         nb_sop_desc = rq_sop->ring.desc_count;
 907
 908         if (rq_data->in_use) {
 909                 /* Allocate data queue resources */
 910                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 911                                    nb_data_desc,
 912                                    sizeof(struct rq_enet_desc));
 913                 if (rc) {
 914                         dev_err(enic, "error in allocation of data rq\n");
 915                         goto err_free_rq_sop;
 916                 }
 917                 nb_data_desc = rq_data->ring.desc_count;
 918         }
 919         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 920                            socket_id, nb_sop_desc + nb_data_desc,
 921                            sizeof(struct cq_enet_rq_desc));
 922         if (rc) {
 923                 dev_err(enic, "error in allocation of cq for rq\n");
 924                 goto err_free_rq_data;
 925         }
 926
 927         /* Allocate the mbuf rings */
 928         rq_sop->mbuf_ring = (struct rte_mbuf **)
 929                 rte_zmalloc_socket("rq->mbuf_ring",
 930                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 931                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 932         if (rq_sop->mbuf_ring == NULL)
 933                 goto err_free_cq;
 934
 935         if (rq_data->in_use) {
 936                 rq_data->mbuf_ring = (struct rte_mbuf **)
 937                         rte_zmalloc_socket("rq->mbuf_ring",
 938                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 939                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 940                 if (rq_data->mbuf_ring == NULL)
 941                         goto err_free_sop_mbuf;
 942         }
 943
 944         rq_sop->free_mbufs = (struct rte_mbuf **)
 945                 rte_zmalloc_socket("rq->free_mbufs",
 946                                    sizeof(struct rte_mbuf *) *
 947                                    ENIC_RX_BURST_MAX,
 948                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 949         if (rq_sop->free_mbufs == NULL)
 950                 goto err_free_data_mbuf;
 951         rq_sop->num_free_mbufs = 0;
 952
 953         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 954
 955         return 0;
 956
 957 err_free_data_mbuf:
 958         rte_free(rq_data->mbuf_ring);
 959 err_free_sop_mbuf:
 960         rte_free(rq_sop->mbuf_ring);
 961 err_free_cq:
 962         /* cleanup on error */
 963         vnic_cq_free(&enic->cq[queue_idx]);
 964 err_free_rq_data:
 965         if (rq_data->in_use)
 966                 vnic_rq_free(rq_data);
 967 err_free_rq_sop:
 968         vnic_rq_free(rq_sop);
 969 err_exit:
 970         return -ENOMEM;
 971 }
 972
 973 void enic_free_wq(void *txq)
 974 {
 975         struct vnic_wq *wq;
 976         struct enic *enic;
 977
 978         if (txq == NULL)
 979                 return;
 980
 981         wq = (struct vnic_wq *)txq;
 982         enic = vnic_dev_priv(wq->vdev);
 983         rte_memzone_free(wq->cqmsg_rz);
 984         vnic_wq_free(wq);
 985         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 986 }
 987
 988 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 989         unsigned int socket_id, uint16_t nb_desc)
 990 {
 991         int err;
 992         struct vnic_wq *wq = &enic->wq[queue_idx];
 993         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 994         char name[NAME_MAX];
 995         static int instance;
 996
 997         wq->socket_id = socket_id;
 998         /*
 999          * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1000          * print an info message for diagnostics.
1001          */
1002         dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1003
1004         /* Allocate queue resources */
1005         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1006                 nb_desc,
1007                 sizeof(struct wq_enet_desc));
1008         if (err) {
1009                 dev_err(enic, "error in allocation of wq\n");
1010                 return err;
1011         }
1012
1013         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1014                 socket_id, nb_desc,
1015                 sizeof(struct cq_enet_wq_desc));
1016         if (err) {
1017                 vnic_wq_free(wq);
1018                 dev_err(enic, "error in allocation of cq for wq\n");
1019         }
1020
1021         /* setup up CQ message */
1022         snprintf((char *)name, sizeof(name),
1023                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1024                 instance++);
1025
1026         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1027                         sizeof(uint32_t), SOCKET_ID_ANY,
1028                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1029         if (!wq->cqmsg_rz)
1030                 return -ENOMEM;
1031
1032         return err;
1033 }
1034
1035 int enic_disable(struct enic *enic)
1036 {
1037         unsigned int i;
1038         int err;
1039
1040         for (i = 0; i < enic->intr_count; i++) {
1041                 vnic_intr_mask(&enic->intr[i]);
1042                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1043         }
1044         enic_rxq_intr_deinit(enic);
1045         rte_intr_disable(&enic->pdev->intr_handle);
1046         rte_intr_callback_unregister(&enic->pdev->intr_handle,
1047                                      enic_intr_handler,
1048                                      (void *)enic->rte_dev);
1049
1050         vnic_dev_disable(enic->vdev);
1051
1052         enic_clsf_destroy(enic);
1053
1054         if (!enic_is_sriov_vf(enic))
1055                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1056
1057         for (i = 0; i < enic->wq_count; i++) {
1058                 err = vnic_wq_disable(&enic->wq[i]);
1059                 if (err)
1060                         return err;
1061         }
1062         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1063                 if (enic->rq[i].in_use) {
1064                         err = vnic_rq_disable(&enic->rq[i]);
1065                         if (err)
1066                                 return err;
1067                 }
1068         }
1069
1070         /* If we were using interrupts, set the interrupt vector to -1
1071          * to disable interrupts.  We are not disabling link notifcations,
1072          * though, as we want the polling of link status to continue working.
1073          */
1074         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1075                 vnic_dev_notify_set(enic->vdev, -1);
1076
1077         vnic_dev_set_reset_flag(enic->vdev, 1);
1078
1079         for (i = 0; i < enic->wq_count; i++)
1080                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1081
1082         for (i = 0; i < enic_vnic_rq_count(enic); i++)
1083                 if (enic->rq[i].in_use)
1084                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1085         for (i = 0; i < enic->cq_count; i++)
1086                 vnic_cq_clean(&enic->cq[i]);
1087         for (i = 0; i < enic->intr_count; i++)
1088                 vnic_intr_clean(&enic->intr[i]);
1089
1090         return 0;
1091 }
1092
1093 static int enic_dev_wait(struct vnic_dev *vdev,
1094         int (*start)(struct vnic_dev *, int),
1095         int (*finished)(struct vnic_dev *, int *),
1096         int arg)
1097 {
1098         int done;
1099         int err;
1100         int i;
1101
1102         err = start(vdev, arg);
1103         if (err)
1104                 return err;
1105
1106         /* Wait for func to complete...2 seconds max */
1107         for (i = 0; i < 2000; i++) {
1108                 err = finished(vdev, &done);
1109                 if (err)
1110                         return err;
1111                 if (done)
1112                         return 0;
1113                 usleep(1000);
1114         }
1115         return -ETIMEDOUT;
1116 }
1117
1118 static int enic_dev_open(struct enic *enic)
1119 {
1120         int err;
1121         int flags = CMD_OPENF_IG_DESCCACHE;
1122
1123         err = enic_dev_wait(enic->vdev, vnic_dev_open,
1124                 vnic_dev_open_done, flags);
1125         if (err)
1126                 dev_err(enic_get_dev(enic),
1127                         "vNIC device open failed, err %d\n", err);
1128
1129         return err;
1130 }
1131
1132 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1133 {
1134         dma_addr_t rss_key_buf_pa;
1135         union vnic_rss_key *rss_key_buf_va = NULL;
1136         int err, i;
1137         u8 name[NAME_MAX];
1138
1139         RTE_ASSERT(user_key != NULL);
1140         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1141         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1142                 &rss_key_buf_pa, name);
1143         if (!rss_key_buf_va)
1144                 return -ENOMEM;
1145
1146         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1147                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1148
1149         err = enic_set_rss_key(enic,
1150                 rss_key_buf_pa,
1151                 sizeof(union vnic_rss_key));
1152
1153         /* Save for later queries */
1154         if (!err) {
1155                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1156                            sizeof(union vnic_rss_key));
1157         }
1158         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1159                 rss_key_buf_va, rss_key_buf_pa);
1160
1161         return err;
1162 }
1163
1164 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1165 {
1166         dma_addr_t rss_cpu_buf_pa;
1167         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1168         int err;
1169         u8 name[NAME_MAX];
1170
1171         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1172         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1173                 &rss_cpu_buf_pa, name);
1174         if (!rss_cpu_buf_va)
1175                 return -ENOMEM;
1176
1177         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1178
1179         err = enic_set_rss_cpu(enic,
1180                 rss_cpu_buf_pa,
1181                 sizeof(union vnic_rss_cpu));
1182
1183         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1184                 rss_cpu_buf_va, rss_cpu_buf_pa);
1185
1186         /* Save for later queries */
1187         if (!err)
1188                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1189         return err;
1190 }
1191
1192 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1193         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1194 {
1195         const u8 tso_ipid_split_en = 0;
1196         int err;
1197
1198         err = enic_set_nic_cfg(enic,
1199                 rss_default_cpu, rss_hash_type,
1200                 rss_hash_bits, rss_base_cpu,
1201                 rss_enable, tso_ipid_split_en,
1202                 enic->ig_vlan_strip_en);
1203
1204         return err;
1205 }
1206
1207 /* Initialize RSS with defaults, called from dev_configure */
1208 int enic_init_rss_nic_cfg(struct enic *enic)
1209 {
1210         static uint8_t default_rss_key[] = {
1211                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1212                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1213                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1214                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1215         };
1216         struct rte_eth_rss_conf rss_conf;
1217         union vnic_rss_cpu rss_cpu;
1218         int ret, i;
1219
1220         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1221         /*
1222          * If setting key for the first time, and the user gives us none, then
1223          * push the default key to NIC.
1224          */
1225         if (rss_conf.rss_key == NULL) {
1226                 rss_conf.rss_key = default_rss_key;
1227                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1228         }
1229         ret = enic_set_rss_conf(enic, &rss_conf);
1230         if (ret) {
1231                 dev_err(enic, "Failed to configure RSS\n");
1232                 return ret;
1233         }
1234         if (enic->rss_enable) {
1235                 /* If enabling RSS, use the default reta */
1236                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1237                         rss_cpu.cpu[i / 4].b[i % 4] =
1238                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1239                 }
1240                 ret = enic_set_rss_reta(enic, &rss_cpu);
1241                 if (ret)
1242                         dev_err(enic, "Failed to set RSS indirection table\n");
1243         }
1244         return ret;
1245 }
1246
1247 int enic_setup_finish(struct enic *enic)
1248 {
1249         enic_init_soft_stats(enic);
1250
1251         /* Default conf */
1252         vnic_dev_packet_filter(enic->vdev,
1253                 1 /* directed  */,
1254                 1 /* multicast */,
1255                 1 /* broadcast */,
1256                 0 /* promisc   */,
1257                 1 /* allmulti  */);
1258
1259         enic->promisc = 0;
1260         enic->allmulti = 1;
1261
1262         return 0;
1263 }
1264
1265 static int enic_rss_conf_valid(struct enic *enic,
1266                                struct rte_eth_rss_conf *rss_conf)
1267 {
1268         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1269         if (enic->flow_type_rss_offloads == 0)
1270                 return 0;
1271         if (rss_conf->rss_key != NULL &&
1272             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1273                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1274                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1275                 return -EINVAL;
1276         }
1277         if (rss_conf->rss_hf != 0 &&
1278             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1279                 dev_err(enic, "Given rss_hf contains none of the supported"
1280                         " types\n");
1281                 return -EINVAL;
1282         }
1283         return 0;
1284 }
1285
1286 /* Set hash type and key according to rss_conf */
1287 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1288 {
1289         struct rte_eth_dev *eth_dev;
1290         uint64_t rss_hf;
1291         u8 rss_hash_type;
1292         u8 rss_enable;
1293         int ret;
1294
1295         RTE_ASSERT(rss_conf != NULL);
1296         ret = enic_rss_conf_valid(enic, rss_conf);
1297         if (ret) {
1298                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1299                 return ret;
1300         }
1301
1302         eth_dev = enic->rte_dev;
1303         rss_hash_type = 0;
1304         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1305         if (enic->rq_count > 1 &&
1306             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1307             rss_hf != 0) {
1308                 rss_enable = 1;
1309                 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1310                               ETH_RSS_NONFRAG_IPV4_OTHER))
1311                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1312                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1313                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1314                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1315                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1316                         if (enic->udp_rss_weak) {
1317                                 /*
1318                                  * 'TCP' is not a typo. The "weak" version of
1319                                  * UDP RSS requires both the TCP and UDP bits
1320                                  * be set. It does enable TCP RSS as well.
1321                                  */
1322                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1323                         }
1324                 }
1325                 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1326                               ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1327                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1328                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1329                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1330                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1331                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1332                         if (enic->udp_rss_weak)
1333                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1334                 }
1335         } else {
1336                 rss_enable = 0;
1337                 rss_hf = 0;
1338         }
1339
1340         /* Set the hash key if provided */
1341         if (rss_enable && rss_conf->rss_key) {
1342                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1343                 if (ret) {
1344                         dev_err(enic, "Failed to set RSS key\n");
1345                         return ret;
1346                 }
1347         }
1348
1349         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1350                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1351                               rss_enable);
1352         if (!ret) {
1353                 enic->rss_hf = rss_hf;
1354                 enic->rss_hash_type = rss_hash_type;
1355                 enic->rss_enable = rss_enable;
1356         } else {
1357                 dev_err(enic, "Failed to update RSS configurations."
1358                         " hash=0x%x\n", rss_hash_type);
1359         }
1360         return ret;
1361 }
1362
1363 int enic_set_vlan_strip(struct enic *enic)
1364 {
1365         /*
1366          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1367          * together. So, re-do niccfg, preserving the current RSS settings.
1368          */
1369         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1370                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1371                                enic->rss_enable);
1372 }
1373
1374 int enic_add_packet_filter(struct enic *enic)
1375 {
1376         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1377         return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1378                 enic->promisc, enic->allmulti);
1379 }
1380
1381 int enic_get_link_status(struct enic *enic)
1382 {
1383         return vnic_dev_link_status(enic->vdev);
1384 }
1385
1386 static void enic_dev_deinit(struct enic *enic)
1387 {
1388         /* stop link status checking */
1389         vnic_dev_notify_unset(enic->vdev);
1390
1391         /* mac_addrs is freed by rte_eth_dev_release_port() */
1392         rte_free(enic->cq);
1393         rte_free(enic->intr);
1394         rte_free(enic->rq);
1395         rte_free(enic->wq);
1396 }
1397
1398
1399 int enic_set_vnic_res(struct enic *enic)
1400 {
1401         struct rte_eth_dev *eth_dev = enic->rte_dev;
1402         int rc = 0;
1403         unsigned int required_rq, required_wq, required_cq, required_intr;
1404
1405         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1406         required_rq = eth_dev->data->nb_rx_queues * 2;
1407         required_wq = eth_dev->data->nb_tx_queues;
1408         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1409         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1410         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1411                 required_intr += eth_dev->data->nb_rx_queues;
1412         }
1413
1414         if (enic->conf_rq_count < required_rq) {
1415                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1416                         eth_dev->data->nb_rx_queues,
1417                         required_rq, enic->conf_rq_count);
1418                 rc = -EINVAL;
1419         }
1420         if (enic->conf_wq_count < required_wq) {
1421                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1422                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1423                 rc = -EINVAL;
1424         }
1425
1426         if (enic->conf_cq_count < required_cq) {
1427                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1428                         required_cq, enic->conf_cq_count);
1429                 rc = -EINVAL;
1430         }
1431         if (enic->conf_intr_count < required_intr) {
1432                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1433                         " interrupts. Required:%u, Configured:%u\n",
1434                         required_intr, enic->conf_intr_count);
1435                 rc = -EINVAL;
1436         }
1437
1438         if (rc == 0) {
1439                 enic->rq_count = eth_dev->data->nb_rx_queues;
1440                 enic->wq_count = eth_dev->data->nb_tx_queues;
1441                 enic->cq_count = enic->rq_count + enic->wq_count;
1442                 enic->intr_count = required_intr;
1443         }
1444
1445         return rc;
1446 }
1447
1448 /* Initialize the completion queue for an RQ */
1449 static int
1450 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1451 {
1452         struct vnic_rq *sop_rq, *data_rq;
1453         unsigned int cq_idx;
1454         int rc = 0;
1455
1456         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1457         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1458         cq_idx = rq_idx;
1459
1460         vnic_cq_clean(&enic->cq[cq_idx]);
1461         vnic_cq_init(&enic->cq[cq_idx],
1462                      0 /* flow_control_enable */,
1463                      1 /* color_enable */,
1464                      0 /* cq_head */,
1465                      0 /* cq_tail */,
1466                      1 /* cq_tail_color */,
1467                      0 /* interrupt_enable */,
1468                      1 /* cq_entry_enable */,
1469                      0 /* cq_message_enable */,
1470                      0 /* interrupt offset */,
1471                      0 /* cq_message_addr */);
1472
1473
1474         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1475                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1476                            sop_rq->ring.desc_count - 1, 1, 0);
1477         if (data_rq->in_use) {
1478                 vnic_rq_init_start(data_rq,
1479                                    enic_cq_rq(enic,
1480                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1481                                    data_rq->ring.desc_count - 1, 1, 0);
1482         }
1483
1484         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1485         if (rc)
1486                 return rc;
1487
1488         if (data_rq->in_use) {
1489                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1490                 if (rc) {
1491                         enic_rxmbuf_queue_release(enic, sop_rq);
1492                         return rc;
1493                 }
1494         }
1495
1496         return 0;
1497 }
1498
1499 /* The Cisco NIC can send and receive packets up to a max packet size
1500  * determined by the NIC type and firmware. There is also an MTU
1501  * configured into the NIC via the CIMC/UCSM management interface
1502  * which can be overridden by this function (up to the max packet size).
1503  * Depending on the network setup, doing so may cause packet drops
1504  * and unexpected behavior.
1505  */
1506 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1507 {
1508         unsigned int rq_idx;
1509         struct vnic_rq *rq;
1510         int rc = 0;
1511         uint16_t old_mtu;       /* previous setting */
1512         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1513         struct rte_eth_dev *eth_dev = enic->rte_dev;
1514
1515         old_mtu = eth_dev->data->mtu;
1516         config_mtu = enic->config.mtu;
1517
1518         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1519                 return -E_RTE_SECONDARY;
1520
1521         if (new_mtu > enic->max_mtu) {
1522                 dev_err(enic,
1523                         "MTU not updated: requested (%u) greater than max (%u)\n",
1524                         new_mtu, enic->max_mtu);
1525                 return -EINVAL;
1526         }
1527         if (new_mtu < ENIC_MIN_MTU) {
1528                 dev_info(enic,
1529                         "MTU not updated: requested (%u) less than min (%u)\n",
1530                         new_mtu, ENIC_MIN_MTU);
1531                 return -EINVAL;
1532         }
1533         if (new_mtu > config_mtu)
1534                 dev_warning(enic,
1535                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1536                         new_mtu, config_mtu);
1537
1538         /* Update the MTU and maximum packet length */
1539         eth_dev->data->mtu = new_mtu;
1540         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1541                 enic_mtu_to_max_rx_pktlen(new_mtu);
1542
1543         /*
1544          * If the device has not started (enic_enable), nothing to do.
1545          * Later, enic_enable() will set up RQs reflecting the new maximum
1546          * packet length.
1547          */
1548         if (!eth_dev->data->dev_started)
1549                 goto set_mtu_done;
1550
1551         /*
1552          * The device has started, re-do RQs on the fly. In the process, we
1553          * pick up the new maximum packet length.
1554          *
1555          * Some applications rely on the ability to change MTU without stopping
1556          * the device. So keep this behavior for now.
1557          */
1558         rte_spinlock_lock(&enic->mtu_lock);
1559
1560         /* Stop traffic on all RQs */
1561         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1562                 rq = &enic->rq[rq_idx];
1563                 if (rq->is_sop && rq->in_use) {
1564                         rc = enic_stop_rq(enic,
1565                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1566                         if (rc) {
1567                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1568                                 goto set_mtu_done;
1569                         }
1570                 }
1571         }
1572
1573         /* replace Rx function with a no-op to avoid getting stale pkts */
1574         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1575         rte_mb();
1576
1577         /* Allow time for threads to exit the real Rx function. */
1578         usleep(100000);
1579
1580         /* now it is safe to reconfigure the RQs */
1581
1582
1583         /* free and reallocate RQs with the new MTU */
1584         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1585                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1586                 if (!rq->in_use)
1587                         continue;
1588
1589                 enic_free_rq(rq);
1590                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1591                                    rq->tot_nb_desc, rq->rx_free_thresh);
1592                 if (rc) {
1593                         dev_err(enic,
1594                                 "Fatal MTU alloc error- No traffic will pass\n");
1595                         goto set_mtu_done;
1596                 }
1597
1598                 rc = enic_reinit_rq(enic, rq_idx);
1599                 if (rc) {
1600                         dev_err(enic,
1601                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1602                         goto set_mtu_done;
1603                 }
1604         }
1605
1606         /* put back the real receive function */
1607         rte_mb();
1608         pick_rx_handler(enic);
1609         rte_mb();
1610
1611         /* restart Rx traffic */
1612         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1613                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1614                 if (rq->is_sop && rq->in_use)
1615                         enic_start_rq(enic, rq_idx);
1616         }
1617
1618 set_mtu_done:
1619         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1620         rte_spinlock_unlock(&enic->mtu_lock);
1621         return rc;
1622 }
1623
1624 static int enic_dev_init(struct enic *enic)
1625 {
1626         int err;
1627         struct rte_eth_dev *eth_dev = enic->rte_dev;
1628
1629         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1630
1631         /* Get vNIC configuration
1632         */
1633         err = enic_get_vnic_config(enic);
1634         if (err) {
1635                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1636                 return err;
1637         }
1638
1639         /* Get available resource counts */
1640         enic_get_res_counts(enic);
1641         if (enic->conf_rq_count == 1) {
1642                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1643                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1644                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1645                 return -EINVAL;
1646         }
1647         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1648         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1649                                enic->conf_cq_count, 8);
1650         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1651                                  enic->conf_intr_count, 8);
1652         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1653                                enic->conf_rq_count, 8);
1654         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1655                                enic->conf_wq_count, 8);
1656         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1657                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1658                 return -1;
1659         }
1660         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1661                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1662                 return -1;
1663         }
1664         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1665                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1666                 return -1;
1667         }
1668         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1669                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1670                 return -1;
1671         }
1672
1673         /* Get the supported filters */
1674         enic_fdir_info(enic);
1675
1676         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1677                                         sizeof(struct rte_ether_addr) *
1678                                         ENIC_UNICAST_PERFECT_FILTERS, 0);
1679         if (!eth_dev->data->mac_addrs) {
1680                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1681                 return -1;
1682         }
1683         rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1684                         eth_dev->data->mac_addrs);
1685
1686         vnic_dev_set_reset_flag(enic->vdev, 0);
1687
1688         LIST_INIT(&enic->flows);
1689
1690         /* set up link status checking */
1691         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1692
1693         enic->overlay_offload = false;
1694         if (enic->disable_overlay && enic->vxlan) {
1695                 /*
1696                  * Explicitly disable overlay offload as the setting is
1697                  * sticky, and resetting vNIC does not disable it.
1698                  */
1699                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1700                                                   OVERLAY_FEATURE_VXLAN,
1701                                                   OVERLAY_OFFLOAD_DISABLE)) {
1702                         dev_err(enic, "failed to disable overlay offload\n");
1703                 } else {
1704                         dev_info(enic, "Overlay offload is disabled\n");
1705                 }
1706         }
1707         if (!enic->disable_overlay && enic->vxlan &&
1708             /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1709             vnic_dev_overlay_offload_ctrl(enic->vdev,
1710                                           OVERLAY_FEATURE_VXLAN,
1711                                           OVERLAY_OFFLOAD_ENABLE) == 0) {
1712                 enic->tx_offload_capa |=
1713                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1714                         DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1715                         DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1716                 enic->tx_offload_mask |=
1717                         PKT_TX_OUTER_IPV6 |
1718                         PKT_TX_OUTER_IPV4 |
1719                         PKT_TX_OUTER_IP_CKSUM |
1720                         PKT_TX_TUNNEL_MASK;
1721                 enic->overlay_offload = true;
1722                 dev_info(enic, "Overlay offload is enabled\n");
1723         }
1724         /*
1725          * Reset the vxlan port if HW vxlan parsing is available. It
1726          * is always enabled regardless of overlay offload
1727          * enable/disable.
1728          */
1729         if (enic->vxlan) {
1730                 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1731                 /*
1732                  * Reset the vxlan port to the default, as the NIC firmware
1733                  * does not reset it automatically and keeps the old setting.
1734                  */
1735                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1736                                                  OVERLAY_CFG_VXLAN_PORT_UPDATE,
1737                                                  ENIC_DEFAULT_VXLAN_PORT)) {
1738                         dev_err(enic, "failed to update vxlan port\n");
1739                         return -EINVAL;
1740                 }
1741         }
1742
1743         return 0;
1744
1745 }
1746
1747 int enic_probe(struct enic *enic)
1748 {
1749         struct rte_pci_device *pdev = enic->pdev;
1750         int err = -1;
1751
1752         dev_debug(enic, "Initializing ENIC PMD\n");
1753
1754         /* if this is a secondary process the hardware is already initialized */
1755         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1756                 return 0;
1757
1758         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1759         enic->bar0.len = pdev->mem_resource[0].len;
1760
1761         /* Register vNIC device */
1762         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1763         if (!enic->vdev) {
1764                 dev_err(enic, "vNIC registration failed, aborting\n");
1765                 goto err_out;
1766         }
1767
1768         LIST_INIT(&enic->memzone_list);
1769         rte_spinlock_init(&enic->memzone_list_lock);
1770
1771         vnic_register_cbacks(enic->vdev,
1772                 enic_alloc_consistent,
1773                 enic_free_consistent);
1774
1775         /*
1776          * Allocate the consistent memory for stats upfront so both primary and
1777          * secondary processes can dump stats.
1778          */
1779         err = vnic_dev_alloc_stats_mem(enic->vdev);
1780         if (err) {
1781                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1782                 goto err_out_unregister;
1783         }
1784         /* Issue device open to get device in known state */
1785         err = enic_dev_open(enic);
1786         if (err) {
1787                 dev_err(enic, "vNIC dev open failed, aborting\n");
1788                 goto err_out_unregister;
1789         }
1790
1791         /* Set ingress vlan rewrite mode before vnic initialization */
1792         dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1793                   enic->ig_vlan_rewrite_mode);
1794         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1795                 enic->ig_vlan_rewrite_mode);
1796         if (err) {
1797                 dev_err(enic,
1798                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1799                 goto err_out_dev_close;
1800         }
1801
1802         /* Issue device init to initialize the vnic-to-switch link.
1803          * We'll start with carrier off and wait for link UP
1804          * notification later to turn on carrier.  We don't need
1805          * to wait here for the vnic-to-switch link initialization
1806          * to complete; link UP notification is the indication that
1807          * the process is complete.
1808          */
1809
1810         err = vnic_dev_init(enic->vdev, 0);
1811         if (err) {
1812                 dev_err(enic, "vNIC dev init failed, aborting\n");
1813                 goto err_out_dev_close;
1814         }
1815
1816         err = enic_dev_init(enic);
1817         if (err) {
1818                 dev_err(enic, "Device initialization failed, aborting\n");
1819                 goto err_out_dev_close;
1820         }
1821
1822         return 0;
1823
1824 err_out_dev_close:
1825         vnic_dev_close(enic->vdev);
1826 err_out_unregister:
1827         vnic_dev_unregister(enic->vdev);
1828 err_out:
1829         return err;
1830 }
1831
1832 void enic_remove(struct enic *enic)
1833 {
1834         enic_dev_deinit(enic);
1835         vnic_dev_close(enic->vdev);
1836         vnic_dev_unregister(enic->vdev);
1837 }