drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11
  12 #include <rte_pci.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_ethdev_driver.h>
  19
  20 #include "enic_compat.h"
  21 #include "enic.h"
  22 #include "wq_enet_desc.h"
  23 #include "rq_enet_desc.h"
  24 #include "cq_enet_desc.h"
  25 #include "vnic_enet.h"
  26 #include "vnic_dev.h"
  27 #include "vnic_wq.h"
  28 #include "vnic_rq.h"
  29 #include "vnic_cq.h"
  30 #include "vnic_intr.h"
  31 #include "vnic_nic.h"
  32
  33 static inline int enic_is_sriov_vf(struct enic *enic)
  34 {
  35         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  36 }
  37
  38 static int is_zero_addr(uint8_t *addr)
  39 {
  40         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  41 }
  42
  43 static int is_mcast_addr(uint8_t *addr)
  44 {
  45         return addr[0] & 1;
  46 }
  47
  48 static int is_eth_addr_valid(uint8_t *addr)
  49 {
  50         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  51 }
  52
  53 static void
  54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  55 {
  56         uint16_t i;
  57
  58         if (!rq || !rq->mbuf_ring) {
  59                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  60                 return;
  61         }
  62
  63         for (i = 0; i < rq->ring.desc_count; i++) {
  64                 if (rq->mbuf_ring[i]) {
  65                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  66                         rq->mbuf_ring[i] = NULL;
  67                 }
  68         }
  69 }
  70
  71 static void enic_free_wq_buf(struct rte_mbuf **buf)
  72 {
  73         struct rte_mbuf *mbuf = *buf;
  74
  75         rte_pktmbuf_free_seg(mbuf);
  76         *buf = NULL;
  77 }
  78
  79 static void enic_log_q_error(struct enic *enic)
  80 {
  81         unsigned int i;
  82         u32 error_status;
  83
  84         for (i = 0; i < enic->wq_count; i++) {
  85                 error_status = vnic_wq_error_status(&enic->wq[i]);
  86                 if (error_status)
  87                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  88                                 error_status);
  89         }
  90
  91         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  92                 if (!enic->rq[i].in_use)
  93                         continue;
  94                 error_status = vnic_rq_error_status(&enic->rq[i]);
  95                 if (error_status)
  96                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  97                                 error_status);
  98         }
  99 }
 100
 101 static void enic_clear_soft_stats(struct enic *enic)
 102 {
 103         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 104         rte_atomic64_clear(&soft_stats->rx_nombuf);
 105         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 106         rte_atomic64_clear(&soft_stats->tx_oversized);
 107 }
 108
 109 static void enic_init_soft_stats(struct enic *enic)
 110 {
 111         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 112         rte_atomic64_init(&soft_stats->rx_nombuf);
 113         rte_atomic64_init(&soft_stats->rx_packet_errors);
 114         rte_atomic64_init(&soft_stats->tx_oversized);
 115         enic_clear_soft_stats(enic);
 116 }
 117
 118 void enic_dev_stats_clear(struct enic *enic)
 119 {
 120         if (vnic_dev_stats_clear(enic->vdev))
 121                 dev_err(enic, "Error in clearing stats\n");
 122         enic_clear_soft_stats(enic);
 123 }
 124
 125 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 126 {
 127         struct vnic_stats *stats;
 128         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 129         int64_t rx_truncated;
 130         uint64_t rx_packet_errors;
 131         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 132
 133         if (ret) {
 134                 dev_err(enic, "Error in getting stats\n");
 135                 return ret;
 136         }
 137
 138         /* The number of truncated packets can only be calculated by
 139          * subtracting a hardware counter from error packets received by
 140          * the driver. Note: this causes transient inaccuracies in the
 141          * ipackets count. Also, the length of truncated packets are
 142          * counted in ibytes even though truncated packets are dropped
 143          * which can make ibytes be slightly higher than it should be.
 144          */
 145         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 146         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 147
 148         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 149         r_stats->opackets = stats->tx.tx_frames_ok;
 150
 151         r_stats->ibytes = stats->rx.rx_bytes_ok;
 152         r_stats->obytes = stats->tx.tx_bytes_ok;
 153
 154         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 155         r_stats->oerrors = stats->tx.tx_errors
 156                            + rte_atomic64_read(&soft_stats->tx_oversized);
 157
 158         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 159
 160         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 161         return 0;
 162 }
 163
 164 int enic_del_mac_address(struct enic *enic, int mac_index)
 165 {
 166         struct rte_eth_dev *eth_dev = enic->rte_dev;
 167         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 168
 169         return vnic_dev_del_addr(enic->vdev, mac_addr);
 170 }
 171
 172 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 173 {
 174         int err;
 175
 176         if (!is_eth_addr_valid(mac_addr)) {
 177                 dev_err(enic, "invalid mac address\n");
 178                 return -EINVAL;
 179         }
 180
 181         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 182         if (err)
 183                 dev_err(enic, "add mac addr failed\n");
 184         return err;
 185 }
 186
 187 static void
 188 enic_free_rq_buf(struct rte_mbuf **mbuf)
 189 {
 190         if (*mbuf == NULL)
 191                 return;
 192
 193         rte_pktmbuf_free(*mbuf);
 194         *mbuf = NULL;
 195 }
 196
 197 void enic_init_vnic_resources(struct enic *enic)
 198 {
 199         unsigned int error_interrupt_enable = 1;
 200         unsigned int error_interrupt_offset = 0;
 201         unsigned int rxq_interrupt_enable = 0;
 202         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 203         unsigned int index = 0;
 204         unsigned int cq_idx;
 205         struct vnic_rq *data_rq;
 206
 207         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 208                 rxq_interrupt_enable = 1;
 209
 210         for (index = 0; index < enic->rq_count; index++) {
 211                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 212
 213                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 214                         cq_idx,
 215                         error_interrupt_enable,
 216                         error_interrupt_offset);
 217
 218                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 219                 if (data_rq->in_use)
 220                         vnic_rq_init(data_rq,
 221                                      cq_idx,
 222                                      error_interrupt_enable,
 223                                      error_interrupt_offset);
 224
 225                 vnic_cq_init(&enic->cq[cq_idx],
 226                         0 /* flow_control_enable */,
 227                         1 /* color_enable */,
 228                         0 /* cq_head */,
 229                         0 /* cq_tail */,
 230                         1 /* cq_tail_color */,
 231                         rxq_interrupt_enable,
 232                         1 /* cq_entry_enable */,
 233                         0 /* cq_message_enable */,
 234                         rxq_interrupt_offset,
 235                         0 /* cq_message_addr */);
 236                 if (rxq_interrupt_enable)
 237                         rxq_interrupt_offset++;
 238         }
 239
 240         for (index = 0; index < enic->wq_count; index++) {
 241                 vnic_wq_init(&enic->wq[index],
 242                         enic_cq_wq(enic, index),
 243                         error_interrupt_enable,
 244                         error_interrupt_offset);
 245                 /* Compute unsupported ol flags for enic_prep_pkts() */
 246                 enic->wq[index].tx_offload_notsup_mask =
 247                         PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 248
 249                 cq_idx = enic_cq_wq(enic, index);
 250                 vnic_cq_init(&enic->cq[cq_idx],
 251                         0 /* flow_control_enable */,
 252                         1 /* color_enable */,
 253                         0 /* cq_head */,
 254                         0 /* cq_tail */,
 255                         1 /* cq_tail_color */,
 256                         0 /* interrupt_enable */,
 257                         0 /* cq_entry_enable */,
 258                         1 /* cq_message_enable */,
 259                         0 /* interrupt offset */,
 260                         (u64)enic->wq[index].cqmsg_rz->iova);
 261         }
 262
 263         for (index = 0; index < enic->intr_count; index++) {
 264                 vnic_intr_init(&enic->intr[index],
 265                                enic->config.intr_timer_usec,
 266                                enic->config.intr_timer_type,
 267                                /*mask_on_assertion*/1);
 268         }
 269 }
 270
 271
 272 static int
 273 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 274 {
 275         struct rte_mbuf *mb;
 276         struct rq_enet_desc *rqd = rq->ring.descs;
 277         unsigned i;
 278         dma_addr_t dma_addr;
 279         uint32_t max_rx_pkt_len;
 280         uint16_t rq_buf_len;
 281
 282         if (!rq->in_use)
 283                 return 0;
 284
 285         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 286                   rq->ring.desc_count);
 287
 288         /*
 289          * If *not* using scatter and the mbuf size is greater than the
 290          * requested max packet size (max_rx_pkt_len), then reduce the
 291          * posted buffer size to max_rx_pkt_len. HW still receives packets
 292          * larger than max_rx_pkt_len, but they will be truncated, which we
 293          * drop in the rx handler. Not ideal, but better than returning
 294          * large packets when the user is not expecting them.
 295          */
 296         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 297         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 298         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 299                 rq_buf_len = max_rx_pkt_len;
 300         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 301                 mb = rte_mbuf_raw_alloc(rq->mp);
 302                 if (mb == NULL) {
 303                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 304                         (unsigned)rq->index);
 305                         return -ENOMEM;
 306                 }
 307
 308                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 309                 dma_addr = (dma_addr_t)(mb->buf_iova
 310                            + RTE_PKTMBUF_HEADROOM);
 311                 rq_enet_desc_enc(rqd, dma_addr,
 312                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 313                                 : RQ_ENET_TYPE_NOT_SOP),
 314                                 rq_buf_len);
 315                 rq->mbuf_ring[i] = mb;
 316         }
 317         /*
 318          * Do not post the buffers to the NIC until we enable the RQ via
 319          * enic_start_rq().
 320          */
 321         rq->need_initial_post = true;
 322         /* Initialize fetch index while RQ is disabled */
 323         iowrite32(0, &rq->ctrl->fetch_index);
 324         return 0;
 325 }
 326
 327 /*
 328  * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
 329  * allocated the buffers and filled the RQ descriptor ring. Just need to push
 330  * the post index to the NIC.
 331  */
 332 static void
 333 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 334 {
 335         if (!rq->in_use || !rq->need_initial_post)
 336                 return;
 337
 338         /* make sure all prior writes are complete before doing the PIO write */
 339         rte_rmb();
 340
 341         /* Post all but the last buffer to VIC. */
 342         rq->posted_index = rq->ring.desc_count - 1;
 343
 344         rq->rx_nb_hold = 0;
 345
 346         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 347                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 348         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 349         rte_rmb();
 350         rq->need_initial_post = false;
 351 }
 352
 353 static void *
 354 enic_alloc_consistent(void *priv, size_t size,
 355         dma_addr_t *dma_handle, u8 *name)
 356 {
 357         void *vaddr;
 358         const struct rte_memzone *rz;
 359         *dma_handle = 0;
 360         struct enic *enic = (struct enic *)priv;
 361         struct enic_memzone_entry *mze;
 362
 363         rz = rte_memzone_reserve_aligned((const char *)name, size,
 364                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 365         if (!rz) {
 366                 pr_err("%s : Failed to allocate memory requested for %s\n",
 367                         __func__, name);
 368                 return NULL;
 369         }
 370
 371         vaddr = rz->addr;
 372         *dma_handle = (dma_addr_t)rz->iova;
 373
 374         mze = rte_malloc("enic memzone entry",
 375                          sizeof(struct enic_memzone_entry), 0);
 376
 377         if (!mze) {
 378                 pr_err("%s : Failed to allocate memory for memzone list\n",
 379                        __func__);
 380                 rte_memzone_free(rz);
 381                 return NULL;
 382         }
 383
 384         mze->rz = rz;
 385
 386         rte_spinlock_lock(&enic->memzone_list_lock);
 387         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 388         rte_spinlock_unlock(&enic->memzone_list_lock);
 389
 390         return vaddr;
 391 }
 392
 393 static void
 394 enic_free_consistent(void *priv,
 395                      __rte_unused size_t size,
 396                      void *vaddr,
 397                      dma_addr_t dma_handle)
 398 {
 399         struct enic_memzone_entry *mze;
 400         struct enic *enic = (struct enic *)priv;
 401
 402         rte_spinlock_lock(&enic->memzone_list_lock);
 403         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 404                 if (mze->rz->addr == vaddr &&
 405                     mze->rz->iova == dma_handle)
 406                         break;
 407         }
 408         if (mze == NULL) {
 409                 rte_spinlock_unlock(&enic->memzone_list_lock);
 410                 dev_warning(enic,
 411                             "Tried to free memory, but couldn't find it in the memzone list\n");
 412                 return;
 413         }
 414         LIST_REMOVE(mze, entries);
 415         rte_spinlock_unlock(&enic->memzone_list_lock);
 416         rte_memzone_free(mze->rz);
 417         rte_free(mze);
 418 }
 419
 420 int enic_link_update(struct enic *enic)
 421 {
 422         struct rte_eth_dev *eth_dev = enic->rte_dev;
 423         struct rte_eth_link link;
 424
 425         memset(&link, 0, sizeof(link));
 426         link.link_status = enic_get_link_status(enic);
 427         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 428         link.link_speed = vnic_dev_port_speed(enic->vdev);
 429
 430         return rte_eth_linkstatus_set(eth_dev, &link);
 431 }
 432
 433 static void
 434 enic_intr_handler(void *arg)
 435 {
 436         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 437         struct enic *enic = pmd_priv(dev);
 438
 439         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 440
 441         enic_link_update(enic);
 442         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 443         enic_log_q_error(enic);
 444 }
 445
 446 static int enic_rxq_intr_init(struct enic *enic)
 447 {
 448         struct rte_intr_handle *intr_handle;
 449         uint32_t rxq_intr_count, i;
 450         int err;
 451
 452         intr_handle = enic->rte_dev->intr_handle;
 453         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 454                 return 0;
 455         /*
 456          * Rx queue interrupts only work when we have MSI-X interrupts,
 457          * one per queue. Sharing one interrupt is technically
 458          * possible with VIC, but it is not worth the complications it brings.
 459          */
 460         if (!rte_intr_cap_multiple(intr_handle)) {
 461                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 462                         " (vfio-pci driver)\n");
 463                 return -ENOTSUP;
 464         }
 465         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 466         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 467         if (err) {
 468                 dev_err(enic, "Failed to enable event fds for Rx queue"
 469                         " interrupts\n");
 470                 return err;
 471         }
 472         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 473                                             rxq_intr_count * sizeof(int), 0);
 474         if (intr_handle->intr_vec == NULL) {
 475                 dev_err(enic, "Failed to allocate intr_vec\n");
 476                 return -ENOMEM;
 477         }
 478         for (i = 0; i < rxq_intr_count; i++)
 479                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 480         return 0;
 481 }
 482
 483 static void enic_rxq_intr_deinit(struct enic *enic)
 484 {
 485         struct rte_intr_handle *intr_handle;
 486
 487         intr_handle = enic->rte_dev->intr_handle;
 488         rte_intr_efd_disable(intr_handle);
 489         if (intr_handle->intr_vec != NULL) {
 490                 rte_free(intr_handle->intr_vec);
 491                 intr_handle->intr_vec = NULL;
 492         }
 493 }
 494
 495 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 496 {
 497         struct wq_enet_desc *desc;
 498         struct vnic_wq *wq;
 499         unsigned int i;
 500
 501         /*
 502          * Fill WQ descriptor fields that never change. Every descriptor is
 503          * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
 504          * descriptors (i.e. request one completion update every 32 packets).
 505          */
 506         wq = &enic->wq[queue_idx];
 507         desc = (struct wq_enet_desc *)wq->ring.descs;
 508         for (i = 0; i < wq->ring.desc_count; i++, desc++) {
 509                 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
 510                 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
 511                         desc->header_length_flags |=
 512                                 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
 513         }
 514 }
 515
 516 /*
 517  * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
 518  * used when that file is not compiled.
 519  */
 520 __rte_weak bool
 521 enic_use_vector_rx_handler(__rte_unused struct enic *enic)
 522 {
 523         return false;
 524 }
 525
 526 static void pick_rx_handler(struct enic *enic)
 527 {
 528         struct rte_eth_dev *eth_dev;
 529
 530         /*
 531          * Preference order:
 532          * 1. The vectorized handler if possible and requested.
 533          * 2. The non-scatter, simplified handler if scatter Rx is not used.
 534          * 3. The default handler as a fallback.
 535          */
 536         eth_dev = enic->rte_dev;
 537         if (enic_use_vector_rx_handler(enic))
 538                 return;
 539         if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 540                 PMD_INIT_LOG(DEBUG, " use the non-scatter Rx handler");
 541                 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
 542         } else {
 543                 PMD_INIT_LOG(DEBUG, " use the normal Rx handler");
 544                 eth_dev->rx_pkt_burst = &enic_recv_pkts;
 545         }
 546 }
 547
 548 int enic_enable(struct enic *enic)
 549 {
 550         unsigned int index;
 551         int err;
 552         struct rte_eth_dev *eth_dev = enic->rte_dev;
 553         uint64_t simple_tx_offloads;
 554         uintptr_t p;
 555
 556         if (enic->enable_avx2_rx) {
 557                 struct rte_mbuf mb_def = { .buf_addr = 0 };
 558
 559                 /*
 560                  * mbuf_initializer contains const-after-init fields of
 561                  * receive mbufs (i.e. 64 bits of fields from rearm_data).
 562                  * It is currently used by the vectorized handler.
 563                  */
 564                 mb_def.nb_segs = 1;
 565                 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 566                 mb_def.port = enic->port_id;
 567                 rte_mbuf_refcnt_set(&mb_def, 1);
 568                 rte_compiler_barrier();
 569                 p = (uintptr_t)&mb_def.rearm_data;
 570                 enic->mbuf_initializer = *(uint64_t *)p;
 571         }
 572
 573         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 574         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 575
 576         /* vnic notification of link status has already been turned on in
 577          * enic_dev_init() which is called during probe time.  Here we are
 578          * just turning on interrupt vector 0 if needed.
 579          */
 580         if (eth_dev->data->dev_conf.intr_conf.lsc)
 581                 vnic_dev_notify_set(enic->vdev, 0);
 582
 583         err = enic_rxq_intr_init(enic);
 584         if (err)
 585                 return err;
 586         if (enic_clsf_init(enic))
 587                 dev_warning(enic, "Init of hash table for clsf failed."\
 588                         "Flow director feature will not work\n");
 589
 590         for (index = 0; index < enic->rq_count; index++) {
 591                 err = enic_alloc_rx_queue_mbufs(enic,
 592                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 593                 if (err) {
 594                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 595                         return err;
 596                 }
 597                 err = enic_alloc_rx_queue_mbufs(enic,
 598                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 599                 if (err) {
 600                         /* release the allocated mbufs for the sop rq*/
 601                         enic_rxmbuf_queue_release(enic,
 602                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 603
 604                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 605                         return err;
 606                 }
 607         }
 608
 609         /*
 610          * Use the simple TX handler if possible. Only checksum offloads
 611          * and vlan insertion are supported.
 612          */
 613         simple_tx_offloads = enic->tx_offload_capa &
 614                 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 615                  DEV_TX_OFFLOAD_VLAN_INSERT |
 616                  DEV_TX_OFFLOAD_IPV4_CKSUM |
 617                  DEV_TX_OFFLOAD_UDP_CKSUM |
 618                  DEV_TX_OFFLOAD_TCP_CKSUM);
 619         if ((eth_dev->data->dev_conf.txmode.offloads &
 620              ~simple_tx_offloads) == 0) {
 621                 PMD_INIT_LOG(DEBUG, " use the simple tx handler");
 622                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 623                 for (index = 0; index < enic->wq_count; index++)
 624                         enic_prep_wq_for_simple_tx(enic, index);
 625         } else {
 626                 PMD_INIT_LOG(DEBUG, " use the default tx handler");
 627                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 628         }
 629
 630         pick_rx_handler(enic);
 631
 632         for (index = 0; index < enic->wq_count; index++)
 633                 enic_start_wq(enic, index);
 634         for (index = 0; index < enic->rq_count; index++)
 635                 enic_start_rq(enic, index);
 636
 637         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 638
 639         vnic_dev_enable_wait(enic->vdev);
 640
 641         /* Register and enable error interrupt */
 642         rte_intr_callback_register(&(enic->pdev->intr_handle),
 643                 enic_intr_handler, (void *)enic->rte_dev);
 644
 645         rte_intr_enable(&(enic->pdev->intr_handle));
 646         /* Unmask LSC interrupt */
 647         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 648
 649         return 0;
 650 }
 651
 652 int enic_alloc_intr_resources(struct enic *enic)
 653 {
 654         int err;
 655         unsigned int i;
 656
 657         dev_info(enic, "vNIC resources used:  "\
 658                 "wq %d rq %d cq %d intr %d\n",
 659                 enic->wq_count, enic_vnic_rq_count(enic),
 660                 enic->cq_count, enic->intr_count);
 661
 662         for (i = 0; i < enic->intr_count; i++) {
 663                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 664                 if (err) {
 665                         enic_free_vnic_resources(enic);
 666                         return err;
 667                 }
 668         }
 669         return 0;
 670 }
 671
 672 void enic_free_rq(void *rxq)
 673 {
 674         struct vnic_rq *rq_sop, *rq_data;
 675         struct enic *enic;
 676
 677         if (rxq == NULL)
 678                 return;
 679
 680         rq_sop = (struct vnic_rq *)rxq;
 681         enic = vnic_dev_priv(rq_sop->vdev);
 682         rq_data = &enic->rq[rq_sop->data_queue_idx];
 683
 684         if (rq_sop->free_mbufs) {
 685                 struct rte_mbuf **mb;
 686                 int i;
 687
 688                 mb = rq_sop->free_mbufs;
 689                 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
 690                      i < ENIC_RX_BURST_MAX; i++)
 691                         rte_pktmbuf_free(mb[i]);
 692                 rte_free(rq_sop->free_mbufs);
 693                 rq_sop->free_mbufs = NULL;
 694                 rq_sop->num_free_mbufs = 0;
 695         }
 696
 697         enic_rxmbuf_queue_release(enic, rq_sop);
 698         if (rq_data->in_use)
 699                 enic_rxmbuf_queue_release(enic, rq_data);
 700
 701         rte_free(rq_sop->mbuf_ring);
 702         if (rq_data->in_use)
 703                 rte_free(rq_data->mbuf_ring);
 704
 705         rq_sop->mbuf_ring = NULL;
 706         rq_data->mbuf_ring = NULL;
 707
 708         vnic_rq_free(rq_sop);
 709         if (rq_data->in_use)
 710                 vnic_rq_free(rq_data);
 711
 712         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 713
 714         rq_sop->in_use = 0;
 715         rq_data->in_use = 0;
 716 }
 717
 718 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 719 {
 720         struct rte_eth_dev *eth_dev = enic->rte_dev;
 721         vnic_wq_enable(&enic->wq[queue_idx]);
 722         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 723 }
 724
 725 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 726 {
 727         struct rte_eth_dev *eth_dev = enic->rte_dev;
 728         int ret;
 729
 730         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 731         if (ret)
 732                 return ret;
 733
 734         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 735         return 0;
 736 }
 737
 738 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 739 {
 740         struct vnic_rq *rq_sop;
 741         struct vnic_rq *rq_data;
 742         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 743         rq_data = &enic->rq[rq_sop->data_queue_idx];
 744         struct rte_eth_dev *eth_dev = enic->rte_dev;
 745
 746         if (rq_data->in_use) {
 747                 vnic_rq_enable(rq_data);
 748                 enic_initial_post_rx(enic, rq_data);
 749         }
 750         rte_mb();
 751         vnic_rq_enable(rq_sop);
 752         enic_initial_post_rx(enic, rq_sop);
 753         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 754 }
 755
 756 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 757 {
 758         int ret1 = 0, ret2 = 0;
 759         struct rte_eth_dev *eth_dev = enic->rte_dev;
 760         struct vnic_rq *rq_sop;
 761         struct vnic_rq *rq_data;
 762         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 763         rq_data = &enic->rq[rq_sop->data_queue_idx];
 764
 765         ret2 = vnic_rq_disable(rq_sop);
 766         rte_mb();
 767         if (rq_data->in_use)
 768                 ret1 = vnic_rq_disable(rq_data);
 769
 770         if (ret2)
 771                 return ret2;
 772         else if (ret1)
 773                 return ret1;
 774
 775         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 776         return 0;
 777 }
 778
 779 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 780         unsigned int socket_id, struct rte_mempool *mp,
 781         uint16_t nb_desc, uint16_t free_thresh)
 782 {
 783         int rc;
 784         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 785         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 786         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 787         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 788         unsigned int mbuf_size, mbufs_per_pkt;
 789         unsigned int nb_sop_desc, nb_data_desc;
 790         uint16_t min_sop, max_sop, min_data, max_data;
 791         uint32_t max_rx_pkt_len;
 792
 793         rq_sop->is_sop = 1;
 794         rq_sop->data_queue_idx = data_queue_idx;
 795         rq_data->is_sop = 0;
 796         rq_data->data_queue_idx = 0;
 797         rq_sop->socket_id = socket_id;
 798         rq_sop->mp = mp;
 799         rq_data->socket_id = socket_id;
 800         rq_data->mp = mp;
 801         rq_sop->in_use = 1;
 802         rq_sop->rx_free_thresh = free_thresh;
 803         rq_data->rx_free_thresh = free_thresh;
 804         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 805                   free_thresh);
 806
 807         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 808                                RTE_PKTMBUF_HEADROOM);
 809         /* max_rx_pkt_len includes the ethernet header and CRC. */
 810         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 811
 812         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 813             DEV_RX_OFFLOAD_SCATTER) {
 814                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 815                 /* ceil((max pkt len)/mbuf_size) */
 816                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 817         } else {
 818                 dev_info(enic, "Scatter rx mode disabled\n");
 819                 mbufs_per_pkt = 1;
 820                 if (max_rx_pkt_len > mbuf_size) {
 821                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 822                                     " larger than the mbuf size (%u), and"
 823                                     " scatter is disabled. Larger packets will"
 824                                     " be truncated.\n",
 825                                     max_rx_pkt_len, mbuf_size);
 826                 }
 827         }
 828
 829         if (mbufs_per_pkt > 1) {
 830                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 831                 rq_sop->data_queue_enable = 1;
 832                 rq_data->in_use = 1;
 833                 /*
 834                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 835                  * receives packet sizes up to the "max" MTU.
 836                  * If not using scatter, we can achieve the effect of dropping
 837                  * larger packets by reducing the size of posted buffers.
 838                  * See enic_alloc_rx_queue_mbufs().
 839                  */
 840                 if (max_rx_pkt_len <
 841                     enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
 842                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 843                                     " when scatter rx mode is in use.\n");
 844                 }
 845         } else {
 846                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 847                          queue_idx);
 848                 rq_sop->data_queue_enable = 0;
 849                 rq_data->in_use = 0;
 850         }
 851
 852         /* number of descriptors have to be a multiple of 32 */
 853         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
 854         nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 855
 856         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 857         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 858
 859         if (mbufs_per_pkt > 1) {
 860                 min_sop = ENIC_RX_BURST_MAX;
 861                 max_sop = ((enic->config.rq_desc_count /
 862                             (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 863                 min_data = min_sop * (mbufs_per_pkt - 1);
 864                 max_data = enic->config.rq_desc_count;
 865         } else {
 866                 min_sop = ENIC_RX_BURST_MAX;
 867                 max_sop = enic->config.rq_desc_count;
 868                 min_data = 0;
 869                 max_data = 0;
 870         }
 871
 872         if (nb_desc < (min_sop + min_data)) {
 873                 dev_warning(enic,
 874                             "Number of rx descs too low, adjusting to minimum\n");
 875                 nb_sop_desc = min_sop;
 876                 nb_data_desc = min_data;
 877         } else if (nb_desc > (max_sop + max_data)) {
 878                 dev_warning(enic,
 879                             "Number of rx_descs too high, adjusting to maximum\n");
 880                 nb_sop_desc = max_sop;
 881                 nb_data_desc = max_data;
 882         }
 883         if (mbufs_per_pkt > 1) {
 884                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 885                          " rx descriptor range is %u to %u\n",
 886                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 887                          max_sop + max_data);
 888         }
 889         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 890                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 891
 892         /* Allocate sop queue resources */
 893         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 894                 nb_sop_desc, sizeof(struct rq_enet_desc));
 895         if (rc) {
 896                 dev_err(enic, "error in allocation of sop rq\n");
 897                 goto err_exit;
 898         }
 899         nb_sop_desc = rq_sop->ring.desc_count;
 900
 901         if (rq_data->in_use) {
 902                 /* Allocate data queue resources */
 903                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 904                                    nb_data_desc,
 905                                    sizeof(struct rq_enet_desc));
 906                 if (rc) {
 907                         dev_err(enic, "error in allocation of data rq\n");
 908                         goto err_free_rq_sop;
 909                 }
 910                 nb_data_desc = rq_data->ring.desc_count;
 911         }
 912         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 913                            socket_id, nb_sop_desc + nb_data_desc,
 914                            sizeof(struct cq_enet_rq_desc));
 915         if (rc) {
 916                 dev_err(enic, "error in allocation of cq for rq\n");
 917                 goto err_free_rq_data;
 918         }
 919
 920         /* Allocate the mbuf rings */
 921         rq_sop->mbuf_ring = (struct rte_mbuf **)
 922                 rte_zmalloc_socket("rq->mbuf_ring",
 923                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 924                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 925         if (rq_sop->mbuf_ring == NULL)
 926                 goto err_free_cq;
 927
 928         if (rq_data->in_use) {
 929                 rq_data->mbuf_ring = (struct rte_mbuf **)
 930                         rte_zmalloc_socket("rq->mbuf_ring",
 931                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 932                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 933                 if (rq_data->mbuf_ring == NULL)
 934                         goto err_free_sop_mbuf;
 935         }
 936
 937         rq_sop->free_mbufs = (struct rte_mbuf **)
 938                 rte_zmalloc_socket("rq->free_mbufs",
 939                                    sizeof(struct rte_mbuf *) *
 940                                    ENIC_RX_BURST_MAX,
 941                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 942         if (rq_sop->free_mbufs == NULL)
 943                 goto err_free_data_mbuf;
 944         rq_sop->num_free_mbufs = 0;
 945
 946         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 947
 948         return 0;
 949
 950 err_free_data_mbuf:
 951         rte_free(rq_data->mbuf_ring);
 952 err_free_sop_mbuf:
 953         rte_free(rq_sop->mbuf_ring);
 954 err_free_cq:
 955         /* cleanup on error */
 956         vnic_cq_free(&enic->cq[queue_idx]);
 957 err_free_rq_data:
 958         if (rq_data->in_use)
 959                 vnic_rq_free(rq_data);
 960 err_free_rq_sop:
 961         vnic_rq_free(rq_sop);
 962 err_exit:
 963         return -ENOMEM;
 964 }
 965
 966 void enic_free_wq(void *txq)
 967 {
 968         struct vnic_wq *wq;
 969         struct enic *enic;
 970
 971         if (txq == NULL)
 972                 return;
 973
 974         wq = (struct vnic_wq *)txq;
 975         enic = vnic_dev_priv(wq->vdev);
 976         rte_memzone_free(wq->cqmsg_rz);
 977         vnic_wq_free(wq);
 978         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 979 }
 980
 981 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 982         unsigned int socket_id, uint16_t nb_desc)
 983 {
 984         int err;
 985         struct vnic_wq *wq = &enic->wq[queue_idx];
 986         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 987         char name[NAME_MAX];
 988         static int instance;
 989
 990         wq->socket_id = socket_id;
 991         /*
 992          * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
 993          * print an info message for diagnostics.
 994          */
 995         dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
 996
 997         /* Allocate queue resources */
 998         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
 999                 nb_desc,
1000                 sizeof(struct wq_enet_desc));
1001         if (err) {
1002                 dev_err(enic, "error in allocation of wq\n");
1003                 return err;
1004         }
1005
1006         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1007                 socket_id, nb_desc,
1008                 sizeof(struct cq_enet_wq_desc));
1009         if (err) {
1010                 vnic_wq_free(wq);
1011                 dev_err(enic, "error in allocation of cq for wq\n");
1012         }
1013
1014         /* setup up CQ message */
1015         snprintf((char *)name, sizeof(name),
1016                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1017                 instance++);
1018
1019         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1020                         sizeof(uint32_t), SOCKET_ID_ANY,
1021                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1022         if (!wq->cqmsg_rz)
1023                 return -ENOMEM;
1024
1025         return err;
1026 }
1027
1028 int enic_disable(struct enic *enic)
1029 {
1030         unsigned int i;
1031         int err;
1032
1033         for (i = 0; i < enic->intr_count; i++) {
1034                 vnic_intr_mask(&enic->intr[i]);
1035                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1036         }
1037         enic_rxq_intr_deinit(enic);
1038         rte_intr_disable(&enic->pdev->intr_handle);
1039         rte_intr_callback_unregister(&enic->pdev->intr_handle,
1040                                      enic_intr_handler,
1041                                      (void *)enic->rte_dev);
1042
1043         vnic_dev_disable(enic->vdev);
1044
1045         enic_clsf_destroy(enic);
1046
1047         if (!enic_is_sriov_vf(enic))
1048                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1049
1050         for (i = 0; i < enic->wq_count; i++) {
1051                 err = vnic_wq_disable(&enic->wq[i]);
1052                 if (err)
1053                         return err;
1054         }
1055         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1056                 if (enic->rq[i].in_use) {
1057                         err = vnic_rq_disable(&enic->rq[i]);
1058                         if (err)
1059                                 return err;
1060                 }
1061         }
1062
1063         /* If we were using interrupts, set the interrupt vector to -1
1064          * to disable interrupts.  We are not disabling link notifcations,
1065          * though, as we want the polling of link status to continue working.
1066          */
1067         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1068                 vnic_dev_notify_set(enic->vdev, -1);
1069
1070         vnic_dev_set_reset_flag(enic->vdev, 1);
1071
1072         for (i = 0; i < enic->wq_count; i++)
1073                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1074
1075         for (i = 0; i < enic_vnic_rq_count(enic); i++)
1076                 if (enic->rq[i].in_use)
1077                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1078         for (i = 0; i < enic->cq_count; i++)
1079                 vnic_cq_clean(&enic->cq[i]);
1080         for (i = 0; i < enic->intr_count; i++)
1081                 vnic_intr_clean(&enic->intr[i]);
1082
1083         return 0;
1084 }
1085
1086 static int enic_dev_wait(struct vnic_dev *vdev,
1087         int (*start)(struct vnic_dev *, int),
1088         int (*finished)(struct vnic_dev *, int *),
1089         int arg)
1090 {
1091         int done;
1092         int err;
1093         int i;
1094
1095         err = start(vdev, arg);
1096         if (err)
1097                 return err;
1098
1099         /* Wait for func to complete...2 seconds max */
1100         for (i = 0; i < 2000; i++) {
1101                 err = finished(vdev, &done);
1102                 if (err)
1103                         return err;
1104                 if (done)
1105                         return 0;
1106                 usleep(1000);
1107         }
1108         return -ETIMEDOUT;
1109 }
1110
1111 static int enic_dev_open(struct enic *enic)
1112 {
1113         int err;
1114         int flags = CMD_OPENF_IG_DESCCACHE;
1115
1116         err = enic_dev_wait(enic->vdev, vnic_dev_open,
1117                 vnic_dev_open_done, flags);
1118         if (err)
1119                 dev_err(enic_get_dev(enic),
1120                         "vNIC device open failed, err %d\n", err);
1121
1122         return err;
1123 }
1124
1125 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1126 {
1127         dma_addr_t rss_key_buf_pa;
1128         union vnic_rss_key *rss_key_buf_va = NULL;
1129         int err, i;
1130         u8 name[NAME_MAX];
1131
1132         RTE_ASSERT(user_key != NULL);
1133         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1134         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1135                 &rss_key_buf_pa, name);
1136         if (!rss_key_buf_va)
1137                 return -ENOMEM;
1138
1139         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1140                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1141
1142         err = enic_set_rss_key(enic,
1143                 rss_key_buf_pa,
1144                 sizeof(union vnic_rss_key));
1145
1146         /* Save for later queries */
1147         if (!err) {
1148                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1149                            sizeof(union vnic_rss_key));
1150         }
1151         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1152                 rss_key_buf_va, rss_key_buf_pa);
1153
1154         return err;
1155 }
1156
1157 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1158 {
1159         dma_addr_t rss_cpu_buf_pa;
1160         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1161         int err;
1162         u8 name[NAME_MAX];
1163
1164         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1165         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1166                 &rss_cpu_buf_pa, name);
1167         if (!rss_cpu_buf_va)
1168                 return -ENOMEM;
1169
1170         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1171
1172         err = enic_set_rss_cpu(enic,
1173                 rss_cpu_buf_pa,
1174                 sizeof(union vnic_rss_cpu));
1175
1176         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1177                 rss_cpu_buf_va, rss_cpu_buf_pa);
1178
1179         /* Save for later queries */
1180         if (!err)
1181                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1182         return err;
1183 }
1184
1185 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1186         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1187 {
1188         const u8 tso_ipid_split_en = 0;
1189         int err;
1190
1191         err = enic_set_nic_cfg(enic,
1192                 rss_default_cpu, rss_hash_type,
1193                 rss_hash_bits, rss_base_cpu,
1194                 rss_enable, tso_ipid_split_en,
1195                 enic->ig_vlan_strip_en);
1196
1197         return err;
1198 }
1199
1200 /* Initialize RSS with defaults, called from dev_configure */
1201 int enic_init_rss_nic_cfg(struct enic *enic)
1202 {
1203         static uint8_t default_rss_key[] = {
1204                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1205                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1206                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1207                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1208         };
1209         struct rte_eth_rss_conf rss_conf;
1210         union vnic_rss_cpu rss_cpu;
1211         int ret, i;
1212
1213         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1214         /*
1215          * If setting key for the first time, and the user gives us none, then
1216          * push the default key to NIC.
1217          */
1218         if (rss_conf.rss_key == NULL) {
1219                 rss_conf.rss_key = default_rss_key;
1220                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1221         }
1222         ret = enic_set_rss_conf(enic, &rss_conf);
1223         if (ret) {
1224                 dev_err(enic, "Failed to configure RSS\n");
1225                 return ret;
1226         }
1227         if (enic->rss_enable) {
1228                 /* If enabling RSS, use the default reta */
1229                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1230                         rss_cpu.cpu[i / 4].b[i % 4] =
1231                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1232                 }
1233                 ret = enic_set_rss_reta(enic, &rss_cpu);
1234                 if (ret)
1235                         dev_err(enic, "Failed to set RSS indirection table\n");
1236         }
1237         return ret;
1238 }
1239
1240 int enic_setup_finish(struct enic *enic)
1241 {
1242         enic_init_soft_stats(enic);
1243
1244         /* Default conf */
1245         vnic_dev_packet_filter(enic->vdev,
1246                 1 /* directed  */,
1247                 1 /* multicast */,
1248                 1 /* broadcast */,
1249                 0 /* promisc   */,
1250                 1 /* allmulti  */);
1251
1252         enic->promisc = 0;
1253         enic->allmulti = 1;
1254
1255         return 0;
1256 }
1257
1258 static int enic_rss_conf_valid(struct enic *enic,
1259                                struct rte_eth_rss_conf *rss_conf)
1260 {
1261         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1262         if (enic->flow_type_rss_offloads == 0)
1263                 return 0;
1264         if (rss_conf->rss_key != NULL &&
1265             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1266                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1267                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1268                 return -EINVAL;
1269         }
1270         if (rss_conf->rss_hf != 0 &&
1271             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1272                 dev_err(enic, "Given rss_hf contains none of the supported"
1273                         " types\n");
1274                 return -EINVAL;
1275         }
1276         return 0;
1277 }
1278
1279 /* Set hash type and key according to rss_conf */
1280 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1281 {
1282         struct rte_eth_dev *eth_dev;
1283         uint64_t rss_hf;
1284         u8 rss_hash_type;
1285         u8 rss_enable;
1286         int ret;
1287
1288         RTE_ASSERT(rss_conf != NULL);
1289         ret = enic_rss_conf_valid(enic, rss_conf);
1290         if (ret) {
1291                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1292                 return ret;
1293         }
1294
1295         eth_dev = enic->rte_dev;
1296         rss_hash_type = 0;
1297         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1298         if (enic->rq_count > 1 &&
1299             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1300             rss_hf != 0) {
1301                 rss_enable = 1;
1302                 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1303                               ETH_RSS_NONFRAG_IPV4_OTHER))
1304                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1305                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1306                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1307                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1308                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1309                         if (enic->udp_rss_weak) {
1310                                 /*
1311                                  * 'TCP' is not a typo. The "weak" version of
1312                                  * UDP RSS requires both the TCP and UDP bits
1313                                  * be set. It does enable TCP RSS as well.
1314                                  */
1315                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1316                         }
1317                 }
1318                 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1319                               ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1320                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1321                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1322                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1323                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1324                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1325                         if (enic->udp_rss_weak)
1326                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1327                 }
1328         } else {
1329                 rss_enable = 0;
1330                 rss_hf = 0;
1331         }
1332
1333         /* Set the hash key if provided */
1334         if (rss_enable && rss_conf->rss_key) {
1335                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1336                 if (ret) {
1337                         dev_err(enic, "Failed to set RSS key\n");
1338                         return ret;
1339                 }
1340         }
1341
1342         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1343                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1344                               rss_enable);
1345         if (!ret) {
1346                 enic->rss_hf = rss_hf;
1347                 enic->rss_hash_type = rss_hash_type;
1348                 enic->rss_enable = rss_enable;
1349         } else {
1350                 dev_err(enic, "Failed to update RSS configurations."
1351                         " hash=0x%x\n", rss_hash_type);
1352         }
1353         return ret;
1354 }
1355
1356 int enic_set_vlan_strip(struct enic *enic)
1357 {
1358         /*
1359          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1360          * together. So, re-do niccfg, preserving the current RSS settings.
1361          */
1362         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1363                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1364                                enic->rss_enable);
1365 }
1366
1367 void enic_add_packet_filter(struct enic *enic)
1368 {
1369         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1370         vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1371                 enic->promisc, enic->allmulti);
1372 }
1373
1374 int enic_get_link_status(struct enic *enic)
1375 {
1376         return vnic_dev_link_status(enic->vdev);
1377 }
1378
1379 static void enic_dev_deinit(struct enic *enic)
1380 {
1381         /* stop link status checking */
1382         vnic_dev_notify_unset(enic->vdev);
1383
1384         /* mac_addrs is freed by rte_eth_dev_release_port() */
1385         rte_free(enic->cq);
1386         rte_free(enic->intr);
1387         rte_free(enic->rq);
1388         rte_free(enic->wq);
1389 }
1390
1391
1392 int enic_set_vnic_res(struct enic *enic)
1393 {
1394         struct rte_eth_dev *eth_dev = enic->rte_dev;
1395         int rc = 0;
1396         unsigned int required_rq, required_wq, required_cq, required_intr;
1397
1398         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1399         required_rq = eth_dev->data->nb_rx_queues * 2;
1400         required_wq = eth_dev->data->nb_tx_queues;
1401         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1402         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1403         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1404                 required_intr += eth_dev->data->nb_rx_queues;
1405         }
1406
1407         if (enic->conf_rq_count < required_rq) {
1408                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1409                         eth_dev->data->nb_rx_queues,
1410                         required_rq, enic->conf_rq_count);
1411                 rc = -EINVAL;
1412         }
1413         if (enic->conf_wq_count < required_wq) {
1414                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1415                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1416                 rc = -EINVAL;
1417         }
1418
1419         if (enic->conf_cq_count < required_cq) {
1420                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1421                         required_cq, enic->conf_cq_count);
1422                 rc = -EINVAL;
1423         }
1424         if (enic->conf_intr_count < required_intr) {
1425                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1426                         " interrupts. Required:%u, Configured:%u\n",
1427                         required_intr, enic->conf_intr_count);
1428                 rc = -EINVAL;
1429         }
1430
1431         if (rc == 0) {
1432                 enic->rq_count = eth_dev->data->nb_rx_queues;
1433                 enic->wq_count = eth_dev->data->nb_tx_queues;
1434                 enic->cq_count = enic->rq_count + enic->wq_count;
1435                 enic->intr_count = required_intr;
1436         }
1437
1438         return rc;
1439 }
1440
1441 /* Initialize the completion queue for an RQ */
1442 static int
1443 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1444 {
1445         struct vnic_rq *sop_rq, *data_rq;
1446         unsigned int cq_idx;
1447         int rc = 0;
1448
1449         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1450         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1451         cq_idx = rq_idx;
1452
1453         vnic_cq_clean(&enic->cq[cq_idx]);
1454         vnic_cq_init(&enic->cq[cq_idx],
1455                      0 /* flow_control_enable */,
1456                      1 /* color_enable */,
1457                      0 /* cq_head */,
1458                      0 /* cq_tail */,
1459                      1 /* cq_tail_color */,
1460                      0 /* interrupt_enable */,
1461                      1 /* cq_entry_enable */,
1462                      0 /* cq_message_enable */,
1463                      0 /* interrupt offset */,
1464                      0 /* cq_message_addr */);
1465
1466
1467         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1468                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1469                            sop_rq->ring.desc_count - 1, 1, 0);
1470         if (data_rq->in_use) {
1471                 vnic_rq_init_start(data_rq,
1472                                    enic_cq_rq(enic,
1473                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1474                                    data_rq->ring.desc_count - 1, 1, 0);
1475         }
1476
1477         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1478         if (rc)
1479                 return rc;
1480
1481         if (data_rq->in_use) {
1482                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1483                 if (rc) {
1484                         enic_rxmbuf_queue_release(enic, sop_rq);
1485                         return rc;
1486                 }
1487         }
1488
1489         return 0;
1490 }
1491
1492 /* The Cisco NIC can send and receive packets up to a max packet size
1493  * determined by the NIC type and firmware. There is also an MTU
1494  * configured into the NIC via the CIMC/UCSM management interface
1495  * which can be overridden by this function (up to the max packet size).
1496  * Depending on the network setup, doing so may cause packet drops
1497  * and unexpected behavior.
1498  */
1499 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1500 {
1501         unsigned int rq_idx;
1502         struct vnic_rq *rq;
1503         int rc = 0;
1504         uint16_t old_mtu;       /* previous setting */
1505         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1506         struct rte_eth_dev *eth_dev = enic->rte_dev;
1507
1508         old_mtu = eth_dev->data->mtu;
1509         config_mtu = enic->config.mtu;
1510
1511         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1512                 return -E_RTE_SECONDARY;
1513
1514         if (new_mtu > enic->max_mtu) {
1515                 dev_err(enic,
1516                         "MTU not updated: requested (%u) greater than max (%u)\n",
1517                         new_mtu, enic->max_mtu);
1518                 return -EINVAL;
1519         }
1520         if (new_mtu < ENIC_MIN_MTU) {
1521                 dev_info(enic,
1522                         "MTU not updated: requested (%u) less than min (%u)\n",
1523                         new_mtu, ENIC_MIN_MTU);
1524                 return -EINVAL;
1525         }
1526         if (new_mtu > config_mtu)
1527                 dev_warning(enic,
1528                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1529                         new_mtu, config_mtu);
1530
1531         /* Update the MTU and maximum packet length */
1532         eth_dev->data->mtu = new_mtu;
1533         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1534                 enic_mtu_to_max_rx_pktlen(new_mtu);
1535
1536         /*
1537          * If the device has not started (enic_enable), nothing to do.
1538          * Later, enic_enable() will set up RQs reflecting the new maximum
1539          * packet length.
1540          */
1541         if (!eth_dev->data->dev_started)
1542                 goto set_mtu_done;
1543
1544         /*
1545          * The device has started, re-do RQs on the fly. In the process, we
1546          * pick up the new maximum packet length.
1547          *
1548          * Some applications rely on the ability to change MTU without stopping
1549          * the device. So keep this behavior for now.
1550          */
1551         rte_spinlock_lock(&enic->mtu_lock);
1552
1553         /* Stop traffic on all RQs */
1554         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1555                 rq = &enic->rq[rq_idx];
1556                 if (rq->is_sop && rq->in_use) {
1557                         rc = enic_stop_rq(enic,
1558                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1559                         if (rc) {
1560                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1561                                 goto set_mtu_done;
1562                         }
1563                 }
1564         }
1565
1566         /* replace Rx function with a no-op to avoid getting stale pkts */
1567         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1568         rte_mb();
1569
1570         /* Allow time for threads to exit the real Rx function. */
1571         usleep(100000);
1572
1573         /* now it is safe to reconfigure the RQs */
1574
1575
1576         /* free and reallocate RQs with the new MTU */
1577         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1578                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1579                 if (!rq->in_use)
1580                         continue;
1581
1582                 enic_free_rq(rq);
1583                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1584                                    rq->tot_nb_desc, rq->rx_free_thresh);
1585                 if (rc) {
1586                         dev_err(enic,
1587                                 "Fatal MTU alloc error- No traffic will pass\n");
1588                         goto set_mtu_done;
1589                 }
1590
1591                 rc = enic_reinit_rq(enic, rq_idx);
1592                 if (rc) {
1593                         dev_err(enic,
1594                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1595                         goto set_mtu_done;
1596                 }
1597         }
1598
1599         /* put back the real receive function */
1600         rte_mb();
1601         pick_rx_handler(enic);
1602         rte_mb();
1603
1604         /* restart Rx traffic */
1605         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1606                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1607                 if (rq->is_sop && rq->in_use)
1608                         enic_start_rq(enic, rq_idx);
1609         }
1610
1611 set_mtu_done:
1612         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1613         rte_spinlock_unlock(&enic->mtu_lock);
1614         return rc;
1615 }
1616
1617 static int enic_dev_init(struct enic *enic)
1618 {
1619         int err;
1620         struct rte_eth_dev *eth_dev = enic->rte_dev;
1621
1622         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1623
1624         /* Get vNIC configuration
1625         */
1626         err = enic_get_vnic_config(enic);
1627         if (err) {
1628                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1629                 return err;
1630         }
1631
1632         /* Get available resource counts */
1633         enic_get_res_counts(enic);
1634         if (enic->conf_rq_count == 1) {
1635                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1636                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1637                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1638                 return -EINVAL;
1639         }
1640         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1641         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1642                                enic->conf_cq_count, 8);
1643         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1644                                  enic->conf_intr_count, 8);
1645         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1646                                enic->conf_rq_count, 8);
1647         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1648                                enic->conf_wq_count, 8);
1649         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1650                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1651                 return -1;
1652         }
1653         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1654                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1655                 return -1;
1656         }
1657         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1658                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1659                 return -1;
1660         }
1661         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1662                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1663                 return -1;
1664         }
1665
1666         /* Get the supported filters */
1667         enic_fdir_info(enic);
1668
1669         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1670                                         sizeof(struct rte_ether_addr) *
1671                                         ENIC_UNICAST_PERFECT_FILTERS, 0);
1672         if (!eth_dev->data->mac_addrs) {
1673                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1674                 return -1;
1675         }
1676         ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1677                         eth_dev->data->mac_addrs);
1678
1679         vnic_dev_set_reset_flag(enic->vdev, 0);
1680
1681         LIST_INIT(&enic->flows);
1682         rte_spinlock_init(&enic->flows_lock);
1683         enic->max_flow_counter = -1;
1684
1685         /* set up link status checking */
1686         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1687
1688         enic->overlay_offload = false;
1689         if (enic->disable_overlay && enic->vxlan) {
1690                 /*
1691                  * Explicitly disable overlay offload as the setting is
1692                  * sticky, and resetting vNIC does not disable it.
1693                  */
1694                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1695                                                   OVERLAY_FEATURE_VXLAN,
1696                                                   OVERLAY_OFFLOAD_DISABLE)) {
1697                         dev_err(enic, "failed to disable overlay offload\n");
1698                 } else {
1699                         dev_info(enic, "Overlay offload is disabled\n");
1700                 }
1701         }
1702         if (!enic->disable_overlay && enic->vxlan &&
1703             /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1704             vnic_dev_overlay_offload_ctrl(enic->vdev,
1705                                           OVERLAY_FEATURE_VXLAN,
1706                                           OVERLAY_OFFLOAD_ENABLE) == 0) {
1707                 enic->tx_offload_capa |=
1708                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1709                         DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1710                         DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1711                 enic->tx_offload_mask |=
1712                         PKT_TX_OUTER_IPV6 |
1713                         PKT_TX_OUTER_IPV4 |
1714                         PKT_TX_OUTER_IP_CKSUM |
1715                         PKT_TX_TUNNEL_MASK;
1716                 enic->overlay_offload = true;
1717                 dev_info(enic, "Overlay offload is enabled\n");
1718         }
1719         /*
1720          * Reset the vxlan port if HW vxlan parsing is available. It
1721          * is always enabled regardless of overlay offload
1722          * enable/disable.
1723          */
1724         if (enic->vxlan) {
1725                 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1726                 /*
1727                  * Reset the vxlan port to the default, as the NIC firmware
1728                  * does not reset it automatically and keeps the old setting.
1729                  */
1730                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1731                                                  OVERLAY_CFG_VXLAN_PORT_UPDATE,
1732                                                  ENIC_DEFAULT_VXLAN_PORT)) {
1733                         dev_err(enic, "failed to update vxlan port\n");
1734                         return -EINVAL;
1735                 }
1736         }
1737
1738         return 0;
1739
1740 }
1741
1742 int enic_probe(struct enic *enic)
1743 {
1744         struct rte_pci_device *pdev = enic->pdev;
1745         int err = -1;
1746
1747         dev_debug(enic, " Initializing ENIC PMD\n");
1748
1749         /* if this is a secondary process the hardware is already initialized */
1750         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1751                 return 0;
1752
1753         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1754         enic->bar0.len = pdev->mem_resource[0].len;
1755
1756         /* Register vNIC device */
1757         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1758         if (!enic->vdev) {
1759                 dev_err(enic, "vNIC registration failed, aborting\n");
1760                 goto err_out;
1761         }
1762
1763         LIST_INIT(&enic->memzone_list);
1764         rte_spinlock_init(&enic->memzone_list_lock);
1765
1766         vnic_register_cbacks(enic->vdev,
1767                 enic_alloc_consistent,
1768                 enic_free_consistent);
1769
1770         /*
1771          * Allocate the consistent memory for stats and counters upfront so
1772          * both primary and secondary processes can access them.
1773          */
1774         err = vnic_dev_alloc_stats_mem(enic->vdev);
1775         if (err) {
1776                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1777                 goto err_out_unregister;
1778         }
1779         err = vnic_dev_alloc_counter_mem(enic->vdev);
1780         if (err) {
1781                 dev_err(enic, "Failed to allocate counter memory, aborting\n");
1782                 goto err_out_unregister;
1783         }
1784
1785         /* Issue device open to get device in known state */
1786         err = enic_dev_open(enic);
1787         if (err) {
1788                 dev_err(enic, "vNIC dev open failed, aborting\n");
1789                 goto err_out_unregister;
1790         }
1791
1792         /* Set ingress vlan rewrite mode before vnic initialization */
1793         dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1794                   enic->ig_vlan_rewrite_mode);
1795         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1796                 enic->ig_vlan_rewrite_mode);
1797         if (err) {
1798                 dev_err(enic,
1799                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1800                 goto err_out_dev_close;
1801         }
1802
1803         /* Issue device init to initialize the vnic-to-switch link.
1804          * We'll start with carrier off and wait for link UP
1805          * notification later to turn on carrier.  We don't need
1806          * to wait here for the vnic-to-switch link initialization
1807          * to complete; link UP notification is the indication that
1808          * the process is complete.
1809          */
1810
1811         err = vnic_dev_init(enic->vdev, 0);
1812         if (err) {
1813                 dev_err(enic, "vNIC dev init failed, aborting\n");
1814                 goto err_out_dev_close;
1815         }
1816
1817         err = enic_dev_init(enic);
1818         if (err) {
1819                 dev_err(enic, "Device initialization failed, aborting\n");
1820                 goto err_out_dev_close;
1821         }
1822
1823         return 0;
1824
1825 err_out_dev_close:
1826         vnic_dev_close(enic->vdev);
1827 err_out_unregister:
1828         vnic_dev_unregister(enic->vdev);
1829 err_out:
1830         return err;
1831 }
1832
1833 void enic_remove(struct enic *enic)
1834 {
1835         enic_dev_deinit(enic);
1836         vnic_dev_close(enic->vdev);
1837         vnic_dev_unregister(enic->vdev);
1838 }