drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11
  12 #include <rte_pci.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_ethdev_driver.h>
  19
  20 #include "enic_compat.h"
  21 #include "enic.h"
  22 #include "wq_enet_desc.h"
  23 #include "rq_enet_desc.h"
  24 #include "cq_enet_desc.h"
  25 #include "vnic_enet.h"
  26 #include "vnic_dev.h"
  27 #include "vnic_wq.h"
  28 #include "vnic_rq.h"
  29 #include "vnic_cq.h"
  30 #include "vnic_intr.h"
  31 #include "vnic_nic.h"
  32
  33 static inline int enic_is_sriov_vf(struct enic *enic)
  34 {
  35         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  36 }
  37
  38 static int is_zero_addr(uint8_t *addr)
  39 {
  40         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  41 }
  42
  43 static int is_mcast_addr(uint8_t *addr)
  44 {
  45         return addr[0] & 1;
  46 }
  47
  48 static int is_eth_addr_valid(uint8_t *addr)
  49 {
  50         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  51 }
  52
  53 static void
  54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  55 {
  56         uint16_t i;
  57
  58         if (!rq || !rq->mbuf_ring) {
  59                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  60                 return;
  61         }
  62
  63         for (i = 0; i < rq->ring.desc_count; i++) {
  64                 if (rq->mbuf_ring[i]) {
  65                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  66                         rq->mbuf_ring[i] = NULL;
  67                 }
  68         }
  69 }
  70
  71 static void enic_free_wq_buf(struct rte_mbuf **buf)
  72 {
  73         struct rte_mbuf *mbuf = *buf;
  74
  75         rte_pktmbuf_free_seg(mbuf);
  76         *buf = NULL;
  77 }
  78
  79 static void enic_log_q_error(struct enic *enic)
  80 {
  81         unsigned int i;
  82         u32 error_status;
  83
  84         for (i = 0; i < enic->wq_count; i++) {
  85                 error_status = vnic_wq_error_status(&enic->wq[i]);
  86                 if (error_status)
  87                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  88                                 error_status);
  89         }
  90
  91         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  92                 if (!enic->rq[i].in_use)
  93                         continue;
  94                 error_status = vnic_rq_error_status(&enic->rq[i]);
  95                 if (error_status)
  96                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  97                                 error_status);
  98         }
  99 }
 100
 101 static void enic_clear_soft_stats(struct enic *enic)
 102 {
 103         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 104         rte_atomic64_clear(&soft_stats->rx_nombuf);
 105         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 106         rte_atomic64_clear(&soft_stats->tx_oversized);
 107 }
 108
 109 static void enic_init_soft_stats(struct enic *enic)
 110 {
 111         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 112         rte_atomic64_init(&soft_stats->rx_nombuf);
 113         rte_atomic64_init(&soft_stats->rx_packet_errors);
 114         rte_atomic64_init(&soft_stats->tx_oversized);
 115         enic_clear_soft_stats(enic);
 116 }
 117
 118 int enic_dev_stats_clear(struct enic *enic)
 119 {
 120         int ret;
 121
 122         ret = vnic_dev_stats_clear(enic->vdev);
 123         if (ret != 0) {
 124                 dev_err(enic, "Error in clearing stats\n");
 125                 return ret;
 126         }
 127         enic_clear_soft_stats(enic);
 128
 129         return 0;
 130 }
 131
 132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 133 {
 134         struct vnic_stats *stats;
 135         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 136         int64_t rx_truncated;
 137         uint64_t rx_packet_errors;
 138         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 139
 140         if (ret) {
 141                 dev_err(enic, "Error in getting stats\n");
 142                 return ret;
 143         }
 144
 145         /* The number of truncated packets can only be calculated by
 146          * subtracting a hardware counter from error packets received by
 147          * the driver. Note: this causes transient inaccuracies in the
 148          * ipackets count. Also, the length of truncated packets are
 149          * counted in ibytes even though truncated packets are dropped
 150          * which can make ibytes be slightly higher than it should be.
 151          */
 152         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 153         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 154
 155         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 156         r_stats->opackets = stats->tx.tx_frames_ok;
 157
 158         r_stats->ibytes = stats->rx.rx_bytes_ok;
 159         r_stats->obytes = stats->tx.tx_bytes_ok;
 160
 161         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 162         r_stats->oerrors = stats->tx.tx_errors
 163                            + rte_atomic64_read(&soft_stats->tx_oversized);
 164
 165         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 166
 167         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 168         return 0;
 169 }
 170
 171 int enic_del_mac_address(struct enic *enic, int mac_index)
 172 {
 173         struct rte_eth_dev *eth_dev = enic->rte_dev;
 174         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 175
 176         return vnic_dev_del_addr(enic->vdev, mac_addr);
 177 }
 178
 179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 180 {
 181         int err;
 182
 183         if (!is_eth_addr_valid(mac_addr)) {
 184                 dev_err(enic, "invalid mac address\n");
 185                 return -EINVAL;
 186         }
 187
 188         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 189         if (err)
 190                 dev_err(enic, "add mac addr failed\n");
 191         return err;
 192 }
 193
 194 static void
 195 enic_free_rq_buf(struct rte_mbuf **mbuf)
 196 {
 197         if (*mbuf == NULL)
 198                 return;
 199
 200         rte_pktmbuf_free(*mbuf);
 201         *mbuf = NULL;
 202 }
 203
 204 void enic_init_vnic_resources(struct enic *enic)
 205 {
 206         unsigned int error_interrupt_enable = 1;
 207         unsigned int error_interrupt_offset = 0;
 208         unsigned int rxq_interrupt_enable = 0;
 209         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 210         unsigned int index = 0;
 211         unsigned int cq_idx;
 212         struct vnic_rq *data_rq;
 213
 214         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 215                 rxq_interrupt_enable = 1;
 216
 217         for (index = 0; index < enic->rq_count; index++) {
 218                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 219
 220                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 221                         cq_idx,
 222                         error_interrupt_enable,
 223                         error_interrupt_offset);
 224
 225                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 226                 if (data_rq->in_use)
 227                         vnic_rq_init(data_rq,
 228                                      cq_idx,
 229                                      error_interrupt_enable,
 230                                      error_interrupt_offset);
 231
 232                 vnic_cq_init(&enic->cq[cq_idx],
 233                         0 /* flow_control_enable */,
 234                         1 /* color_enable */,
 235                         0 /* cq_head */,
 236                         0 /* cq_tail */,
 237                         1 /* cq_tail_color */,
 238                         rxq_interrupt_enable,
 239                         1 /* cq_entry_enable */,
 240                         0 /* cq_message_enable */,
 241                         rxq_interrupt_offset,
 242                         0 /* cq_message_addr */);
 243                 if (rxq_interrupt_enable)
 244                         rxq_interrupt_offset++;
 245         }
 246
 247         for (index = 0; index < enic->wq_count; index++) {
 248                 vnic_wq_init(&enic->wq[index],
 249                         enic_cq_wq(enic, index),
 250                         error_interrupt_enable,
 251                         error_interrupt_offset);
 252                 /* Compute unsupported ol flags for enic_prep_pkts() */
 253                 enic->wq[index].tx_offload_notsup_mask =
 254                         PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 255
 256                 cq_idx = enic_cq_wq(enic, index);
 257                 vnic_cq_init(&enic->cq[cq_idx],
 258                         0 /* flow_control_enable */,
 259                         1 /* color_enable */,
 260                         0 /* cq_head */,
 261                         0 /* cq_tail */,
 262                         1 /* cq_tail_color */,
 263                         0 /* interrupt_enable */,
 264                         0 /* cq_entry_enable */,
 265                         1 /* cq_message_enable */,
 266                         0 /* interrupt offset */,
 267                         (u64)enic->wq[index].cqmsg_rz->iova);
 268         }
 269
 270         for (index = 0; index < enic->intr_count; index++) {
 271                 vnic_intr_init(&enic->intr[index],
 272                                enic->config.intr_timer_usec,
 273                                enic->config.intr_timer_type,
 274                                /*mask_on_assertion*/1);
 275         }
 276 }
 277
 278
 279 static int
 280 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 281 {
 282         struct rte_mbuf *mb;
 283         struct rq_enet_desc *rqd = rq->ring.descs;
 284         unsigned i;
 285         dma_addr_t dma_addr;
 286         uint32_t max_rx_pkt_len;
 287         uint16_t rq_buf_len;
 288
 289         if (!rq->in_use)
 290                 return 0;
 291
 292         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 293                   rq->ring.desc_count);
 294
 295         /*
 296          * If *not* using scatter and the mbuf size is greater than the
 297          * requested max packet size (max_rx_pkt_len), then reduce the
 298          * posted buffer size to max_rx_pkt_len. HW still receives packets
 299          * larger than max_rx_pkt_len, but they will be truncated, which we
 300          * drop in the rx handler. Not ideal, but better than returning
 301          * large packets when the user is not expecting them.
 302          */
 303         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 304         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 305         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 306                 rq_buf_len = max_rx_pkt_len;
 307         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 308                 mb = rte_mbuf_raw_alloc(rq->mp);
 309                 if (mb == NULL) {
 310                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 311                         (unsigned)rq->index);
 312                         return -ENOMEM;
 313                 }
 314
 315                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 316                 dma_addr = (dma_addr_t)(mb->buf_iova
 317                            + RTE_PKTMBUF_HEADROOM);
 318                 rq_enet_desc_enc(rqd, dma_addr,
 319                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 320                                 : RQ_ENET_TYPE_NOT_SOP),
 321                                 rq_buf_len);
 322                 rq->mbuf_ring[i] = mb;
 323         }
 324         /*
 325          * Do not post the buffers to the NIC until we enable the RQ via
 326          * enic_start_rq().
 327          */
 328         rq->need_initial_post = true;
 329         /* Initialize fetch index while RQ is disabled */
 330         iowrite32(0, &rq->ctrl->fetch_index);
 331         return 0;
 332 }
 333
 334 /*
 335  * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
 336  * allocated the buffers and filled the RQ descriptor ring. Just need to push
 337  * the post index to the NIC.
 338  */
 339 static void
 340 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 341 {
 342         if (!rq->in_use || !rq->need_initial_post)
 343                 return;
 344
 345         /* make sure all prior writes are complete before doing the PIO write */
 346         rte_rmb();
 347
 348         /* Post all but the last buffer to VIC. */
 349         rq->posted_index = rq->ring.desc_count - 1;
 350
 351         rq->rx_nb_hold = 0;
 352
 353         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 354                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 355         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 356         rte_rmb();
 357         rq->need_initial_post = false;
 358 }
 359
 360 static void *
 361 enic_alloc_consistent(void *priv, size_t size,
 362         dma_addr_t *dma_handle, u8 *name)
 363 {
 364         void *vaddr;
 365         const struct rte_memzone *rz;
 366         *dma_handle = 0;
 367         struct enic *enic = (struct enic *)priv;
 368         struct enic_memzone_entry *mze;
 369
 370         rz = rte_memzone_reserve_aligned((const char *)name, size,
 371                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 372         if (!rz) {
 373                 pr_err("%s : Failed to allocate memory requested for %s\n",
 374                         __func__, name);
 375                 return NULL;
 376         }
 377
 378         vaddr = rz->addr;
 379         *dma_handle = (dma_addr_t)rz->iova;
 380
 381         mze = rte_malloc("enic memzone entry",
 382                          sizeof(struct enic_memzone_entry), 0);
 383
 384         if (!mze) {
 385                 pr_err("%s : Failed to allocate memory for memzone list\n",
 386                        __func__);
 387                 rte_memzone_free(rz);
 388                 return NULL;
 389         }
 390
 391         mze->rz = rz;
 392
 393         rte_spinlock_lock(&enic->memzone_list_lock);
 394         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 395         rte_spinlock_unlock(&enic->memzone_list_lock);
 396
 397         return vaddr;
 398 }
 399
 400 static void
 401 enic_free_consistent(void *priv,
 402                      __rte_unused size_t size,
 403                      void *vaddr,
 404                      dma_addr_t dma_handle)
 405 {
 406         struct enic_memzone_entry *mze;
 407         struct enic *enic = (struct enic *)priv;
 408
 409         rte_spinlock_lock(&enic->memzone_list_lock);
 410         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 411                 if (mze->rz->addr == vaddr &&
 412                     mze->rz->iova == dma_handle)
 413                         break;
 414         }
 415         if (mze == NULL) {
 416                 rte_spinlock_unlock(&enic->memzone_list_lock);
 417                 dev_warning(enic,
 418                             "Tried to free memory, but couldn't find it in the memzone list\n");
 419                 return;
 420         }
 421         LIST_REMOVE(mze, entries);
 422         rte_spinlock_unlock(&enic->memzone_list_lock);
 423         rte_memzone_free(mze->rz);
 424         rte_free(mze);
 425 }
 426
 427 int enic_link_update(struct rte_eth_dev *eth_dev)
 428 {
 429         struct enic *enic = pmd_priv(eth_dev);
 430         struct rte_eth_link link;
 431
 432         memset(&link, 0, sizeof(link));
 433         link.link_status = enic_get_link_status(enic);
 434         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 435         link.link_speed = vnic_dev_port_speed(enic->vdev);
 436
 437         return rte_eth_linkstatus_set(eth_dev, &link);
 438 }
 439
 440 static void
 441 enic_intr_handler(void *arg)
 442 {
 443         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 444         struct enic *enic = pmd_priv(dev);
 445
 446         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 447
 448         enic_link_update(dev);
 449         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 450         enic_log_q_error(enic);
 451 }
 452
 453 static int enic_rxq_intr_init(struct enic *enic)
 454 {
 455         struct rte_intr_handle *intr_handle;
 456         uint32_t rxq_intr_count, i;
 457         int err;
 458
 459         intr_handle = enic->rte_dev->intr_handle;
 460         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 461                 return 0;
 462         /*
 463          * Rx queue interrupts only work when we have MSI-X interrupts,
 464          * one per queue. Sharing one interrupt is technically
 465          * possible with VIC, but it is not worth the complications it brings.
 466          */
 467         if (!rte_intr_cap_multiple(intr_handle)) {
 468                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 469                         " (vfio-pci driver)\n");
 470                 return -ENOTSUP;
 471         }
 472         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 473         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 474         if (err) {
 475                 dev_err(enic, "Failed to enable event fds for Rx queue"
 476                         " interrupts\n");
 477                 return err;
 478         }
 479         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 480                                             rxq_intr_count * sizeof(int), 0);
 481         if (intr_handle->intr_vec == NULL) {
 482                 dev_err(enic, "Failed to allocate intr_vec\n");
 483                 return -ENOMEM;
 484         }
 485         for (i = 0; i < rxq_intr_count; i++)
 486                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 487         return 0;
 488 }
 489
 490 static void enic_rxq_intr_deinit(struct enic *enic)
 491 {
 492         struct rte_intr_handle *intr_handle;
 493
 494         intr_handle = enic->rte_dev->intr_handle;
 495         rte_intr_efd_disable(intr_handle);
 496         if (intr_handle->intr_vec != NULL) {
 497                 rte_free(intr_handle->intr_vec);
 498                 intr_handle->intr_vec = NULL;
 499         }
 500 }
 501
 502 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 503 {
 504         struct wq_enet_desc *desc;
 505         struct vnic_wq *wq;
 506         unsigned int i;
 507
 508         /*
 509          * Fill WQ descriptor fields that never change. Every descriptor is
 510          * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
 511          * descriptors (i.e. request one completion update every 32 packets).
 512          */
 513         wq = &enic->wq[queue_idx];
 514         desc = (struct wq_enet_desc *)wq->ring.descs;
 515         for (i = 0; i < wq->ring.desc_count; i++, desc++) {
 516                 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
 517                 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
 518                         desc->header_length_flags |=
 519                                 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
 520         }
 521 }
 522
 523 /*
 524  * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
 525  * used when that file is not compiled.
 526  */
 527 __rte_weak bool
 528 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
 529 {
 530         return false;
 531 }
 532
 533 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
 534 {
 535         struct enic *enic = pmd_priv(eth_dev);
 536
 537         /*
 538          * Preference order:
 539          * 1. The vectorized handler if possible and requested.
 540          * 2. The non-scatter, simplified handler if scatter Rx is not used.
 541          * 3. The default handler as a fallback.
 542          */
 543         if (enic_use_vector_rx_handler(eth_dev))
 544                 return;
 545         if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 546                 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
 547                 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
 548         } else {
 549                 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
 550                 eth_dev->rx_pkt_burst = &enic_recv_pkts;
 551         }
 552 }
 553
 554 /* Secondary process uses this to set the Tx handler */
 555 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
 556 {
 557         struct enic *enic = pmd_priv(eth_dev);
 558
 559         if (enic->use_simple_tx_handler) {
 560                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 561                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 562         } else {
 563                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 564                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 565         }
 566 }
 567
 568 int enic_enable(struct enic *enic)
 569 {
 570         unsigned int index;
 571         int err;
 572         struct rte_eth_dev *eth_dev = enic->rte_dev;
 573         uint64_t simple_tx_offloads;
 574         uintptr_t p;
 575
 576         if (enic->enable_avx2_rx) {
 577                 struct rte_mbuf mb_def = { .buf_addr = 0 };
 578
 579                 /*
 580                  * mbuf_initializer contains const-after-init fields of
 581                  * receive mbufs (i.e. 64 bits of fields from rearm_data).
 582                  * It is currently used by the vectorized handler.
 583                  */
 584                 mb_def.nb_segs = 1;
 585                 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 586                 mb_def.port = enic->port_id;
 587                 rte_mbuf_refcnt_set(&mb_def, 1);
 588                 rte_compiler_barrier();
 589                 p = (uintptr_t)&mb_def.rearm_data;
 590                 enic->mbuf_initializer = *(uint64_t *)p;
 591         }
 592
 593         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 594         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 595
 596         /* vnic notification of link status has already been turned on in
 597          * enic_dev_init() which is called during probe time.  Here we are
 598          * just turning on interrupt vector 0 if needed.
 599          */
 600         if (eth_dev->data->dev_conf.intr_conf.lsc)
 601                 vnic_dev_notify_set(enic->vdev, 0);
 602
 603         err = enic_rxq_intr_init(enic);
 604         if (err)
 605                 return err;
 606         if (enic_clsf_init(enic))
 607                 dev_warning(enic, "Init of hash table for clsf failed."\
 608                         "Flow director feature will not work\n");
 609
 610         for (index = 0; index < enic->rq_count; index++) {
 611                 err = enic_alloc_rx_queue_mbufs(enic,
 612                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 613                 if (err) {
 614                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 615                         return err;
 616                 }
 617                 err = enic_alloc_rx_queue_mbufs(enic,
 618                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 619                 if (err) {
 620                         /* release the allocated mbufs for the sop rq*/
 621                         enic_rxmbuf_queue_release(enic,
 622                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 623
 624                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 625                         return err;
 626                 }
 627         }
 628
 629         /*
 630          * Use the simple TX handler if possible. Only checksum offloads
 631          * and vlan insertion are supported.
 632          */
 633         simple_tx_offloads = enic->tx_offload_capa &
 634                 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 635                  DEV_TX_OFFLOAD_VLAN_INSERT |
 636                  DEV_TX_OFFLOAD_IPV4_CKSUM |
 637                  DEV_TX_OFFLOAD_UDP_CKSUM |
 638                  DEV_TX_OFFLOAD_TCP_CKSUM);
 639         if ((eth_dev->data->dev_conf.txmode.offloads &
 640              ~simple_tx_offloads) == 0) {
 641                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 642                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 643                 for (index = 0; index < enic->wq_count; index++)
 644                         enic_prep_wq_for_simple_tx(enic, index);
 645                 enic->use_simple_tx_handler = 1;
 646         } else {
 647                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 648                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 649         }
 650
 651         enic_pick_rx_handler(eth_dev);
 652
 653         for (index = 0; index < enic->wq_count; index++)
 654                 enic_start_wq(enic, index);
 655         for (index = 0; index < enic->rq_count; index++)
 656                 enic_start_rq(enic, index);
 657
 658         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 659
 660         vnic_dev_enable_wait(enic->vdev);
 661
 662         /* Register and enable error interrupt */
 663         rte_intr_callback_register(&(enic->pdev->intr_handle),
 664                 enic_intr_handler, (void *)enic->rte_dev);
 665
 666         rte_intr_enable(&(enic->pdev->intr_handle));
 667         /* Unmask LSC interrupt */
 668         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 669
 670         return 0;
 671 }
 672
 673 int enic_alloc_intr_resources(struct enic *enic)
 674 {
 675         int err;
 676         unsigned int i;
 677
 678         dev_info(enic, "vNIC resources used:  "\
 679                 "wq %d rq %d cq %d intr %d\n",
 680                 enic->wq_count, enic_vnic_rq_count(enic),
 681                 enic->cq_count, enic->intr_count);
 682
 683         for (i = 0; i < enic->intr_count; i++) {
 684                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 685                 if (err) {
 686                         enic_free_vnic_resources(enic);
 687                         return err;
 688                 }
 689         }
 690         return 0;
 691 }
 692
 693 void enic_free_rq(void *rxq)
 694 {
 695         struct vnic_rq *rq_sop, *rq_data;
 696         struct enic *enic;
 697
 698         if (rxq == NULL)
 699                 return;
 700
 701         rq_sop = (struct vnic_rq *)rxq;
 702         enic = vnic_dev_priv(rq_sop->vdev);
 703         rq_data = &enic->rq[rq_sop->data_queue_idx];
 704
 705         if (rq_sop->free_mbufs) {
 706                 struct rte_mbuf **mb;
 707                 int i;
 708
 709                 mb = rq_sop->free_mbufs;
 710                 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
 711                      i < ENIC_RX_BURST_MAX; i++)
 712                         rte_pktmbuf_free(mb[i]);
 713                 rte_free(rq_sop->free_mbufs);
 714                 rq_sop->free_mbufs = NULL;
 715                 rq_sop->num_free_mbufs = 0;
 716         }
 717
 718         enic_rxmbuf_queue_release(enic, rq_sop);
 719         if (rq_data->in_use)
 720                 enic_rxmbuf_queue_release(enic, rq_data);
 721
 722         rte_free(rq_sop->mbuf_ring);
 723         if (rq_data->in_use)
 724                 rte_free(rq_data->mbuf_ring);
 725
 726         rq_sop->mbuf_ring = NULL;
 727         rq_data->mbuf_ring = NULL;
 728
 729         vnic_rq_free(rq_sop);
 730         if (rq_data->in_use)
 731                 vnic_rq_free(rq_data);
 732
 733         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 734
 735         rq_sop->in_use = 0;
 736         rq_data->in_use = 0;
 737 }
 738
 739 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 740 {
 741         struct rte_eth_dev_data *data = enic->dev_data;
 742         vnic_wq_enable(&enic->wq[queue_idx]);
 743         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 744 }
 745
 746 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 747 {
 748         struct rte_eth_dev_data *data = enic->dev_data;
 749         int ret;
 750
 751         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 752         if (ret)
 753                 return ret;
 754
 755         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 756         return 0;
 757 }
 758
 759 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 760 {
 761         struct rte_eth_dev_data *data = enic->dev_data;
 762         struct vnic_rq *rq_sop;
 763         struct vnic_rq *rq_data;
 764         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 765         rq_data = &enic->rq[rq_sop->data_queue_idx];
 766
 767         if (rq_data->in_use) {
 768                 vnic_rq_enable(rq_data);
 769                 enic_initial_post_rx(enic, rq_data);
 770         }
 771         rte_mb();
 772         vnic_rq_enable(rq_sop);
 773         enic_initial_post_rx(enic, rq_sop);
 774         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 775 }
 776
 777 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 778 {
 779         struct rte_eth_dev_data *data = enic->dev_data;
 780         int ret1 = 0, ret2 = 0;
 781         struct vnic_rq *rq_sop;
 782         struct vnic_rq *rq_data;
 783         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 784         rq_data = &enic->rq[rq_sop->data_queue_idx];
 785
 786         ret2 = vnic_rq_disable(rq_sop);
 787         rte_mb();
 788         if (rq_data->in_use)
 789                 ret1 = vnic_rq_disable(rq_data);
 790
 791         if (ret2)
 792                 return ret2;
 793         else if (ret1)
 794                 return ret1;
 795
 796         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 797         return 0;
 798 }
 799
 800 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 801         unsigned int socket_id, struct rte_mempool *mp,
 802         uint16_t nb_desc, uint16_t free_thresh)
 803 {
 804         int rc;
 805         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 806         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 807         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 808         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 809         unsigned int mbuf_size, mbufs_per_pkt;
 810         unsigned int nb_sop_desc, nb_data_desc;
 811         uint16_t min_sop, max_sop, min_data, max_data;
 812         uint32_t max_rx_pkt_len;
 813
 814         rq_sop->is_sop = 1;
 815         rq_sop->data_queue_idx = data_queue_idx;
 816         rq_data->is_sop = 0;
 817         rq_data->data_queue_idx = 0;
 818         rq_sop->socket_id = socket_id;
 819         rq_sop->mp = mp;
 820         rq_data->socket_id = socket_id;
 821         rq_data->mp = mp;
 822         rq_sop->in_use = 1;
 823         rq_sop->rx_free_thresh = free_thresh;
 824         rq_data->rx_free_thresh = free_thresh;
 825         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 826                   free_thresh);
 827
 828         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 829                                RTE_PKTMBUF_HEADROOM);
 830         /* max_rx_pkt_len includes the ethernet header and CRC. */
 831         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 832
 833         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 834             DEV_RX_OFFLOAD_SCATTER) {
 835                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 836                 /* ceil((max pkt len)/mbuf_size) */
 837                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 838         } else {
 839                 dev_info(enic, "Scatter rx mode disabled\n");
 840                 mbufs_per_pkt = 1;
 841                 if (max_rx_pkt_len > mbuf_size) {
 842                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 843                                     " larger than the mbuf size (%u), and"
 844                                     " scatter is disabled. Larger packets will"
 845                                     " be truncated.\n",
 846                                     max_rx_pkt_len, mbuf_size);
 847                 }
 848         }
 849
 850         if (mbufs_per_pkt > 1) {
 851                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 852                 rq_sop->data_queue_enable = 1;
 853                 rq_data->in_use = 1;
 854                 /*
 855                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 856                  * receives packet sizes up to the "max" MTU.
 857                  * If not using scatter, we can achieve the effect of dropping
 858                  * larger packets by reducing the size of posted buffers.
 859                  * See enic_alloc_rx_queue_mbufs().
 860                  */
 861                 if (max_rx_pkt_len <
 862                     enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
 863                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 864                                     " when scatter rx mode is in use.\n");
 865                 }
 866         } else {
 867                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 868                          queue_idx);
 869                 rq_sop->data_queue_enable = 0;
 870                 rq_data->in_use = 0;
 871         }
 872
 873         /* number of descriptors have to be a multiple of 32 */
 874         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
 875         nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 876
 877         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 878         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 879
 880         if (mbufs_per_pkt > 1) {
 881                 min_sop = ENIC_RX_BURST_MAX;
 882                 max_sop = ((enic->config.rq_desc_count /
 883                             (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 884                 min_data = min_sop * (mbufs_per_pkt - 1);
 885                 max_data = enic->config.rq_desc_count;
 886         } else {
 887                 min_sop = ENIC_RX_BURST_MAX;
 888                 max_sop = enic->config.rq_desc_count;
 889                 min_data = 0;
 890                 max_data = 0;
 891         }
 892
 893         if (nb_desc < (min_sop + min_data)) {
 894                 dev_warning(enic,
 895                             "Number of rx descs too low, adjusting to minimum\n");
 896                 nb_sop_desc = min_sop;
 897                 nb_data_desc = min_data;
 898         } else if (nb_desc > (max_sop + max_data)) {
 899                 dev_warning(enic,
 900                             "Number of rx_descs too high, adjusting to maximum\n");
 901                 nb_sop_desc = max_sop;
 902                 nb_data_desc = max_data;
 903         }
 904         if (mbufs_per_pkt > 1) {
 905                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 906                          " rx descriptor range is %u to %u\n",
 907                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 908                          max_sop + max_data);
 909         }
 910         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 911                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 912
 913         /* Allocate sop queue resources */
 914         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 915                 nb_sop_desc, sizeof(struct rq_enet_desc));
 916         if (rc) {
 917                 dev_err(enic, "error in allocation of sop rq\n");
 918                 goto err_exit;
 919         }
 920         nb_sop_desc = rq_sop->ring.desc_count;
 921
 922         if (rq_data->in_use) {
 923                 /* Allocate data queue resources */
 924                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 925                                    nb_data_desc,
 926                                    sizeof(struct rq_enet_desc));
 927                 if (rc) {
 928                         dev_err(enic, "error in allocation of data rq\n");
 929                         goto err_free_rq_sop;
 930                 }
 931                 nb_data_desc = rq_data->ring.desc_count;
 932         }
 933         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 934                            socket_id, nb_sop_desc + nb_data_desc,
 935                            sizeof(struct cq_enet_rq_desc));
 936         if (rc) {
 937                 dev_err(enic, "error in allocation of cq for rq\n");
 938                 goto err_free_rq_data;
 939         }
 940
 941         /* Allocate the mbuf rings */
 942         rq_sop->mbuf_ring = (struct rte_mbuf **)
 943                 rte_zmalloc_socket("rq->mbuf_ring",
 944                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 945                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 946         if (rq_sop->mbuf_ring == NULL)
 947                 goto err_free_cq;
 948
 949         if (rq_data->in_use) {
 950                 rq_data->mbuf_ring = (struct rte_mbuf **)
 951                         rte_zmalloc_socket("rq->mbuf_ring",
 952                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 953                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 954                 if (rq_data->mbuf_ring == NULL)
 955                         goto err_free_sop_mbuf;
 956         }
 957
 958         rq_sop->free_mbufs = (struct rte_mbuf **)
 959                 rte_zmalloc_socket("rq->free_mbufs",
 960                                    sizeof(struct rte_mbuf *) *
 961                                    ENIC_RX_BURST_MAX,
 962                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 963         if (rq_sop->free_mbufs == NULL)
 964                 goto err_free_data_mbuf;
 965         rq_sop->num_free_mbufs = 0;
 966
 967         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 968
 969         return 0;
 970
 971 err_free_data_mbuf:
 972         rte_free(rq_data->mbuf_ring);
 973 err_free_sop_mbuf:
 974         rte_free(rq_sop->mbuf_ring);
 975 err_free_cq:
 976         /* cleanup on error */
 977         vnic_cq_free(&enic->cq[queue_idx]);
 978 err_free_rq_data:
 979         if (rq_data->in_use)
 980                 vnic_rq_free(rq_data);
 981 err_free_rq_sop:
 982         vnic_rq_free(rq_sop);
 983 err_exit:
 984         return -ENOMEM;
 985 }
 986
 987 void enic_free_wq(void *txq)
 988 {
 989         struct vnic_wq *wq;
 990         struct enic *enic;
 991
 992         if (txq == NULL)
 993                 return;
 994
 995         wq = (struct vnic_wq *)txq;
 996         enic = vnic_dev_priv(wq->vdev);
 997         rte_memzone_free(wq->cqmsg_rz);
 998         vnic_wq_free(wq);
 999         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1000 }
1001
1002 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1003         unsigned int socket_id, uint16_t nb_desc)
1004 {
1005         int err;
1006         struct vnic_wq *wq = &enic->wq[queue_idx];
1007         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
1008         char name[NAME_MAX];
1009         static int instance;
1010
1011         wq->socket_id = socket_id;
1012         /*
1013          * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1014          * print an info message for diagnostics.
1015          */
1016         dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1017
1018         /* Allocate queue resources */
1019         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1020                 nb_desc,
1021                 sizeof(struct wq_enet_desc));
1022         if (err) {
1023                 dev_err(enic, "error in allocation of wq\n");
1024                 return err;
1025         }
1026
1027         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1028                 socket_id, nb_desc,
1029                 sizeof(struct cq_enet_wq_desc));
1030         if (err) {
1031                 vnic_wq_free(wq);
1032                 dev_err(enic, "error in allocation of cq for wq\n");
1033         }
1034
1035         /* setup up CQ message */
1036         snprintf((char *)name, sizeof(name),
1037                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1038                 instance++);
1039
1040         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1041                         sizeof(uint32_t), SOCKET_ID_ANY,
1042                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1043         if (!wq->cqmsg_rz)
1044                 return -ENOMEM;
1045
1046         return err;
1047 }
1048
1049 int enic_disable(struct enic *enic)
1050 {
1051         unsigned int i;
1052         int err;
1053
1054         for (i = 0; i < enic->intr_count; i++) {
1055                 vnic_intr_mask(&enic->intr[i]);
1056                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1057         }
1058         enic_rxq_intr_deinit(enic);
1059         rte_intr_disable(&enic->pdev->intr_handle);
1060         rte_intr_callback_unregister(&enic->pdev->intr_handle,
1061                                      enic_intr_handler,
1062                                      (void *)enic->rte_dev);
1063
1064         vnic_dev_disable(enic->vdev);
1065
1066         enic_clsf_destroy(enic);
1067
1068         if (!enic_is_sriov_vf(enic))
1069                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1070
1071         for (i = 0; i < enic->wq_count; i++) {
1072                 err = vnic_wq_disable(&enic->wq[i]);
1073                 if (err)
1074                         return err;
1075         }
1076         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1077                 if (enic->rq[i].in_use) {
1078                         err = vnic_rq_disable(&enic->rq[i]);
1079                         if (err)
1080                                 return err;
1081                 }
1082         }
1083
1084         /* If we were using interrupts, set the interrupt vector to -1
1085          * to disable interrupts.  We are not disabling link notifcations,
1086          * though, as we want the polling of link status to continue working.
1087          */
1088         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1089                 vnic_dev_notify_set(enic->vdev, -1);
1090
1091         vnic_dev_set_reset_flag(enic->vdev, 1);
1092
1093         for (i = 0; i < enic->wq_count; i++)
1094                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1095
1096         for (i = 0; i < enic_vnic_rq_count(enic); i++)
1097                 if (enic->rq[i].in_use)
1098                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1099         for (i = 0; i < enic->cq_count; i++)
1100                 vnic_cq_clean(&enic->cq[i]);
1101         for (i = 0; i < enic->intr_count; i++)
1102                 vnic_intr_clean(&enic->intr[i]);
1103
1104         return 0;
1105 }
1106
1107 static int enic_dev_wait(struct vnic_dev *vdev,
1108         int (*start)(struct vnic_dev *, int),
1109         int (*finished)(struct vnic_dev *, int *),
1110         int arg)
1111 {
1112         int done;
1113         int err;
1114         int i;
1115
1116         err = start(vdev, arg);
1117         if (err)
1118                 return err;
1119
1120         /* Wait for func to complete...2 seconds max */
1121         for (i = 0; i < 2000; i++) {
1122                 err = finished(vdev, &done);
1123                 if (err)
1124                         return err;
1125                 if (done)
1126                         return 0;
1127                 usleep(1000);
1128         }
1129         return -ETIMEDOUT;
1130 }
1131
1132 static int enic_dev_open(struct enic *enic)
1133 {
1134         int err;
1135         int flags = CMD_OPENF_IG_DESCCACHE;
1136
1137         err = enic_dev_wait(enic->vdev, vnic_dev_open,
1138                 vnic_dev_open_done, flags);
1139         if (err)
1140                 dev_err(enic_get_dev(enic),
1141                         "vNIC device open failed, err %d\n", err);
1142
1143         return err;
1144 }
1145
1146 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1147 {
1148         dma_addr_t rss_key_buf_pa;
1149         union vnic_rss_key *rss_key_buf_va = NULL;
1150         int err, i;
1151         u8 name[NAME_MAX];
1152
1153         RTE_ASSERT(user_key != NULL);
1154         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1155         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1156                 &rss_key_buf_pa, name);
1157         if (!rss_key_buf_va)
1158                 return -ENOMEM;
1159
1160         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1161                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1162
1163         err = enic_set_rss_key(enic,
1164                 rss_key_buf_pa,
1165                 sizeof(union vnic_rss_key));
1166
1167         /* Save for later queries */
1168         if (!err) {
1169                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1170                            sizeof(union vnic_rss_key));
1171         }
1172         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1173                 rss_key_buf_va, rss_key_buf_pa);
1174
1175         return err;
1176 }
1177
1178 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1179 {
1180         dma_addr_t rss_cpu_buf_pa;
1181         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1182         int err;
1183         u8 name[NAME_MAX];
1184
1185         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1186         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1187                 &rss_cpu_buf_pa, name);
1188         if (!rss_cpu_buf_va)
1189                 return -ENOMEM;
1190
1191         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1192
1193         err = enic_set_rss_cpu(enic,
1194                 rss_cpu_buf_pa,
1195                 sizeof(union vnic_rss_cpu));
1196
1197         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1198                 rss_cpu_buf_va, rss_cpu_buf_pa);
1199
1200         /* Save for later queries */
1201         if (!err)
1202                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1203         return err;
1204 }
1205
1206 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1207         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1208 {
1209         const u8 tso_ipid_split_en = 0;
1210         int err;
1211
1212         err = enic_set_nic_cfg(enic,
1213                 rss_default_cpu, rss_hash_type,
1214                 rss_hash_bits, rss_base_cpu,
1215                 rss_enable, tso_ipid_split_en,
1216                 enic->ig_vlan_strip_en);
1217
1218         return err;
1219 }
1220
1221 /* Initialize RSS with defaults, called from dev_configure */
1222 int enic_init_rss_nic_cfg(struct enic *enic)
1223 {
1224         static uint8_t default_rss_key[] = {
1225                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1226                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1227                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1228                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1229         };
1230         struct rte_eth_rss_conf rss_conf;
1231         union vnic_rss_cpu rss_cpu;
1232         int ret, i;
1233
1234         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1235         /*
1236          * If setting key for the first time, and the user gives us none, then
1237          * push the default key to NIC.
1238          */
1239         if (rss_conf.rss_key == NULL) {
1240                 rss_conf.rss_key = default_rss_key;
1241                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1242         }
1243         ret = enic_set_rss_conf(enic, &rss_conf);
1244         if (ret) {
1245                 dev_err(enic, "Failed to configure RSS\n");
1246                 return ret;
1247         }
1248         if (enic->rss_enable) {
1249                 /* If enabling RSS, use the default reta */
1250                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1251                         rss_cpu.cpu[i / 4].b[i % 4] =
1252                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1253                 }
1254                 ret = enic_set_rss_reta(enic, &rss_cpu);
1255                 if (ret)
1256                         dev_err(enic, "Failed to set RSS indirection table\n");
1257         }
1258         return ret;
1259 }
1260
1261 int enic_setup_finish(struct enic *enic)
1262 {
1263         enic_init_soft_stats(enic);
1264
1265         /* Default conf */
1266         vnic_dev_packet_filter(enic->vdev,
1267                 1 /* directed  */,
1268                 1 /* multicast */,
1269                 1 /* broadcast */,
1270                 0 /* promisc   */,
1271                 1 /* allmulti  */);
1272
1273         enic->promisc = 0;
1274         enic->allmulti = 1;
1275
1276         return 0;
1277 }
1278
1279 static int enic_rss_conf_valid(struct enic *enic,
1280                                struct rte_eth_rss_conf *rss_conf)
1281 {
1282         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1283         if (enic->flow_type_rss_offloads == 0)
1284                 return 0;
1285         if (rss_conf->rss_key != NULL &&
1286             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1287                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1288                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1289                 return -EINVAL;
1290         }
1291         if (rss_conf->rss_hf != 0 &&
1292             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1293                 dev_err(enic, "Given rss_hf contains none of the supported"
1294                         " types\n");
1295                 return -EINVAL;
1296         }
1297         return 0;
1298 }
1299
1300 /* Set hash type and key according to rss_conf */
1301 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1302 {
1303         struct rte_eth_dev *eth_dev;
1304         uint64_t rss_hf;
1305         u8 rss_hash_type;
1306         u8 rss_enable;
1307         int ret;
1308
1309         RTE_ASSERT(rss_conf != NULL);
1310         ret = enic_rss_conf_valid(enic, rss_conf);
1311         if (ret) {
1312                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1313                 return ret;
1314         }
1315
1316         eth_dev = enic->rte_dev;
1317         rss_hash_type = 0;
1318         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1319         if (enic->rq_count > 1 &&
1320             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1321             rss_hf != 0) {
1322                 rss_enable = 1;
1323                 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1324                               ETH_RSS_NONFRAG_IPV4_OTHER))
1325                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1326                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1327                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1328                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1329                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1330                         if (enic->udp_rss_weak) {
1331                                 /*
1332                                  * 'TCP' is not a typo. The "weak" version of
1333                                  * UDP RSS requires both the TCP and UDP bits
1334                                  * be set. It does enable TCP RSS as well.
1335                                  */
1336                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1337                         }
1338                 }
1339                 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1340                               ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1341                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1342                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1343                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1344                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1345                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1346                         if (enic->udp_rss_weak)
1347                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1348                 }
1349         } else {
1350                 rss_enable = 0;
1351                 rss_hf = 0;
1352         }
1353
1354         /* Set the hash key if provided */
1355         if (rss_enable && rss_conf->rss_key) {
1356                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1357                 if (ret) {
1358                         dev_err(enic, "Failed to set RSS key\n");
1359                         return ret;
1360                 }
1361         }
1362
1363         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1364                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1365                               rss_enable);
1366         if (!ret) {
1367                 enic->rss_hf = rss_hf;
1368                 enic->rss_hash_type = rss_hash_type;
1369                 enic->rss_enable = rss_enable;
1370         } else {
1371                 dev_err(enic, "Failed to update RSS configurations."
1372                         " hash=0x%x\n", rss_hash_type);
1373         }
1374         return ret;
1375 }
1376
1377 int enic_set_vlan_strip(struct enic *enic)
1378 {
1379         /*
1380          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1381          * together. So, re-do niccfg, preserving the current RSS settings.
1382          */
1383         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1384                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1385                                enic->rss_enable);
1386 }
1387
1388 int enic_add_packet_filter(struct enic *enic)
1389 {
1390         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1391         return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1392                 enic->promisc, enic->allmulti);
1393 }
1394
1395 int enic_get_link_status(struct enic *enic)
1396 {
1397         return vnic_dev_link_status(enic->vdev);
1398 }
1399
1400 static void enic_dev_deinit(struct enic *enic)
1401 {
1402         /* stop link status checking */
1403         vnic_dev_notify_unset(enic->vdev);
1404
1405         /* mac_addrs is freed by rte_eth_dev_release_port() */
1406         rte_free(enic->cq);
1407         rte_free(enic->intr);
1408         rte_free(enic->rq);
1409         rte_free(enic->wq);
1410 }
1411
1412
1413 int enic_set_vnic_res(struct enic *enic)
1414 {
1415         struct rte_eth_dev *eth_dev = enic->rte_dev;
1416         int rc = 0;
1417         unsigned int required_rq, required_wq, required_cq, required_intr;
1418
1419         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1420         required_rq = eth_dev->data->nb_rx_queues * 2;
1421         required_wq = eth_dev->data->nb_tx_queues;
1422         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1423         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1424         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1425                 required_intr += eth_dev->data->nb_rx_queues;
1426         }
1427
1428         if (enic->conf_rq_count < required_rq) {
1429                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1430                         eth_dev->data->nb_rx_queues,
1431                         required_rq, enic->conf_rq_count);
1432                 rc = -EINVAL;
1433         }
1434         if (enic->conf_wq_count < required_wq) {
1435                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1436                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1437                 rc = -EINVAL;
1438         }
1439
1440         if (enic->conf_cq_count < required_cq) {
1441                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1442                         required_cq, enic->conf_cq_count);
1443                 rc = -EINVAL;
1444         }
1445         if (enic->conf_intr_count < required_intr) {
1446                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1447                         " interrupts. Required:%u, Configured:%u\n",
1448                         required_intr, enic->conf_intr_count);
1449                 rc = -EINVAL;
1450         }
1451
1452         if (rc == 0) {
1453                 enic->rq_count = eth_dev->data->nb_rx_queues;
1454                 enic->wq_count = eth_dev->data->nb_tx_queues;
1455                 enic->cq_count = enic->rq_count + enic->wq_count;
1456                 enic->intr_count = required_intr;
1457         }
1458
1459         return rc;
1460 }
1461
1462 /* Initialize the completion queue for an RQ */
1463 static int
1464 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1465 {
1466         struct vnic_rq *sop_rq, *data_rq;
1467         unsigned int cq_idx;
1468         int rc = 0;
1469
1470         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1471         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1472         cq_idx = rq_idx;
1473
1474         vnic_cq_clean(&enic->cq[cq_idx]);
1475         vnic_cq_init(&enic->cq[cq_idx],
1476                      0 /* flow_control_enable */,
1477                      1 /* color_enable */,
1478                      0 /* cq_head */,
1479                      0 /* cq_tail */,
1480                      1 /* cq_tail_color */,
1481                      0 /* interrupt_enable */,
1482                      1 /* cq_entry_enable */,
1483                      0 /* cq_message_enable */,
1484                      0 /* interrupt offset */,
1485                      0 /* cq_message_addr */);
1486
1487
1488         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1489                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1490                            sop_rq->ring.desc_count - 1, 1, 0);
1491         if (data_rq->in_use) {
1492                 vnic_rq_init_start(data_rq,
1493                                    enic_cq_rq(enic,
1494                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1495                                    data_rq->ring.desc_count - 1, 1, 0);
1496         }
1497
1498         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1499         if (rc)
1500                 return rc;
1501
1502         if (data_rq->in_use) {
1503                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1504                 if (rc) {
1505                         enic_rxmbuf_queue_release(enic, sop_rq);
1506                         return rc;
1507                 }
1508         }
1509
1510         return 0;
1511 }
1512
1513 /* The Cisco NIC can send and receive packets up to a max packet size
1514  * determined by the NIC type and firmware. There is also an MTU
1515  * configured into the NIC via the CIMC/UCSM management interface
1516  * which can be overridden by this function (up to the max packet size).
1517  * Depending on the network setup, doing so may cause packet drops
1518  * and unexpected behavior.
1519  */
1520 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1521 {
1522         unsigned int rq_idx;
1523         struct vnic_rq *rq;
1524         int rc = 0;
1525         uint16_t old_mtu;       /* previous setting */
1526         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1527         struct rte_eth_dev *eth_dev = enic->rte_dev;
1528
1529         old_mtu = eth_dev->data->mtu;
1530         config_mtu = enic->config.mtu;
1531
1532         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1533                 return -E_RTE_SECONDARY;
1534
1535         if (new_mtu > enic->max_mtu) {
1536                 dev_err(enic,
1537                         "MTU not updated: requested (%u) greater than max (%u)\n",
1538                         new_mtu, enic->max_mtu);
1539                 return -EINVAL;
1540         }
1541         if (new_mtu < ENIC_MIN_MTU) {
1542                 dev_info(enic,
1543                         "MTU not updated: requested (%u) less than min (%u)\n",
1544                         new_mtu, ENIC_MIN_MTU);
1545                 return -EINVAL;
1546         }
1547         if (new_mtu > config_mtu)
1548                 dev_warning(enic,
1549                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1550                         new_mtu, config_mtu);
1551
1552         /* Update the MTU and maximum packet length */
1553         eth_dev->data->mtu = new_mtu;
1554         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1555                 enic_mtu_to_max_rx_pktlen(new_mtu);
1556
1557         /*
1558          * If the device has not started (enic_enable), nothing to do.
1559          * Later, enic_enable() will set up RQs reflecting the new maximum
1560          * packet length.
1561          */
1562         if (!eth_dev->data->dev_started)
1563                 goto set_mtu_done;
1564
1565         /*
1566          * The device has started, re-do RQs on the fly. In the process, we
1567          * pick up the new maximum packet length.
1568          *
1569          * Some applications rely on the ability to change MTU without stopping
1570          * the device. So keep this behavior for now.
1571          */
1572         rte_spinlock_lock(&enic->mtu_lock);
1573
1574         /* Stop traffic on all RQs */
1575         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1576                 rq = &enic->rq[rq_idx];
1577                 if (rq->is_sop && rq->in_use) {
1578                         rc = enic_stop_rq(enic,
1579                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1580                         if (rc) {
1581                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1582                                 goto set_mtu_done;
1583                         }
1584                 }
1585         }
1586
1587         /* replace Rx function with a no-op to avoid getting stale pkts */
1588         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1589         rte_mb();
1590
1591         /* Allow time for threads to exit the real Rx function. */
1592         usleep(100000);
1593
1594         /* now it is safe to reconfigure the RQs */
1595
1596
1597         /* free and reallocate RQs with the new MTU */
1598         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1599                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1600                 if (!rq->in_use)
1601                         continue;
1602
1603                 enic_free_rq(rq);
1604                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1605                                    rq->tot_nb_desc, rq->rx_free_thresh);
1606                 if (rc) {
1607                         dev_err(enic,
1608                                 "Fatal MTU alloc error- No traffic will pass\n");
1609                         goto set_mtu_done;
1610                 }
1611
1612                 rc = enic_reinit_rq(enic, rq_idx);
1613                 if (rc) {
1614                         dev_err(enic,
1615                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1616                         goto set_mtu_done;
1617                 }
1618         }
1619
1620         /* put back the real receive function */
1621         rte_mb();
1622         enic_pick_rx_handler(eth_dev);
1623         rte_mb();
1624
1625         /* restart Rx traffic */
1626         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1627                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1628                 if (rq->is_sop && rq->in_use)
1629                         enic_start_rq(enic, rq_idx);
1630         }
1631
1632 set_mtu_done:
1633         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1634         rte_spinlock_unlock(&enic->mtu_lock);
1635         return rc;
1636 }
1637
1638 static int enic_dev_init(struct enic *enic)
1639 {
1640         int err;
1641         struct rte_eth_dev *eth_dev = enic->rte_dev;
1642
1643         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1644
1645         /* Get vNIC configuration
1646         */
1647         err = enic_get_vnic_config(enic);
1648         if (err) {
1649                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1650                 return err;
1651         }
1652
1653         /* Get available resource counts */
1654         enic_get_res_counts(enic);
1655         if (enic->conf_rq_count == 1) {
1656                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1657                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1658                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1659                 return -EINVAL;
1660         }
1661         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1662         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1663                                enic->conf_cq_count, 8);
1664         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1665                                  enic->conf_intr_count, 8);
1666         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1667                                enic->conf_rq_count, 8);
1668         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1669                                enic->conf_wq_count, 8);
1670         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1671                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1672                 return -1;
1673         }
1674         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1675                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1676                 return -1;
1677         }
1678         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1679                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1680                 return -1;
1681         }
1682         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1683                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1684                 return -1;
1685         }
1686
1687         /* Get the supported filters */
1688         enic_fdir_info(enic);
1689
1690         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1691                                         sizeof(struct rte_ether_addr) *
1692                                         ENIC_UNICAST_PERFECT_FILTERS, 0);
1693         if (!eth_dev->data->mac_addrs) {
1694                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1695                 return -1;
1696         }
1697         rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1698                         eth_dev->data->mac_addrs);
1699
1700         vnic_dev_set_reset_flag(enic->vdev, 0);
1701
1702         LIST_INIT(&enic->flows);
1703
1704         /* set up link status checking */
1705         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1706
1707         /*
1708          * When Geneve with options offload is available, always disable it
1709          * first as it can interfere with user flow rules.
1710          */
1711         if (enic->geneve_opt_avail &&
1712             vnic_dev_overlay_offload_ctrl(enic->vdev,
1713                         OVERLAY_FEATURE_GENEVE,
1714                         OVERLAY_OFFLOAD_DISABLE)) {
1715                 dev_err(enic, "failed to disable geneve+option\n");
1716         }
1717         enic->overlay_offload = false;
1718         if (enic->disable_overlay && enic->vxlan) {
1719                 /*
1720                  * Explicitly disable overlay offload as the setting is
1721                  * sticky, and resetting vNIC does not disable it.
1722                  */
1723                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1724                                                   OVERLAY_FEATURE_VXLAN,
1725                                                   OVERLAY_OFFLOAD_DISABLE)) {
1726                         dev_err(enic, "failed to disable overlay offload\n");
1727                 } else {
1728                         dev_info(enic, "Overlay offload is disabled\n");
1729                 }
1730         }
1731         if (!enic->disable_overlay && enic->vxlan &&
1732             /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1733             vnic_dev_overlay_offload_ctrl(enic->vdev,
1734                                           OVERLAY_FEATURE_VXLAN,
1735                                           OVERLAY_OFFLOAD_ENABLE) == 0) {
1736                 enic->tx_offload_capa |=
1737                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1738                         DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1739                         DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1740                 enic->tx_offload_mask |=
1741                         PKT_TX_OUTER_IPV6 |
1742                         PKT_TX_OUTER_IPV4 |
1743                         PKT_TX_OUTER_IP_CKSUM |
1744                         PKT_TX_TUNNEL_MASK;
1745                 enic->overlay_offload = true;
1746                 dev_info(enic, "Overlay offload is enabled\n");
1747         }
1748         /* Geneve with options offload requires overlay offload */
1749         if (enic->overlay_offload && enic->geneve_opt_avail &&
1750             enic->geneve_opt_request) {
1751                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1752                                 OVERLAY_FEATURE_GENEVE,
1753                                 OVERLAY_OFFLOAD_ENABLE)) {
1754                         dev_err(enic, "failed to enable geneve+option\n");
1755                 } else {
1756                         enic->geneve_opt_enabled = 1;
1757                         dev_info(enic, "Geneve with options is enabled\n");
1758                 }
1759         }
1760         /*
1761          * Reset the vxlan port if HW vxlan parsing is available. It
1762          * is always enabled regardless of overlay offload
1763          * enable/disable.
1764          */
1765         if (enic->vxlan) {
1766                 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1767                 /*
1768                  * Reset the vxlan port to the default, as the NIC firmware
1769                  * does not reset it automatically and keeps the old setting.
1770                  */
1771                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1772                                                  OVERLAY_CFG_VXLAN_PORT_UPDATE,
1773                                                  ENIC_DEFAULT_VXLAN_PORT)) {
1774                         dev_err(enic, "failed to update vxlan port\n");
1775                         return -EINVAL;
1776                 }
1777         }
1778
1779         return 0;
1780
1781 }
1782
1783 int enic_probe(struct enic *enic)
1784 {
1785         struct rte_pci_device *pdev = enic->pdev;
1786         int err = -1;
1787
1788         dev_debug(enic, "Initializing ENIC PMD\n");
1789
1790         /* if this is a secondary process the hardware is already initialized */
1791         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1792                 return 0;
1793
1794         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1795         enic->bar0.len = pdev->mem_resource[0].len;
1796
1797         /* Register vNIC device */
1798         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1799         if (!enic->vdev) {
1800                 dev_err(enic, "vNIC registration failed, aborting\n");
1801                 goto err_out;
1802         }
1803
1804         LIST_INIT(&enic->memzone_list);
1805         rte_spinlock_init(&enic->memzone_list_lock);
1806
1807         vnic_register_cbacks(enic->vdev,
1808                 enic_alloc_consistent,
1809                 enic_free_consistent);
1810
1811         /*
1812          * Allocate the consistent memory for stats upfront so both primary and
1813          * secondary processes can dump stats.
1814          */
1815         err = vnic_dev_alloc_stats_mem(enic->vdev);
1816         if (err) {
1817                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1818                 goto err_out_unregister;
1819         }
1820         /* Issue device open to get device in known state */
1821         err = enic_dev_open(enic);
1822         if (err) {
1823                 dev_err(enic, "vNIC dev open failed, aborting\n");
1824                 goto err_out_unregister;
1825         }
1826
1827         /* Set ingress vlan rewrite mode before vnic initialization */
1828         dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1829                   enic->ig_vlan_rewrite_mode);
1830         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1831                 enic->ig_vlan_rewrite_mode);
1832         if (err) {
1833                 dev_err(enic,
1834                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1835                 goto err_out_dev_close;
1836         }
1837
1838         /* Issue device init to initialize the vnic-to-switch link.
1839          * We'll start with carrier off and wait for link UP
1840          * notification later to turn on carrier.  We don't need
1841          * to wait here for the vnic-to-switch link initialization
1842          * to complete; link UP notification is the indication that
1843          * the process is complete.
1844          */
1845
1846         err = vnic_dev_init(enic->vdev, 0);
1847         if (err) {
1848                 dev_err(enic, "vNIC dev init failed, aborting\n");
1849                 goto err_out_dev_close;
1850         }
1851
1852         err = enic_dev_init(enic);
1853         if (err) {
1854                 dev_err(enic, "Device initialization failed, aborting\n");
1855                 goto err_out_dev_close;
1856         }
1857
1858         return 0;
1859
1860 err_out_dev_close:
1861         vnic_dev_close(enic->vdev);
1862 err_out_unregister:
1863         vnic_dev_unregister(enic->vdev);
1864 err_out:
1865         return err;
1866 }
1867
1868 void enic_remove(struct enic *enic)
1869 {
1870         enic_dev_deinit(enic);
1871         vnic_dev_close(enic->vdev);
1872         vnic_dev_unregister(enic->vdev);
1873 }