drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11
  12 #include <rte_pci.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_ethdev_driver.h>
  19
  20 #include "enic_compat.h"
  21 #include "enic.h"
  22 #include "wq_enet_desc.h"
  23 #include "rq_enet_desc.h"
  24 #include "cq_enet_desc.h"
  25 #include "vnic_enet.h"
  26 #include "vnic_dev.h"
  27 #include "vnic_wq.h"
  28 #include "vnic_rq.h"
  29 #include "vnic_cq.h"
  30 #include "vnic_intr.h"
  31 #include "vnic_nic.h"
  32
  33 static inline int enic_is_sriov_vf(struct enic *enic)
  34 {
  35         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  36 }
  37
  38 static int is_zero_addr(uint8_t *addr)
  39 {
  40         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  41 }
  42
  43 static int is_mcast_addr(uint8_t *addr)
  44 {
  45         return addr[0] & 1;
  46 }
  47
  48 static int is_eth_addr_valid(uint8_t *addr)
  49 {
  50         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  51 }
  52
  53 static void
  54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  55 {
  56         uint16_t i;
  57
  58         if (!rq || !rq->mbuf_ring) {
  59                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  60                 return;
  61         }
  62
  63         for (i = 0; i < rq->ring.desc_count; i++) {
  64                 if (rq->mbuf_ring[i]) {
  65                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  66                         rq->mbuf_ring[i] = NULL;
  67                 }
  68         }
  69 }
  70
  71 static void enic_free_wq_buf(struct rte_mbuf **buf)
  72 {
  73         struct rte_mbuf *mbuf = *buf;
  74
  75         rte_pktmbuf_free_seg(mbuf);
  76         *buf = NULL;
  77 }
  78
  79 static void enic_log_q_error(struct enic *enic)
  80 {
  81         unsigned int i;
  82         u32 error_status;
  83
  84         for (i = 0; i < enic->wq_count; i++) {
  85                 error_status = vnic_wq_error_status(&enic->wq[i]);
  86                 if (error_status)
  87                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  88                                 error_status);
  89         }
  90
  91         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  92                 if (!enic->rq[i].in_use)
  93                         continue;
  94                 error_status = vnic_rq_error_status(&enic->rq[i]);
  95                 if (error_status)
  96                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  97                                 error_status);
  98         }
  99 }
 100
 101 static void enic_clear_soft_stats(struct enic *enic)
 102 {
 103         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 104         rte_atomic64_clear(&soft_stats->rx_nombuf);
 105         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 106         rte_atomic64_clear(&soft_stats->tx_oversized);
 107 }
 108
 109 static void enic_init_soft_stats(struct enic *enic)
 110 {
 111         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 112         rte_atomic64_init(&soft_stats->rx_nombuf);
 113         rte_atomic64_init(&soft_stats->rx_packet_errors);
 114         rte_atomic64_init(&soft_stats->tx_oversized);
 115         enic_clear_soft_stats(enic);
 116 }
 117
 118 int enic_dev_stats_clear(struct enic *enic)
 119 {
 120         int ret;
 121
 122         ret = vnic_dev_stats_clear(enic->vdev);
 123         if (ret != 0) {
 124                 dev_err(enic, "Error in clearing stats\n");
 125                 return ret;
 126         }
 127         enic_clear_soft_stats(enic);
 128
 129         return 0;
 130 }
 131
 132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 133 {
 134         struct vnic_stats *stats;
 135         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 136         int64_t rx_truncated;
 137         uint64_t rx_packet_errors;
 138         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 139
 140         if (ret) {
 141                 dev_err(enic, "Error in getting stats\n");
 142                 return ret;
 143         }
 144
 145         /* The number of truncated packets can only be calculated by
 146          * subtracting a hardware counter from error packets received by
 147          * the driver. Note: this causes transient inaccuracies in the
 148          * ipackets count. Also, the length of truncated packets are
 149          * counted in ibytes even though truncated packets are dropped
 150          * which can make ibytes be slightly higher than it should be.
 151          */
 152         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 153         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 154
 155         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 156         r_stats->opackets = stats->tx.tx_frames_ok;
 157
 158         r_stats->ibytes = stats->rx.rx_bytes_ok;
 159         r_stats->obytes = stats->tx.tx_bytes_ok;
 160
 161         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 162         r_stats->oerrors = stats->tx.tx_errors
 163                            + rte_atomic64_read(&soft_stats->tx_oversized);
 164
 165         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 166
 167         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 168         return 0;
 169 }
 170
 171 int enic_del_mac_address(struct enic *enic, int mac_index)
 172 {
 173         struct rte_eth_dev *eth_dev = enic->rte_dev;
 174         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 175
 176         return vnic_dev_del_addr(enic->vdev, mac_addr);
 177 }
 178
 179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 180 {
 181         int err;
 182
 183         if (!is_eth_addr_valid(mac_addr)) {
 184                 dev_err(enic, "invalid mac address\n");
 185                 return -EINVAL;
 186         }
 187
 188         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 189         if (err)
 190                 dev_err(enic, "add mac addr failed\n");
 191         return err;
 192 }
 193
 194 static void
 195 enic_free_rq_buf(struct rte_mbuf **mbuf)
 196 {
 197         if (*mbuf == NULL)
 198                 return;
 199
 200         rte_pktmbuf_free(*mbuf);
 201         *mbuf = NULL;
 202 }
 203
 204 void enic_init_vnic_resources(struct enic *enic)
 205 {
 206         unsigned int error_interrupt_enable = 1;
 207         unsigned int error_interrupt_offset = 0;
 208         unsigned int rxq_interrupt_enable = 0;
 209         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 210         unsigned int index = 0;
 211         unsigned int cq_idx;
 212         struct vnic_rq *data_rq;
 213
 214         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 215                 rxq_interrupt_enable = 1;
 216
 217         for (index = 0; index < enic->rq_count; index++) {
 218                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 219
 220                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 221                         cq_idx,
 222                         error_interrupt_enable,
 223                         error_interrupt_offset);
 224
 225                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 226                 if (data_rq->in_use)
 227                         vnic_rq_init(data_rq,
 228                                      cq_idx,
 229                                      error_interrupt_enable,
 230                                      error_interrupt_offset);
 231
 232                 vnic_cq_init(&enic->cq[cq_idx],
 233                         0 /* flow_control_enable */,
 234                         1 /* color_enable */,
 235                         0 /* cq_head */,
 236                         0 /* cq_tail */,
 237                         1 /* cq_tail_color */,
 238                         rxq_interrupt_enable,
 239                         1 /* cq_entry_enable */,
 240                         0 /* cq_message_enable */,
 241                         rxq_interrupt_offset,
 242                         0 /* cq_message_addr */);
 243                 if (rxq_interrupt_enable)
 244                         rxq_interrupt_offset++;
 245         }
 246
 247         for (index = 0; index < enic->wq_count; index++) {
 248                 vnic_wq_init(&enic->wq[index],
 249                         enic_cq_wq(enic, index),
 250                         error_interrupt_enable,
 251                         error_interrupt_offset);
 252                 /* Compute unsupported ol flags for enic_prep_pkts() */
 253                 enic->wq[index].tx_offload_notsup_mask =
 254                         PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 255
 256                 cq_idx = enic_cq_wq(enic, index);
 257                 vnic_cq_init(&enic->cq[cq_idx],
 258                         0 /* flow_control_enable */,
 259                         1 /* color_enable */,
 260                         0 /* cq_head */,
 261                         0 /* cq_tail */,
 262                         1 /* cq_tail_color */,
 263                         0 /* interrupt_enable */,
 264                         0 /* cq_entry_enable */,
 265                         1 /* cq_message_enable */,
 266                         0 /* interrupt offset */,
 267                         (u64)enic->wq[index].cqmsg_rz->iova);
 268         }
 269
 270         for (index = 0; index < enic->intr_count; index++) {
 271                 vnic_intr_init(&enic->intr[index],
 272                                enic->config.intr_timer_usec,
 273                                enic->config.intr_timer_type,
 274                                /*mask_on_assertion*/1);
 275         }
 276 }
 277
 278
 279 static int
 280 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 281 {
 282         struct rte_mbuf *mb;
 283         struct rq_enet_desc *rqd = rq->ring.descs;
 284         unsigned i;
 285         dma_addr_t dma_addr;
 286         uint32_t max_rx_pkt_len;
 287         uint16_t rq_buf_len;
 288
 289         if (!rq->in_use)
 290                 return 0;
 291
 292         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 293                   rq->ring.desc_count);
 294
 295         /*
 296          * If *not* using scatter and the mbuf size is greater than the
 297          * requested max packet size (max_rx_pkt_len), then reduce the
 298          * posted buffer size to max_rx_pkt_len. HW still receives packets
 299          * larger than max_rx_pkt_len, but they will be truncated, which we
 300          * drop in the rx handler. Not ideal, but better than returning
 301          * large packets when the user is not expecting them.
 302          */
 303         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 304         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 305         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 306                 rq_buf_len = max_rx_pkt_len;
 307         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 308                 mb = rte_mbuf_raw_alloc(rq->mp);
 309                 if (mb == NULL) {
 310                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 311                         (unsigned)rq->index);
 312                         return -ENOMEM;
 313                 }
 314
 315                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 316                 dma_addr = (dma_addr_t)(mb->buf_iova
 317                            + RTE_PKTMBUF_HEADROOM);
 318                 rq_enet_desc_enc(rqd, dma_addr,
 319                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 320                                 : RQ_ENET_TYPE_NOT_SOP),
 321                                 rq_buf_len);
 322                 rq->mbuf_ring[i] = mb;
 323         }
 324         /*
 325          * Do not post the buffers to the NIC until we enable the RQ via
 326          * enic_start_rq().
 327          */
 328         rq->need_initial_post = true;
 329         /* Initialize fetch index while RQ is disabled */
 330         iowrite32(0, &rq->ctrl->fetch_index);
 331         return 0;
 332 }
 333
 334 /*
 335  * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
 336  * allocated the buffers and filled the RQ descriptor ring. Just need to push
 337  * the post index to the NIC.
 338  */
 339 static void
 340 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 341 {
 342         if (!rq->in_use || !rq->need_initial_post)
 343                 return;
 344
 345         /* make sure all prior writes are complete before doing the PIO write */
 346         rte_rmb();
 347
 348         /* Post all but the last buffer to VIC. */
 349         rq->posted_index = rq->ring.desc_count - 1;
 350
 351         rq->rx_nb_hold = 0;
 352
 353         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 354                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 355         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 356         rte_rmb();
 357         rq->need_initial_post = false;
 358 }
 359
 360 void *
 361 enic_alloc_consistent(void *priv, size_t size,
 362         dma_addr_t *dma_handle, u8 *name)
 363 {
 364         void *vaddr;
 365         const struct rte_memzone *rz;
 366         *dma_handle = 0;
 367         struct enic *enic = (struct enic *)priv;
 368         struct enic_memzone_entry *mze;
 369
 370         rz = rte_memzone_reserve_aligned((const char *)name, size,
 371                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 372         if (!rz) {
 373                 pr_err("%s : Failed to allocate memory requested for %s\n",
 374                         __func__, name);
 375                 return NULL;
 376         }
 377
 378         vaddr = rz->addr;
 379         *dma_handle = (dma_addr_t)rz->iova;
 380
 381         mze = rte_malloc("enic memzone entry",
 382                          sizeof(struct enic_memzone_entry), 0);
 383
 384         if (!mze) {
 385                 pr_err("%s : Failed to allocate memory for memzone list\n",
 386                        __func__);
 387                 rte_memzone_free(rz);
 388                 return NULL;
 389         }
 390
 391         mze->rz = rz;
 392
 393         rte_spinlock_lock(&enic->memzone_list_lock);
 394         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 395         rte_spinlock_unlock(&enic->memzone_list_lock);
 396
 397         return vaddr;
 398 }
 399
 400 void
 401 enic_free_consistent(void *priv,
 402                      __rte_unused size_t size,
 403                      void *vaddr,
 404                      dma_addr_t dma_handle)
 405 {
 406         struct enic_memzone_entry *mze;
 407         struct enic *enic = (struct enic *)priv;
 408
 409         rte_spinlock_lock(&enic->memzone_list_lock);
 410         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 411                 if (mze->rz->addr == vaddr &&
 412                     mze->rz->iova == dma_handle)
 413                         break;
 414         }
 415         if (mze == NULL) {
 416                 rte_spinlock_unlock(&enic->memzone_list_lock);
 417                 dev_warning(enic,
 418                             "Tried to free memory, but couldn't find it in the memzone list\n");
 419                 return;
 420         }
 421         LIST_REMOVE(mze, entries);
 422         rte_spinlock_unlock(&enic->memzone_list_lock);
 423         rte_memzone_free(mze->rz);
 424         rte_free(mze);
 425 }
 426
 427 int enic_link_update(struct rte_eth_dev *eth_dev)
 428 {
 429         struct enic *enic = pmd_priv(eth_dev);
 430         struct rte_eth_link link;
 431
 432         memset(&link, 0, sizeof(link));
 433         link.link_status = enic_get_link_status(enic);
 434         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 435         link.link_speed = vnic_dev_port_speed(enic->vdev);
 436
 437         return rte_eth_linkstatus_set(eth_dev, &link);
 438 }
 439
 440 static void
 441 enic_intr_handler(void *arg)
 442 {
 443         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 444         struct enic *enic = pmd_priv(dev);
 445
 446         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 447
 448         enic_link_update(dev);
 449         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 450         enic_log_q_error(enic);
 451 }
 452
 453 static int enic_rxq_intr_init(struct enic *enic)
 454 {
 455         struct rte_intr_handle *intr_handle;
 456         uint32_t rxq_intr_count, i;
 457         int err;
 458
 459         intr_handle = enic->rte_dev->intr_handle;
 460         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 461                 return 0;
 462         /*
 463          * Rx queue interrupts only work when we have MSI-X interrupts,
 464          * one per queue. Sharing one interrupt is technically
 465          * possible with VIC, but it is not worth the complications it brings.
 466          */
 467         if (!rte_intr_cap_multiple(intr_handle)) {
 468                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 469                         " (vfio-pci driver)\n");
 470                 return -ENOTSUP;
 471         }
 472         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 473         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 474         if (err) {
 475                 dev_err(enic, "Failed to enable event fds for Rx queue"
 476                         " interrupts\n");
 477                 return err;
 478         }
 479         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 480                                             rxq_intr_count * sizeof(int), 0);
 481         if (intr_handle->intr_vec == NULL) {
 482                 dev_err(enic, "Failed to allocate intr_vec\n");
 483                 return -ENOMEM;
 484         }
 485         for (i = 0; i < rxq_intr_count; i++)
 486                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 487         return 0;
 488 }
 489
 490 static void enic_rxq_intr_deinit(struct enic *enic)
 491 {
 492         struct rte_intr_handle *intr_handle;
 493
 494         intr_handle = enic->rte_dev->intr_handle;
 495         rte_intr_efd_disable(intr_handle);
 496         if (intr_handle->intr_vec != NULL) {
 497                 rte_free(intr_handle->intr_vec);
 498                 intr_handle->intr_vec = NULL;
 499         }
 500 }
 501
 502 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 503 {
 504         struct wq_enet_desc *desc;
 505         struct vnic_wq *wq;
 506         unsigned int i;
 507
 508         /*
 509          * Fill WQ descriptor fields that never change. Every descriptor is
 510          * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
 511          * descriptors (i.e. request one completion update every 32 packets).
 512          */
 513         wq = &enic->wq[queue_idx];
 514         desc = (struct wq_enet_desc *)wq->ring.descs;
 515         for (i = 0; i < wq->ring.desc_count; i++, desc++) {
 516                 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
 517                 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
 518                         desc->header_length_flags |=
 519                                 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
 520         }
 521 }
 522
 523 /*
 524  * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
 525  * used when that file is not compiled.
 526  */
 527 __rte_weak bool
 528 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
 529 {
 530         return false;
 531 }
 532
 533 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
 534 {
 535         struct enic *enic = pmd_priv(eth_dev);
 536
 537         /*
 538          * Preference order:
 539          * 1. The vectorized handler if possible and requested.
 540          * 2. The non-scatter, simplified handler if scatter Rx is not used.
 541          * 3. The default handler as a fallback.
 542          */
 543         if (enic_use_vector_rx_handler(eth_dev))
 544                 return;
 545         if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 546                 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
 547                 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
 548         } else {
 549                 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
 550                 eth_dev->rx_pkt_burst = &enic_recv_pkts;
 551         }
 552 }
 553
 554 /* Secondary process uses this to set the Tx handler */
 555 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
 556 {
 557         struct enic *enic = pmd_priv(eth_dev);
 558
 559         if (enic->use_simple_tx_handler) {
 560                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 561                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 562         } else {
 563                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 564                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 565         }
 566 }
 567
 568 int enic_enable(struct enic *enic)
 569 {
 570         unsigned int index;
 571         int err;
 572         struct rte_eth_dev *eth_dev = enic->rte_dev;
 573         uint64_t simple_tx_offloads;
 574         uintptr_t p;
 575
 576         if (enic->enable_avx2_rx) {
 577                 struct rte_mbuf mb_def = { .buf_addr = 0 };
 578
 579                 /*
 580                  * mbuf_initializer contains const-after-init fields of
 581                  * receive mbufs (i.e. 64 bits of fields from rearm_data).
 582                  * It is currently used by the vectorized handler.
 583                  */
 584                 mb_def.nb_segs = 1;
 585                 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 586                 mb_def.port = enic->port_id;
 587                 rte_mbuf_refcnt_set(&mb_def, 1);
 588                 rte_compiler_barrier();
 589                 p = (uintptr_t)&mb_def.rearm_data;
 590                 enic->mbuf_initializer = *(uint64_t *)p;
 591         }
 592
 593         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 594         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 595
 596         /* vnic notification of link status has already been turned on in
 597          * enic_dev_init() which is called during probe time.  Here we are
 598          * just turning on interrupt vector 0 if needed.
 599          */
 600         if (eth_dev->data->dev_conf.intr_conf.lsc)
 601                 vnic_dev_notify_set(enic->vdev, 0);
 602
 603         err = enic_rxq_intr_init(enic);
 604         if (err)
 605                 return err;
 606         if (enic_clsf_init(enic))
 607                 dev_warning(enic, "Init of hash table for clsf failed."\
 608                         "Flow director feature will not work\n");
 609
 610         if (enic_fm_init(enic))
 611                 dev_warning(enic, "Init of flowman failed.\n");
 612
 613         for (index = 0; index < enic->rq_count; index++) {
 614                 err = enic_alloc_rx_queue_mbufs(enic,
 615                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 616                 if (err) {
 617                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 618                         return err;
 619                 }
 620                 err = enic_alloc_rx_queue_mbufs(enic,
 621                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 622                 if (err) {
 623                         /* release the allocated mbufs for the sop rq*/
 624                         enic_rxmbuf_queue_release(enic,
 625                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 626
 627                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 628                         return err;
 629                 }
 630         }
 631
 632         /*
 633          * Use the simple TX handler if possible. Only checksum offloads
 634          * and vlan insertion are supported.
 635          */
 636         simple_tx_offloads = enic->tx_offload_capa &
 637                 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 638                  DEV_TX_OFFLOAD_VLAN_INSERT |
 639                  DEV_TX_OFFLOAD_IPV4_CKSUM |
 640                  DEV_TX_OFFLOAD_UDP_CKSUM |
 641                  DEV_TX_OFFLOAD_TCP_CKSUM);
 642         if ((eth_dev->data->dev_conf.txmode.offloads &
 643              ~simple_tx_offloads) == 0) {
 644                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 645                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 646                 for (index = 0; index < enic->wq_count; index++)
 647                         enic_prep_wq_for_simple_tx(enic, index);
 648                 enic->use_simple_tx_handler = 1;
 649         } else {
 650                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 651                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 652         }
 653
 654         enic_pick_rx_handler(eth_dev);
 655
 656         for (index = 0; index < enic->wq_count; index++)
 657                 enic_start_wq(enic, index);
 658         for (index = 0; index < enic->rq_count; index++)
 659                 enic_start_rq(enic, index);
 660
 661         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 662
 663         vnic_dev_enable_wait(enic->vdev);
 664
 665         /* Register and enable error interrupt */
 666         rte_intr_callback_register(&(enic->pdev->intr_handle),
 667                 enic_intr_handler, (void *)enic->rte_dev);
 668
 669         rte_intr_enable(&(enic->pdev->intr_handle));
 670         /* Unmask LSC interrupt */
 671         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 672
 673         return 0;
 674 }
 675
 676 int enic_alloc_intr_resources(struct enic *enic)
 677 {
 678         int err;
 679         unsigned int i;
 680
 681         dev_info(enic, "vNIC resources used:  "\
 682                 "wq %d rq %d cq %d intr %d\n",
 683                 enic->wq_count, enic_vnic_rq_count(enic),
 684                 enic->cq_count, enic->intr_count);
 685
 686         for (i = 0; i < enic->intr_count; i++) {
 687                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 688                 if (err) {
 689                         enic_free_vnic_resources(enic);
 690                         return err;
 691                 }
 692         }
 693         return 0;
 694 }
 695
 696 void enic_free_rq(void *rxq)
 697 {
 698         struct vnic_rq *rq_sop, *rq_data;
 699         struct enic *enic;
 700
 701         if (rxq == NULL)
 702                 return;
 703
 704         rq_sop = (struct vnic_rq *)rxq;
 705         enic = vnic_dev_priv(rq_sop->vdev);
 706         rq_data = &enic->rq[rq_sop->data_queue_idx];
 707
 708         if (rq_sop->free_mbufs) {
 709                 struct rte_mbuf **mb;
 710                 int i;
 711
 712                 mb = rq_sop->free_mbufs;
 713                 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
 714                      i < ENIC_RX_BURST_MAX; i++)
 715                         rte_pktmbuf_free(mb[i]);
 716                 rte_free(rq_sop->free_mbufs);
 717                 rq_sop->free_mbufs = NULL;
 718                 rq_sop->num_free_mbufs = 0;
 719         }
 720
 721         enic_rxmbuf_queue_release(enic, rq_sop);
 722         if (rq_data->in_use)
 723                 enic_rxmbuf_queue_release(enic, rq_data);
 724
 725         rte_free(rq_sop->mbuf_ring);
 726         if (rq_data->in_use)
 727                 rte_free(rq_data->mbuf_ring);
 728
 729         rq_sop->mbuf_ring = NULL;
 730         rq_data->mbuf_ring = NULL;
 731
 732         vnic_rq_free(rq_sop);
 733         if (rq_data->in_use)
 734                 vnic_rq_free(rq_data);
 735
 736         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 737
 738         rq_sop->in_use = 0;
 739         rq_data->in_use = 0;
 740 }
 741
 742 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 743 {
 744         struct rte_eth_dev_data *data = enic->dev_data;
 745         vnic_wq_enable(&enic->wq[queue_idx]);
 746         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 747 }
 748
 749 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 750 {
 751         struct rte_eth_dev_data *data = enic->dev_data;
 752         int ret;
 753
 754         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 755         if (ret)
 756                 return ret;
 757
 758         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 759         return 0;
 760 }
 761
 762 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 763 {
 764         struct rte_eth_dev_data *data = enic->dev_data;
 765         struct vnic_rq *rq_sop;
 766         struct vnic_rq *rq_data;
 767         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 768         rq_data = &enic->rq[rq_sop->data_queue_idx];
 769
 770         if (rq_data->in_use) {
 771                 vnic_rq_enable(rq_data);
 772                 enic_initial_post_rx(enic, rq_data);
 773         }
 774         rte_mb();
 775         vnic_rq_enable(rq_sop);
 776         enic_initial_post_rx(enic, rq_sop);
 777         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 778 }
 779
 780 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 781 {
 782         struct rte_eth_dev_data *data = enic->dev_data;
 783         int ret1 = 0, ret2 = 0;
 784         struct vnic_rq *rq_sop;
 785         struct vnic_rq *rq_data;
 786         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 787         rq_data = &enic->rq[rq_sop->data_queue_idx];
 788
 789         ret2 = vnic_rq_disable(rq_sop);
 790         rte_mb();
 791         if (rq_data->in_use)
 792                 ret1 = vnic_rq_disable(rq_data);
 793
 794         if (ret2)
 795                 return ret2;
 796         else if (ret1)
 797                 return ret1;
 798
 799         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 800         return 0;
 801 }
 802
 803 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 804         unsigned int socket_id, struct rte_mempool *mp,
 805         uint16_t nb_desc, uint16_t free_thresh)
 806 {
 807         int rc;
 808         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 809         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 810         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 811         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 812         unsigned int mbuf_size, mbufs_per_pkt;
 813         unsigned int nb_sop_desc, nb_data_desc;
 814         uint16_t min_sop, max_sop, min_data, max_data;
 815         uint32_t max_rx_pkt_len;
 816
 817         rq_sop->is_sop = 1;
 818         rq_sop->data_queue_idx = data_queue_idx;
 819         rq_data->is_sop = 0;
 820         rq_data->data_queue_idx = 0;
 821         rq_sop->socket_id = socket_id;
 822         rq_sop->mp = mp;
 823         rq_data->socket_id = socket_id;
 824         rq_data->mp = mp;
 825         rq_sop->in_use = 1;
 826         rq_sop->rx_free_thresh = free_thresh;
 827         rq_data->rx_free_thresh = free_thresh;
 828         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 829                   free_thresh);
 830
 831         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 832                                RTE_PKTMBUF_HEADROOM);
 833         /* max_rx_pkt_len includes the ethernet header and CRC. */
 834         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 835
 836         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 837             DEV_RX_OFFLOAD_SCATTER) {
 838                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 839                 /* ceil((max pkt len)/mbuf_size) */
 840                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 841         } else {
 842                 dev_info(enic, "Scatter rx mode disabled\n");
 843                 mbufs_per_pkt = 1;
 844                 if (max_rx_pkt_len > mbuf_size) {
 845                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 846                                     " larger than the mbuf size (%u), and"
 847                                     " scatter is disabled. Larger packets will"
 848                                     " be truncated.\n",
 849                                     max_rx_pkt_len, mbuf_size);
 850                 }
 851         }
 852
 853         if (mbufs_per_pkt > 1) {
 854                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 855                 rq_sop->data_queue_enable = 1;
 856                 rq_data->in_use = 1;
 857                 /*
 858                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 859                  * receives packet sizes up to the "max" MTU.
 860                  * If not using scatter, we can achieve the effect of dropping
 861                  * larger packets by reducing the size of posted buffers.
 862                  * See enic_alloc_rx_queue_mbufs().
 863                  */
 864                 if (max_rx_pkt_len <
 865                     enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
 866                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 867                                     " when scatter rx mode is in use.\n");
 868                 }
 869         } else {
 870                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 871                          queue_idx);
 872                 rq_sop->data_queue_enable = 0;
 873                 rq_data->in_use = 0;
 874         }
 875
 876         /* number of descriptors have to be a multiple of 32 */
 877         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
 878         nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 879
 880         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 881         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 882
 883         if (mbufs_per_pkt > 1) {
 884                 min_sop = ENIC_RX_BURST_MAX;
 885                 max_sop = ((enic->config.rq_desc_count /
 886                             (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 887                 min_data = min_sop * (mbufs_per_pkt - 1);
 888                 max_data = enic->config.rq_desc_count;
 889         } else {
 890                 min_sop = ENIC_RX_BURST_MAX;
 891                 max_sop = enic->config.rq_desc_count;
 892                 min_data = 0;
 893                 max_data = 0;
 894         }
 895
 896         if (nb_desc < (min_sop + min_data)) {
 897                 dev_warning(enic,
 898                             "Number of rx descs too low, adjusting to minimum\n");
 899                 nb_sop_desc = min_sop;
 900                 nb_data_desc = min_data;
 901         } else if (nb_desc > (max_sop + max_data)) {
 902                 dev_warning(enic,
 903                             "Number of rx_descs too high, adjusting to maximum\n");
 904                 nb_sop_desc = max_sop;
 905                 nb_data_desc = max_data;
 906         }
 907         if (mbufs_per_pkt > 1) {
 908                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 909                          " rx descriptor range is %u to %u\n",
 910                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 911                          max_sop + max_data);
 912         }
 913         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 914                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 915
 916         /* Allocate sop queue resources */
 917         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 918                 nb_sop_desc, sizeof(struct rq_enet_desc));
 919         if (rc) {
 920                 dev_err(enic, "error in allocation of sop rq\n");
 921                 goto err_exit;
 922         }
 923         nb_sop_desc = rq_sop->ring.desc_count;
 924
 925         if (rq_data->in_use) {
 926                 /* Allocate data queue resources */
 927                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 928                                    nb_data_desc,
 929                                    sizeof(struct rq_enet_desc));
 930                 if (rc) {
 931                         dev_err(enic, "error in allocation of data rq\n");
 932                         goto err_free_rq_sop;
 933                 }
 934                 nb_data_desc = rq_data->ring.desc_count;
 935         }
 936         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 937                            socket_id, nb_sop_desc + nb_data_desc,
 938                            sizeof(struct cq_enet_rq_desc));
 939         if (rc) {
 940                 dev_err(enic, "error in allocation of cq for rq\n");
 941                 goto err_free_rq_data;
 942         }
 943
 944         /* Allocate the mbuf rings */
 945         rq_sop->mbuf_ring = (struct rte_mbuf **)
 946                 rte_zmalloc_socket("rq->mbuf_ring",
 947                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 948                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 949         if (rq_sop->mbuf_ring == NULL)
 950                 goto err_free_cq;
 951
 952         if (rq_data->in_use) {
 953                 rq_data->mbuf_ring = (struct rte_mbuf **)
 954                         rte_zmalloc_socket("rq->mbuf_ring",
 955                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 956                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 957                 if (rq_data->mbuf_ring == NULL)
 958                         goto err_free_sop_mbuf;
 959         }
 960
 961         rq_sop->free_mbufs = (struct rte_mbuf **)
 962                 rte_zmalloc_socket("rq->free_mbufs",
 963                                    sizeof(struct rte_mbuf *) *
 964                                    ENIC_RX_BURST_MAX,
 965                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 966         if (rq_sop->free_mbufs == NULL)
 967                 goto err_free_data_mbuf;
 968         rq_sop->num_free_mbufs = 0;
 969
 970         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 971
 972         return 0;
 973
 974 err_free_data_mbuf:
 975         rte_free(rq_data->mbuf_ring);
 976 err_free_sop_mbuf:
 977         rte_free(rq_sop->mbuf_ring);
 978 err_free_cq:
 979         /* cleanup on error */
 980         vnic_cq_free(&enic->cq[queue_idx]);
 981 err_free_rq_data:
 982         if (rq_data->in_use)
 983                 vnic_rq_free(rq_data);
 984 err_free_rq_sop:
 985         vnic_rq_free(rq_sop);
 986 err_exit:
 987         return -ENOMEM;
 988 }
 989
 990 void enic_free_wq(void *txq)
 991 {
 992         struct vnic_wq *wq;
 993         struct enic *enic;
 994
 995         if (txq == NULL)
 996                 return;
 997
 998         wq = (struct vnic_wq *)txq;
 999         enic = vnic_dev_priv(wq->vdev);
1000         rte_memzone_free(wq->cqmsg_rz);
1001         vnic_wq_free(wq);
1002         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1003 }
1004
1005 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1006         unsigned int socket_id, uint16_t nb_desc)
1007 {
1008         int err;
1009         struct vnic_wq *wq = &enic->wq[queue_idx];
1010         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
1011         char name[NAME_MAX];
1012         static int instance;
1013
1014         wq->socket_id = socket_id;
1015         /*
1016          * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1017          * print an info message for diagnostics.
1018          */
1019         dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1020
1021         /* Allocate queue resources */
1022         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1023                 nb_desc,
1024                 sizeof(struct wq_enet_desc));
1025         if (err) {
1026                 dev_err(enic, "error in allocation of wq\n");
1027                 return err;
1028         }
1029
1030         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1031                 socket_id, nb_desc,
1032                 sizeof(struct cq_enet_wq_desc));
1033         if (err) {
1034                 vnic_wq_free(wq);
1035                 dev_err(enic, "error in allocation of cq for wq\n");
1036         }
1037
1038         /* setup up CQ message */
1039         snprintf((char *)name, sizeof(name),
1040                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1041                 instance++);
1042
1043         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1044                         sizeof(uint32_t), SOCKET_ID_ANY,
1045                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1046         if (!wq->cqmsg_rz)
1047                 return -ENOMEM;
1048
1049         return err;
1050 }
1051
1052 int enic_disable(struct enic *enic)
1053 {
1054         unsigned int i;
1055         int err;
1056
1057         for (i = 0; i < enic->intr_count; i++) {
1058                 vnic_intr_mask(&enic->intr[i]);
1059                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1060         }
1061         enic_rxq_intr_deinit(enic);
1062         rte_intr_disable(&enic->pdev->intr_handle);
1063         rte_intr_callback_unregister(&enic->pdev->intr_handle,
1064                                      enic_intr_handler,
1065                                      (void *)enic->rte_dev);
1066
1067         vnic_dev_disable(enic->vdev);
1068
1069         enic_clsf_destroy(enic);
1070         enic_fm_destroy(enic);
1071
1072         if (!enic_is_sriov_vf(enic))
1073                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1074
1075         for (i = 0; i < enic->wq_count; i++) {
1076                 err = vnic_wq_disable(&enic->wq[i]);
1077                 if (err)
1078                         return err;
1079         }
1080         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1081                 if (enic->rq[i].in_use) {
1082                         err = vnic_rq_disable(&enic->rq[i]);
1083                         if (err)
1084                                 return err;
1085                 }
1086         }
1087
1088         /* If we were using interrupts, set the interrupt vector to -1
1089          * to disable interrupts.  We are not disabling link notifcations,
1090          * though, as we want the polling of link status to continue working.
1091          */
1092         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1093                 vnic_dev_notify_set(enic->vdev, -1);
1094
1095         vnic_dev_set_reset_flag(enic->vdev, 1);
1096
1097         for (i = 0; i < enic->wq_count; i++)
1098                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1099
1100         for (i = 0; i < enic_vnic_rq_count(enic); i++)
1101                 if (enic->rq[i].in_use)
1102                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1103         for (i = 0; i < enic->cq_count; i++)
1104                 vnic_cq_clean(&enic->cq[i]);
1105         for (i = 0; i < enic->intr_count; i++)
1106                 vnic_intr_clean(&enic->intr[i]);
1107
1108         return 0;
1109 }
1110
1111 static int enic_dev_wait(struct vnic_dev *vdev,
1112         int (*start)(struct vnic_dev *, int),
1113         int (*finished)(struct vnic_dev *, int *),
1114         int arg)
1115 {
1116         int done;
1117         int err;
1118         int i;
1119
1120         err = start(vdev, arg);
1121         if (err)
1122                 return err;
1123
1124         /* Wait for func to complete...2 seconds max */
1125         for (i = 0; i < 2000; i++) {
1126                 err = finished(vdev, &done);
1127                 if (err)
1128                         return err;
1129                 if (done)
1130                         return 0;
1131                 usleep(1000);
1132         }
1133         return -ETIMEDOUT;
1134 }
1135
1136 static int enic_dev_open(struct enic *enic)
1137 {
1138         int err;
1139         int flags = CMD_OPENF_IG_DESCCACHE;
1140
1141         err = enic_dev_wait(enic->vdev, vnic_dev_open,
1142                 vnic_dev_open_done, flags);
1143         if (err)
1144                 dev_err(enic_get_dev(enic),
1145                         "vNIC device open failed, err %d\n", err);
1146
1147         return err;
1148 }
1149
1150 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1151 {
1152         dma_addr_t rss_key_buf_pa;
1153         union vnic_rss_key *rss_key_buf_va = NULL;
1154         int err, i;
1155         u8 name[NAME_MAX];
1156
1157         RTE_ASSERT(user_key != NULL);
1158         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1159         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1160                 &rss_key_buf_pa, name);
1161         if (!rss_key_buf_va)
1162                 return -ENOMEM;
1163
1164         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1165                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1166
1167         err = enic_set_rss_key(enic,
1168                 rss_key_buf_pa,
1169                 sizeof(union vnic_rss_key));
1170
1171         /* Save for later queries */
1172         if (!err) {
1173                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1174                            sizeof(union vnic_rss_key));
1175         }
1176         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1177                 rss_key_buf_va, rss_key_buf_pa);
1178
1179         return err;
1180 }
1181
1182 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1183 {
1184         dma_addr_t rss_cpu_buf_pa;
1185         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1186         int err;
1187         u8 name[NAME_MAX];
1188
1189         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1190         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1191                 &rss_cpu_buf_pa, name);
1192         if (!rss_cpu_buf_va)
1193                 return -ENOMEM;
1194
1195         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1196
1197         err = enic_set_rss_cpu(enic,
1198                 rss_cpu_buf_pa,
1199                 sizeof(union vnic_rss_cpu));
1200
1201         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1202                 rss_cpu_buf_va, rss_cpu_buf_pa);
1203
1204         /* Save for later queries */
1205         if (!err)
1206                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1207         return err;
1208 }
1209
1210 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1211         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1212 {
1213         const u8 tso_ipid_split_en = 0;
1214         int err;
1215
1216         err = enic_set_nic_cfg(enic,
1217                 rss_default_cpu, rss_hash_type,
1218                 rss_hash_bits, rss_base_cpu,
1219                 rss_enable, tso_ipid_split_en,
1220                 enic->ig_vlan_strip_en);
1221
1222         return err;
1223 }
1224
1225 /* Initialize RSS with defaults, called from dev_configure */
1226 int enic_init_rss_nic_cfg(struct enic *enic)
1227 {
1228         static uint8_t default_rss_key[] = {
1229                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1230                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1231                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1232                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1233         };
1234         struct rte_eth_rss_conf rss_conf;
1235         union vnic_rss_cpu rss_cpu;
1236         int ret, i;
1237
1238         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1239         /*
1240          * If setting key for the first time, and the user gives us none, then
1241          * push the default key to NIC.
1242          */
1243         if (rss_conf.rss_key == NULL) {
1244                 rss_conf.rss_key = default_rss_key;
1245                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1246         }
1247         ret = enic_set_rss_conf(enic, &rss_conf);
1248         if (ret) {
1249                 dev_err(enic, "Failed to configure RSS\n");
1250                 return ret;
1251         }
1252         if (enic->rss_enable) {
1253                 /* If enabling RSS, use the default reta */
1254                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1255                         rss_cpu.cpu[i / 4].b[i % 4] =
1256                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1257                 }
1258                 ret = enic_set_rss_reta(enic, &rss_cpu);
1259                 if (ret)
1260                         dev_err(enic, "Failed to set RSS indirection table\n");
1261         }
1262         return ret;
1263 }
1264
1265 int enic_setup_finish(struct enic *enic)
1266 {
1267         enic_init_soft_stats(enic);
1268
1269         /* Default conf */
1270         vnic_dev_packet_filter(enic->vdev,
1271                 1 /* directed  */,
1272                 1 /* multicast */,
1273                 1 /* broadcast */,
1274                 0 /* promisc   */,
1275                 1 /* allmulti  */);
1276
1277         enic->promisc = 0;
1278         enic->allmulti = 1;
1279
1280         return 0;
1281 }
1282
1283 static int enic_rss_conf_valid(struct enic *enic,
1284                                struct rte_eth_rss_conf *rss_conf)
1285 {
1286         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1287         if (enic->flow_type_rss_offloads == 0)
1288                 return 0;
1289         if (rss_conf->rss_key != NULL &&
1290             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1291                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1292                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1293                 return -EINVAL;
1294         }
1295         if (rss_conf->rss_hf != 0 &&
1296             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1297                 dev_err(enic, "Given rss_hf contains none of the supported"
1298                         " types\n");
1299                 return -EINVAL;
1300         }
1301         return 0;
1302 }
1303
1304 /* Set hash type and key according to rss_conf */
1305 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1306 {
1307         struct rte_eth_dev *eth_dev;
1308         uint64_t rss_hf;
1309         u8 rss_hash_type;
1310         u8 rss_enable;
1311         int ret;
1312
1313         RTE_ASSERT(rss_conf != NULL);
1314         ret = enic_rss_conf_valid(enic, rss_conf);
1315         if (ret) {
1316                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1317                 return ret;
1318         }
1319
1320         eth_dev = enic->rte_dev;
1321         rss_hash_type = 0;
1322         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1323         if (enic->rq_count > 1 &&
1324             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1325             rss_hf != 0) {
1326                 rss_enable = 1;
1327                 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1328                               ETH_RSS_NONFRAG_IPV4_OTHER))
1329                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1330                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1331                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1332                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1333                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1334                         if (enic->udp_rss_weak) {
1335                                 /*
1336                                  * 'TCP' is not a typo. The "weak" version of
1337                                  * UDP RSS requires both the TCP and UDP bits
1338                                  * be set. It does enable TCP RSS as well.
1339                                  */
1340                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1341                         }
1342                 }
1343                 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1344                               ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1345                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1346                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1347                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1348                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1349                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1350                         if (enic->udp_rss_weak)
1351                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1352                 }
1353         } else {
1354                 rss_enable = 0;
1355                 rss_hf = 0;
1356         }
1357
1358         /* Set the hash key if provided */
1359         if (rss_enable && rss_conf->rss_key) {
1360                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1361                 if (ret) {
1362                         dev_err(enic, "Failed to set RSS key\n");
1363                         return ret;
1364                 }
1365         }
1366
1367         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1368                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1369                               rss_enable);
1370         if (!ret) {
1371                 enic->rss_hf = rss_hf;
1372                 enic->rss_hash_type = rss_hash_type;
1373                 enic->rss_enable = rss_enable;
1374         } else {
1375                 dev_err(enic, "Failed to update RSS configurations."
1376                         " hash=0x%x\n", rss_hash_type);
1377         }
1378         return ret;
1379 }
1380
1381 int enic_set_vlan_strip(struct enic *enic)
1382 {
1383         /*
1384          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1385          * together. So, re-do niccfg, preserving the current RSS settings.
1386          */
1387         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1388                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1389                                enic->rss_enable);
1390 }
1391
1392 int enic_add_packet_filter(struct enic *enic)
1393 {
1394         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1395         return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1396                 enic->promisc, enic->allmulti);
1397 }
1398
1399 int enic_get_link_status(struct enic *enic)
1400 {
1401         return vnic_dev_link_status(enic->vdev);
1402 }
1403
1404 static void enic_dev_deinit(struct enic *enic)
1405 {
1406         /* stop link status checking */
1407         vnic_dev_notify_unset(enic->vdev);
1408
1409         /* mac_addrs is freed by rte_eth_dev_release_port() */
1410         rte_free(enic->cq);
1411         rte_free(enic->intr);
1412         rte_free(enic->rq);
1413         rte_free(enic->wq);
1414 }
1415
1416
1417 int enic_set_vnic_res(struct enic *enic)
1418 {
1419         struct rte_eth_dev *eth_dev = enic->rte_dev;
1420         int rc = 0;
1421         unsigned int required_rq, required_wq, required_cq, required_intr;
1422
1423         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1424         required_rq = eth_dev->data->nb_rx_queues * 2;
1425         required_wq = eth_dev->data->nb_tx_queues;
1426         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1427         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1428         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1429                 required_intr += eth_dev->data->nb_rx_queues;
1430         }
1431
1432         if (enic->conf_rq_count < required_rq) {
1433                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1434                         eth_dev->data->nb_rx_queues,
1435                         required_rq, enic->conf_rq_count);
1436                 rc = -EINVAL;
1437         }
1438         if (enic->conf_wq_count < required_wq) {
1439                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1440                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1441                 rc = -EINVAL;
1442         }
1443
1444         if (enic->conf_cq_count < required_cq) {
1445                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1446                         required_cq, enic->conf_cq_count);
1447                 rc = -EINVAL;
1448         }
1449         if (enic->conf_intr_count < required_intr) {
1450                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1451                         " interrupts. Required:%u, Configured:%u\n",
1452                         required_intr, enic->conf_intr_count);
1453                 rc = -EINVAL;
1454         }
1455
1456         if (rc == 0) {
1457                 enic->rq_count = eth_dev->data->nb_rx_queues;
1458                 enic->wq_count = eth_dev->data->nb_tx_queues;
1459                 enic->cq_count = enic->rq_count + enic->wq_count;
1460                 enic->intr_count = required_intr;
1461         }
1462
1463         return rc;
1464 }
1465
1466 /* Initialize the completion queue for an RQ */
1467 static int
1468 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1469 {
1470         struct vnic_rq *sop_rq, *data_rq;
1471         unsigned int cq_idx;
1472         int rc = 0;
1473
1474         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1475         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1476         cq_idx = rq_idx;
1477
1478         vnic_cq_clean(&enic->cq[cq_idx]);
1479         vnic_cq_init(&enic->cq[cq_idx],
1480                      0 /* flow_control_enable */,
1481                      1 /* color_enable */,
1482                      0 /* cq_head */,
1483                      0 /* cq_tail */,
1484                      1 /* cq_tail_color */,
1485                      0 /* interrupt_enable */,
1486                      1 /* cq_entry_enable */,
1487                      0 /* cq_message_enable */,
1488                      0 /* interrupt offset */,
1489                      0 /* cq_message_addr */);
1490
1491
1492         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1493                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1494                            sop_rq->ring.desc_count - 1, 1, 0);
1495         if (data_rq->in_use) {
1496                 vnic_rq_init_start(data_rq,
1497                                    enic_cq_rq(enic,
1498                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1499                                    data_rq->ring.desc_count - 1, 1, 0);
1500         }
1501
1502         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1503         if (rc)
1504                 return rc;
1505
1506         if (data_rq->in_use) {
1507                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1508                 if (rc) {
1509                         enic_rxmbuf_queue_release(enic, sop_rq);
1510                         return rc;
1511                 }
1512         }
1513
1514         return 0;
1515 }
1516
1517 /* The Cisco NIC can send and receive packets up to a max packet size
1518  * determined by the NIC type and firmware. There is also an MTU
1519  * configured into the NIC via the CIMC/UCSM management interface
1520  * which can be overridden by this function (up to the max packet size).
1521  * Depending on the network setup, doing so may cause packet drops
1522  * and unexpected behavior.
1523  */
1524 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1525 {
1526         unsigned int rq_idx;
1527         struct vnic_rq *rq;
1528         int rc = 0;
1529         uint16_t old_mtu;       /* previous setting */
1530         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1531         struct rte_eth_dev *eth_dev = enic->rte_dev;
1532
1533         old_mtu = eth_dev->data->mtu;
1534         config_mtu = enic->config.mtu;
1535
1536         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1537                 return -E_RTE_SECONDARY;
1538
1539         if (new_mtu > enic->max_mtu) {
1540                 dev_err(enic,
1541                         "MTU not updated: requested (%u) greater than max (%u)\n",
1542                         new_mtu, enic->max_mtu);
1543                 return -EINVAL;
1544         }
1545         if (new_mtu < ENIC_MIN_MTU) {
1546                 dev_info(enic,
1547                         "MTU not updated: requested (%u) less than min (%u)\n",
1548                         new_mtu, ENIC_MIN_MTU);
1549                 return -EINVAL;
1550         }
1551         if (new_mtu > config_mtu)
1552                 dev_warning(enic,
1553                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1554                         new_mtu, config_mtu);
1555
1556         /* Update the MTU and maximum packet length */
1557         eth_dev->data->mtu = new_mtu;
1558         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1559                 enic_mtu_to_max_rx_pktlen(new_mtu);
1560
1561         /*
1562          * If the device has not started (enic_enable), nothing to do.
1563          * Later, enic_enable() will set up RQs reflecting the new maximum
1564          * packet length.
1565          */
1566         if (!eth_dev->data->dev_started)
1567                 goto set_mtu_done;
1568
1569         /*
1570          * The device has started, re-do RQs on the fly. In the process, we
1571          * pick up the new maximum packet length.
1572          *
1573          * Some applications rely on the ability to change MTU without stopping
1574          * the device. So keep this behavior for now.
1575          */
1576         rte_spinlock_lock(&enic->mtu_lock);
1577
1578         /* Stop traffic on all RQs */
1579         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1580                 rq = &enic->rq[rq_idx];
1581                 if (rq->is_sop && rq->in_use) {
1582                         rc = enic_stop_rq(enic,
1583                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1584                         if (rc) {
1585                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1586                                 goto set_mtu_done;
1587                         }
1588                 }
1589         }
1590
1591         /* replace Rx function with a no-op to avoid getting stale pkts */
1592         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1593         rte_mb();
1594
1595         /* Allow time for threads to exit the real Rx function. */
1596         usleep(100000);
1597
1598         /* now it is safe to reconfigure the RQs */
1599
1600
1601         /* free and reallocate RQs with the new MTU */
1602         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1603                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1604                 if (!rq->in_use)
1605                         continue;
1606
1607                 enic_free_rq(rq);
1608                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1609                                    rq->tot_nb_desc, rq->rx_free_thresh);
1610                 if (rc) {
1611                         dev_err(enic,
1612                                 "Fatal MTU alloc error- No traffic will pass\n");
1613                         goto set_mtu_done;
1614                 }
1615
1616                 rc = enic_reinit_rq(enic, rq_idx);
1617                 if (rc) {
1618                         dev_err(enic,
1619                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1620                         goto set_mtu_done;
1621                 }
1622         }
1623
1624         /* put back the real receive function */
1625         rte_mb();
1626         enic_pick_rx_handler(eth_dev);
1627         rte_mb();
1628
1629         /* restart Rx traffic */
1630         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1631                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1632                 if (rq->is_sop && rq->in_use)
1633                         enic_start_rq(enic, rq_idx);
1634         }
1635
1636 set_mtu_done:
1637         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1638         rte_spinlock_unlock(&enic->mtu_lock);
1639         return rc;
1640 }
1641
1642 static int enic_dev_init(struct enic *enic)
1643 {
1644         int err;
1645         struct rte_eth_dev *eth_dev = enic->rte_dev;
1646
1647         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1648
1649         /* Get vNIC configuration
1650         */
1651         err = enic_get_vnic_config(enic);
1652         if (err) {
1653                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1654                 return err;
1655         }
1656
1657         /* Get available resource counts */
1658         enic_get_res_counts(enic);
1659         if (enic->conf_rq_count == 1) {
1660                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1661                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1662                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1663                 return -EINVAL;
1664         }
1665         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1666         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1667                                enic->conf_cq_count, 8);
1668         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1669                                  enic->conf_intr_count, 8);
1670         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1671                                enic->conf_rq_count, 8);
1672         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1673                                enic->conf_wq_count, 8);
1674         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1675                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1676                 return -1;
1677         }
1678         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1679                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1680                 return -1;
1681         }
1682         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1683                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1684                 return -1;
1685         }
1686         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1687                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1688                 return -1;
1689         }
1690
1691         /* Get the supported filters */
1692         enic_fdir_info(enic);
1693
1694         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1695                                         sizeof(struct rte_ether_addr) *
1696                                         ENIC_UNICAST_PERFECT_FILTERS, 0);
1697         if (!eth_dev->data->mac_addrs) {
1698                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1699                 return -1;
1700         }
1701         rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1702                         eth_dev->data->mac_addrs);
1703
1704         vnic_dev_set_reset_flag(enic->vdev, 0);
1705
1706         LIST_INIT(&enic->flows);
1707
1708         /* set up link status checking */
1709         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1710
1711         /*
1712          * When Geneve with options offload is available, always disable it
1713          * first as it can interfere with user flow rules.
1714          */
1715         if (enic->geneve_opt_avail &&
1716             vnic_dev_overlay_offload_ctrl(enic->vdev,
1717                         OVERLAY_FEATURE_GENEVE,
1718                         OVERLAY_OFFLOAD_DISABLE)) {
1719                 dev_err(enic, "failed to disable geneve+option\n");
1720         }
1721         enic->overlay_offload = false;
1722         if (enic->disable_overlay && enic->vxlan) {
1723                 /*
1724                  * Explicitly disable overlay offload as the setting is
1725                  * sticky, and resetting vNIC does not disable it.
1726                  */
1727                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1728                                                   OVERLAY_FEATURE_VXLAN,
1729                                                   OVERLAY_OFFLOAD_DISABLE)) {
1730                         dev_err(enic, "failed to disable overlay offload\n");
1731                 } else {
1732                         dev_info(enic, "Overlay offload is disabled\n");
1733                 }
1734         }
1735         if (!enic->disable_overlay && enic->vxlan &&
1736             /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1737             vnic_dev_overlay_offload_ctrl(enic->vdev,
1738                                           OVERLAY_FEATURE_VXLAN,
1739                                           OVERLAY_OFFLOAD_ENABLE) == 0) {
1740                 enic->tx_offload_capa |=
1741                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1742                         DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1743                         DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1744                 enic->tx_offload_mask |=
1745                         PKT_TX_OUTER_IPV6 |
1746                         PKT_TX_OUTER_IPV4 |
1747                         PKT_TX_OUTER_IP_CKSUM |
1748                         PKT_TX_TUNNEL_MASK;
1749                 enic->overlay_offload = true;
1750                 dev_info(enic, "Overlay offload is enabled\n");
1751         }
1752         /* Geneve with options offload requires overlay offload */
1753         if (enic->overlay_offload && enic->geneve_opt_avail &&
1754             enic->geneve_opt_request) {
1755                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1756                                 OVERLAY_FEATURE_GENEVE,
1757                                 OVERLAY_OFFLOAD_ENABLE)) {
1758                         dev_err(enic, "failed to enable geneve+option\n");
1759                 } else {
1760                         enic->geneve_opt_enabled = 1;
1761                         dev_info(enic, "Geneve with options is enabled\n");
1762                 }
1763         }
1764         /*
1765          * Reset the vxlan port if HW vxlan parsing is available. It
1766          * is always enabled regardless of overlay offload
1767          * enable/disable.
1768          */
1769         if (enic->vxlan) {
1770                 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1771                 /*
1772                  * Reset the vxlan port to the default, as the NIC firmware
1773                  * does not reset it automatically and keeps the old setting.
1774                  */
1775                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1776                                                  OVERLAY_CFG_VXLAN_PORT_UPDATE,
1777                                                  ENIC_DEFAULT_VXLAN_PORT)) {
1778                         dev_err(enic, "failed to update vxlan port\n");
1779                         return -EINVAL;
1780                 }
1781         }
1782
1783         return 0;
1784
1785 }
1786
1787 int enic_probe(struct enic *enic)
1788 {
1789         struct rte_pci_device *pdev = enic->pdev;
1790         int err = -1;
1791
1792         dev_debug(enic, "Initializing ENIC PMD\n");
1793
1794         /* if this is a secondary process the hardware is already initialized */
1795         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1796                 return 0;
1797
1798         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1799         enic->bar0.len = pdev->mem_resource[0].len;
1800
1801         /* Register vNIC device */
1802         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1803         if (!enic->vdev) {
1804                 dev_err(enic, "vNIC registration failed, aborting\n");
1805                 goto err_out;
1806         }
1807
1808         LIST_INIT(&enic->memzone_list);
1809         rte_spinlock_init(&enic->memzone_list_lock);
1810
1811         vnic_register_cbacks(enic->vdev,
1812                 enic_alloc_consistent,
1813                 enic_free_consistent);
1814
1815         /*
1816          * Allocate the consistent memory for stats upfront so both primary and
1817          * secondary processes can dump stats.
1818          */
1819         err = vnic_dev_alloc_stats_mem(enic->vdev);
1820         if (err) {
1821                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1822                 goto err_out_unregister;
1823         }
1824         /* Issue device open to get device in known state */
1825         err = enic_dev_open(enic);
1826         if (err) {
1827                 dev_err(enic, "vNIC dev open failed, aborting\n");
1828                 goto err_out_unregister;
1829         }
1830
1831         /* Set ingress vlan rewrite mode before vnic initialization */
1832         dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1833                   enic->ig_vlan_rewrite_mode);
1834         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1835                 enic->ig_vlan_rewrite_mode);
1836         if (err) {
1837                 dev_err(enic,
1838                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1839                 goto err_out_dev_close;
1840         }
1841
1842         /* Issue device init to initialize the vnic-to-switch link.
1843          * We'll start with carrier off and wait for link UP
1844          * notification later to turn on carrier.  We don't need
1845          * to wait here for the vnic-to-switch link initialization
1846          * to complete; link UP notification is the indication that
1847          * the process is complete.
1848          */
1849
1850         err = vnic_dev_init(enic->vdev, 0);
1851         if (err) {
1852                 dev_err(enic, "vNIC dev init failed, aborting\n");
1853                 goto err_out_dev_close;
1854         }
1855
1856         err = enic_dev_init(enic);
1857         if (err) {
1858                 dev_err(enic, "Device initialization failed, aborting\n");
1859                 goto err_out_dev_close;
1860         }
1861
1862         return 0;
1863
1864 err_out_dev_close:
1865         vnic_dev_close(enic->vdev);
1866 err_out_unregister:
1867         vnic_dev_unregister(enic->vdev);
1868 err_out:
1869         return err;
1870 }
1871
1872 void enic_remove(struct enic *enic)
1873 {
1874         enic_dev_deinit(enic);
1875         vnic_dev_close(enic->vdev);
1876         vnic_dev_unregister(enic->vdev);
1877 }