drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11
  12 #include <rte_pci.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_string_fns.h>
  18 #include <ethdev_driver.h>
  19 #include <rte_geneve.h>
  20
  21 #include "enic_compat.h"
  22 #include "enic.h"
  23 #include "wq_enet_desc.h"
  24 #include "rq_enet_desc.h"
  25 #include "cq_enet_desc.h"
  26 #include "vnic_enet.h"
  27 #include "vnic_dev.h"
  28 #include "vnic_wq.h"
  29 #include "vnic_rq.h"
  30 #include "vnic_cq.h"
  31 #include "vnic_intr.h"
  32 #include "vnic_nic.h"
  33
  34 static inline int enic_is_sriov_vf(struct enic *enic)
  35 {
  36         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  37 }
  38
  39 static int is_zero_addr(uint8_t *addr)
  40 {
  41         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  42 }
  43
  44 static int is_mcast_addr(uint8_t *addr)
  45 {
  46         return addr[0] & 1;
  47 }
  48
  49 static int is_eth_addr_valid(uint8_t *addr)
  50 {
  51         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  52 }
  53
  54 void
  55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  56 {
  57         uint16_t i;
  58
  59         if (!rq || !rq->mbuf_ring) {
  60                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  61                 return;
  62         }
  63
  64         for (i = 0; i < rq->ring.desc_count; i++) {
  65                 if (rq->mbuf_ring[i]) {
  66                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  67                         rq->mbuf_ring[i] = NULL;
  68                 }
  69         }
  70 }
  71
  72 void enic_free_wq_buf(struct rte_mbuf **buf)
  73 {
  74         struct rte_mbuf *mbuf = *buf;
  75
  76         rte_pktmbuf_free_seg(mbuf);
  77         *buf = NULL;
  78 }
  79
  80 static void enic_log_q_error(struct enic *enic)
  81 {
  82         unsigned int i;
  83         uint32_t error_status;
  84
  85         for (i = 0; i < enic->wq_count; i++) {
  86                 error_status = vnic_wq_error_status(&enic->wq[i]);
  87                 if (error_status)
  88                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  89                                 error_status);
  90         }
  91
  92         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  93                 if (!enic->rq[i].in_use)
  94                         continue;
  95                 error_status = vnic_rq_error_status(&enic->rq[i]);
  96                 if (error_status)
  97                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  98                                 error_status);
  99         }
 100 }
 101
 102 static void enic_clear_soft_stats(struct enic *enic)
 103 {
 104         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 105         rte_atomic64_clear(&soft_stats->rx_nombuf);
 106         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 107         rte_atomic64_clear(&soft_stats->tx_oversized);
 108 }
 109
 110 static void enic_init_soft_stats(struct enic *enic)
 111 {
 112         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 113         rte_atomic64_init(&soft_stats->rx_nombuf);
 114         rte_atomic64_init(&soft_stats->rx_packet_errors);
 115         rte_atomic64_init(&soft_stats->tx_oversized);
 116         enic_clear_soft_stats(enic);
 117 }
 118
 119 int enic_dev_stats_clear(struct enic *enic)
 120 {
 121         int ret;
 122
 123         ret = vnic_dev_stats_clear(enic->vdev);
 124         if (ret != 0) {
 125                 dev_err(enic, "Error in clearing stats\n");
 126                 return ret;
 127         }
 128         enic_clear_soft_stats(enic);
 129
 130         return 0;
 131 }
 132
 133 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 134 {
 135         struct vnic_stats *stats;
 136         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 137         int64_t rx_truncated;
 138         uint64_t rx_packet_errors;
 139         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 140
 141         if (ret) {
 142                 dev_err(enic, "Error in getting stats\n");
 143                 return ret;
 144         }
 145
 146         /* The number of truncated packets can only be calculated by
 147          * subtracting a hardware counter from error packets received by
 148          * the driver. Note: this causes transient inaccuracies in the
 149          * ipackets count. Also, the length of truncated packets are
 150          * counted in ibytes even though truncated packets are dropped
 151          * which can make ibytes be slightly higher than it should be.
 152          */
 153         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 154         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 155
 156         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 157         r_stats->opackets = stats->tx.tx_frames_ok;
 158
 159         r_stats->ibytes = stats->rx.rx_bytes_ok;
 160         r_stats->obytes = stats->tx.tx_bytes_ok;
 161
 162         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 163         r_stats->oerrors = stats->tx.tx_errors
 164                            + rte_atomic64_read(&soft_stats->tx_oversized);
 165
 166         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 167
 168         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 169         return 0;
 170 }
 171
 172 int enic_del_mac_address(struct enic *enic, int mac_index)
 173 {
 174         struct rte_eth_dev *eth_dev = enic->rte_dev;
 175         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 176
 177         return vnic_dev_del_addr(enic->vdev, mac_addr);
 178 }
 179
 180 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 181 {
 182         int err;
 183
 184         if (!is_eth_addr_valid(mac_addr)) {
 185                 dev_err(enic, "invalid mac address\n");
 186                 return -EINVAL;
 187         }
 188
 189         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 190         if (err)
 191                 dev_err(enic, "add mac addr failed\n");
 192         return err;
 193 }
 194
 195 void enic_free_rq_buf(struct rte_mbuf **mbuf)
 196 {
 197         if (*mbuf == NULL)
 198                 return;
 199
 200         rte_pktmbuf_free(*mbuf);
 201         *mbuf = NULL;
 202 }
 203
 204 void enic_init_vnic_resources(struct enic *enic)
 205 {
 206         unsigned int error_interrupt_enable = 1;
 207         unsigned int error_interrupt_offset = 0;
 208         unsigned int rxq_interrupt_enable = 0;
 209         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 210         unsigned int index = 0;
 211         unsigned int cq_idx;
 212         struct vnic_rq *data_rq;
 213
 214         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 215                 rxq_interrupt_enable = 1;
 216
 217         for (index = 0; index < enic->rq_count; index++) {
 218                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 219
 220                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 221                         cq_idx,
 222                         error_interrupt_enable,
 223                         error_interrupt_offset);
 224
 225                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index, enic)];
 226                 if (data_rq->in_use)
 227                         vnic_rq_init(data_rq,
 228                                      cq_idx,
 229                                      error_interrupt_enable,
 230                                      error_interrupt_offset);
 231                 vnic_cq_init(&enic->cq[cq_idx],
 232                         0 /* flow_control_enable */,
 233                         1 /* color_enable */,
 234                         0 /* cq_head */,
 235                         0 /* cq_tail */,
 236                         1 /* cq_tail_color */,
 237                         rxq_interrupt_enable,
 238                         1 /* cq_entry_enable */,
 239                         0 /* cq_message_enable */,
 240                         rxq_interrupt_offset,
 241                         0 /* cq_message_addr */);
 242                 if (rxq_interrupt_enable)
 243                         rxq_interrupt_offset++;
 244         }
 245
 246         for (index = 0; index < enic->wq_count; index++) {
 247                 vnic_wq_init(&enic->wq[index],
 248                         enic_cq_wq(enic, index),
 249                         error_interrupt_enable,
 250                         error_interrupt_offset);
 251                 /* Compute unsupported ol flags for enic_prep_pkts() */
 252                 enic->wq[index].tx_offload_notsup_mask =
 253                         RTE_MBUF_F_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 254
 255                 cq_idx = enic_cq_wq(enic, index);
 256                 vnic_cq_init(&enic->cq[cq_idx],
 257                         0 /* flow_control_enable */,
 258                         1 /* color_enable */,
 259                         0 /* cq_head */,
 260                         0 /* cq_tail */,
 261                         1 /* cq_tail_color */,
 262                         0 /* interrupt_enable */,
 263                         0 /* cq_entry_enable */,
 264                         1 /* cq_message_enable */,
 265                         0 /* interrupt offset */,
 266                         (uint64_t)enic->wq[index].cqmsg_rz->iova);
 267         }
 268
 269         for (index = 0; index < enic->intr_count; index++) {
 270                 vnic_intr_init(&enic->intr[index],
 271                                enic->config.intr_timer_usec,
 272                                enic->config.intr_timer_type,
 273                                /*mask_on_assertion*/1);
 274         }
 275 }
 276
 277
 278 int
 279 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 280 {
 281         struct rte_mbuf *mb;
 282         struct rq_enet_desc *rqd = rq->ring.descs;
 283         unsigned i;
 284         dma_addr_t dma_addr;
 285         uint32_t max_rx_pktlen;
 286         uint16_t rq_buf_len;
 287
 288         if (!rq->in_use)
 289                 return 0;
 290
 291         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 292                   rq->ring.desc_count);
 293
 294         /*
 295          * If *not* using scatter and the mbuf size is greater than the
 296          * requested max packet size (mtu + eth overhead), then reduce the
 297          * posted buffer size to max packet size. HW still receives packets
 298          * larger than max packet size, but they will be truncated, which we
 299          * drop in the rx handler. Not ideal, but better than returning
 300          * large packets when the user is not expecting them.
 301          */
 302         max_rx_pktlen = enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu);
 303         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 304         if (max_rx_pktlen < rq_buf_len && !rq->data_queue_enable)
 305                 rq_buf_len = max_rx_pktlen;
 306         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 307                 mb = rte_mbuf_raw_alloc(rq->mp);
 308                 if (mb == NULL) {
 309                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 310                         (unsigned)rq->index);
 311                         return -ENOMEM;
 312                 }
 313
 314                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 315                 dma_addr = (dma_addr_t)(mb->buf_iova
 316                            + RTE_PKTMBUF_HEADROOM);
 317                 rq_enet_desc_enc(rqd, dma_addr,
 318                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 319                                 : RQ_ENET_TYPE_NOT_SOP),
 320                                 rq_buf_len);
 321                 rq->mbuf_ring[i] = mb;
 322         }
 323         /*
 324          * Do not post the buffers to the NIC until we enable the RQ via
 325          * enic_start_rq().
 326          */
 327         rq->need_initial_post = true;
 328         /* Initialize fetch index while RQ is disabled */
 329         iowrite32(0, &rq->ctrl->fetch_index);
 330         return 0;
 331 }
 332
 333 /*
 334  * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
 335  * allocated the buffers and filled the RQ descriptor ring. Just need to push
 336  * the post index to the NIC.
 337  */
 338 static void
 339 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 340 {
 341         if (!rq->in_use || !rq->need_initial_post)
 342                 return;
 343
 344         /* make sure all prior writes are complete before doing the PIO write */
 345         rte_rmb();
 346
 347         /* Post all but the last buffer to VIC. */
 348         rq->posted_index = rq->ring.desc_count - 1;
 349
 350         rq->rx_nb_hold = 0;
 351
 352         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 353                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 354         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 355         rte_rmb();
 356         rq->need_initial_post = false;
 357 }
 358
 359 void *
 360 enic_alloc_consistent(void *priv, size_t size,
 361         dma_addr_t *dma_handle, uint8_t *name)
 362 {
 363         void *vaddr;
 364         const struct rte_memzone *rz;
 365         *dma_handle = 0;
 366         struct enic *enic = (struct enic *)priv;
 367         struct enic_memzone_entry *mze;
 368
 369         rz = rte_memzone_reserve_aligned((const char *)name, size,
 370                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
 371         if (!rz) {
 372                 pr_err("%s : Failed to allocate memory requested for %s\n",
 373                         __func__, name);
 374                 return NULL;
 375         }
 376
 377         vaddr = rz->addr;
 378         *dma_handle = (dma_addr_t)rz->iova;
 379
 380         mze = rte_malloc("enic memzone entry",
 381                          sizeof(struct enic_memzone_entry), 0);
 382
 383         if (!mze) {
 384                 pr_err("%s : Failed to allocate memory for memzone list\n",
 385                        __func__);
 386                 rte_memzone_free(rz);
 387                 return NULL;
 388         }
 389
 390         mze->rz = rz;
 391
 392         rte_spinlock_lock(&enic->memzone_list_lock);
 393         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 394         rte_spinlock_unlock(&enic->memzone_list_lock);
 395
 396         return vaddr;
 397 }
 398
 399 void
 400 enic_free_consistent(void *priv,
 401                      __rte_unused size_t size,
 402                      void *vaddr,
 403                      dma_addr_t dma_handle)
 404 {
 405         struct enic_memzone_entry *mze;
 406         struct enic *enic = (struct enic *)priv;
 407
 408         rte_spinlock_lock(&enic->memzone_list_lock);
 409         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 410                 if (mze->rz->addr == vaddr &&
 411                     mze->rz->iova == dma_handle)
 412                         break;
 413         }
 414         if (mze == NULL) {
 415                 rte_spinlock_unlock(&enic->memzone_list_lock);
 416                 dev_warning(enic,
 417                             "Tried to free memory, but couldn't find it in the memzone list\n");
 418                 return;
 419         }
 420         LIST_REMOVE(mze, entries);
 421         rte_spinlock_unlock(&enic->memzone_list_lock);
 422         rte_memzone_free(mze->rz);
 423         rte_free(mze);
 424 }
 425
 426 int enic_link_update(struct rte_eth_dev *eth_dev)
 427 {
 428         struct enic *enic = pmd_priv(eth_dev);
 429         struct rte_eth_link link;
 430
 431         memset(&link, 0, sizeof(link));
 432         link.link_status = enic_get_link_status(enic);
 433         link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
 434         link.link_speed = vnic_dev_port_speed(enic->vdev);
 435
 436         return rte_eth_linkstatus_set(eth_dev, &link);
 437 }
 438
 439 static void
 440 enic_intr_handler(void *arg)
 441 {
 442         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 443         struct enic *enic = pmd_priv(dev);
 444
 445         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 446
 447         enic_link_update(dev);
 448         rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 449         enic_log_q_error(enic);
 450         /* Re-enable irq in case of INTx */
 451         rte_intr_ack(&enic->pdev->intr_handle);
 452 }
 453
 454 static int enic_rxq_intr_init(struct enic *enic)
 455 {
 456         struct rte_intr_handle *intr_handle;
 457         uint32_t rxq_intr_count, i;
 458         int err;
 459
 460         intr_handle = enic->rte_dev->intr_handle;
 461         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 462                 return 0;
 463         /*
 464          * Rx queue interrupts only work when we have MSI-X interrupts,
 465          * one per queue. Sharing one interrupt is technically
 466          * possible with VIC, but it is not worth the complications it brings.
 467          */
 468         if (!rte_intr_cap_multiple(intr_handle)) {
 469                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 470                         " (vfio-pci driver)\n");
 471                 return -ENOTSUP;
 472         }
 473         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 474         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 475         if (err) {
 476                 dev_err(enic, "Failed to enable event fds for Rx queue"
 477                         " interrupts\n");
 478                 return err;
 479         }
 480         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 481                                             rxq_intr_count * sizeof(int), 0);
 482         if (intr_handle->intr_vec == NULL) {
 483                 dev_err(enic, "Failed to allocate intr_vec\n");
 484                 return -ENOMEM;
 485         }
 486         for (i = 0; i < rxq_intr_count; i++)
 487                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 488         return 0;
 489 }
 490
 491 static void enic_rxq_intr_deinit(struct enic *enic)
 492 {
 493         struct rte_intr_handle *intr_handle;
 494
 495         intr_handle = enic->rte_dev->intr_handle;
 496         rte_intr_efd_disable(intr_handle);
 497         if (intr_handle->intr_vec != NULL) {
 498                 rte_free(intr_handle->intr_vec);
 499                 intr_handle->intr_vec = NULL;
 500         }
 501 }
 502
 503 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 504 {
 505         struct wq_enet_desc *desc;
 506         struct vnic_wq *wq;
 507         unsigned int i;
 508
 509         /*
 510          * Fill WQ descriptor fields that never change. Every descriptor is
 511          * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
 512          * descriptors (i.e. request one completion update every 32 packets).
 513          */
 514         wq = &enic->wq[queue_idx];
 515         desc = (struct wq_enet_desc *)wq->ring.descs;
 516         for (i = 0; i < wq->ring.desc_count; i++, desc++) {
 517                 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
 518                 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
 519                         desc->header_length_flags |=
 520                                 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
 521         }
 522 }
 523
 524 /*
 525  * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
 526  * used when that file is not compiled.
 527  */
 528 __rte_weak bool
 529 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
 530 {
 531         return false;
 532 }
 533
 534 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
 535 {
 536         struct enic *enic = pmd_priv(eth_dev);
 537
 538         if (enic->cq64) {
 539                 ENICPMD_LOG(DEBUG, " use the normal Rx handler for 64B CQ entry");
 540                 eth_dev->rx_pkt_burst = &enic_recv_pkts_64;
 541                 return;
 542         }
 543         /*
 544          * Preference order:
 545          * 1. The vectorized handler if possible and requested.
 546          * 2. The non-scatter, simplified handler if scatter Rx is not used.
 547          * 3. The default handler as a fallback.
 548          */
 549         if (enic_use_vector_rx_handler(eth_dev))
 550                 return;
 551         if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 552                 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
 553                 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
 554         } else {
 555                 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
 556                 eth_dev->rx_pkt_burst = &enic_recv_pkts;
 557         }
 558 }
 559
 560 /* Secondary process uses this to set the Tx handler */
 561 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
 562 {
 563         struct enic *enic = pmd_priv(eth_dev);
 564
 565         if (enic->use_simple_tx_handler) {
 566                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 567                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 568         } else {
 569                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 570                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 571         }
 572 }
 573
 574 int enic_enable(struct enic *enic)
 575 {
 576         unsigned int index;
 577         int err;
 578         struct rte_eth_dev *eth_dev = enic->rte_dev;
 579         uint64_t simple_tx_offloads;
 580         uintptr_t p;
 581
 582         if (enic->enable_avx2_rx) {
 583                 struct rte_mbuf mb_def = { .buf_addr = 0 };
 584
 585                 /*
 586                  * mbuf_initializer contains const-after-init fields of
 587                  * receive mbufs (i.e. 64 bits of fields from rearm_data).
 588                  * It is currently used by the vectorized handler.
 589                  */
 590                 mb_def.nb_segs = 1;
 591                 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 592                 mb_def.port = enic->port_id;
 593                 rte_mbuf_refcnt_set(&mb_def, 1);
 594                 rte_compiler_barrier();
 595                 p = (uintptr_t)&mb_def.rearm_data;
 596                 enic->mbuf_initializer = *(uint64_t *)p;
 597         }
 598
 599         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 600         eth_dev->data->dev_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
 601
 602         /* vnic notification of link status has already been turned on in
 603          * enic_dev_init() which is called during probe time.  Here we are
 604          * just turning on interrupt vector 0 if needed.
 605          */
 606         if (eth_dev->data->dev_conf.intr_conf.lsc)
 607                 vnic_dev_notify_set(enic->vdev, 0);
 608
 609         err = enic_rxq_intr_init(enic);
 610         if (err)
 611                 return err;
 612
 613         /* Initialize flowman if not already initialized during probe */
 614         if (enic->fm == NULL && enic_fm_init(enic))
 615                 dev_warning(enic, "Init of flowman failed.\n");
 616
 617         for (index = 0; index < enic->rq_count; index++) {
 618                 err = enic_alloc_rx_queue_mbufs(enic,
 619                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 620                 if (err) {
 621                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 622                         return err;
 623                 }
 624                 err = enic_alloc_rx_queue_mbufs(enic,
 625                         &enic->rq[enic_rte_rq_idx_to_data_idx(index, enic)]);
 626                 if (err) {
 627                         /* release the allocated mbufs for the sop rq*/
 628                         enic_rxmbuf_queue_release(enic,
 629                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 630
 631                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 632                         return err;
 633                 }
 634         }
 635
 636         /*
 637          * Use the simple TX handler if possible. Only checksum offloads
 638          * and vlan insertion are supported.
 639          */
 640         simple_tx_offloads = enic->tx_offload_capa &
 641                 (RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 642                  RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
 643                  RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
 644                  RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
 645                  RTE_ETH_TX_OFFLOAD_TCP_CKSUM);
 646         if ((eth_dev->data->dev_conf.txmode.offloads &
 647              ~simple_tx_offloads) == 0) {
 648                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 649                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 650                 for (index = 0; index < enic->wq_count; index++)
 651                         enic_prep_wq_for_simple_tx(enic, index);
 652                 enic->use_simple_tx_handler = 1;
 653         } else {
 654                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 655                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 656         }
 657
 658         enic_pick_rx_handler(eth_dev);
 659
 660         for (index = 0; index < enic->wq_count; index++)
 661                 enic_start_wq(enic, index);
 662         for (index = 0; index < enic->rq_count; index++)
 663                 enic_start_rq(enic, index);
 664
 665         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 666
 667         vnic_dev_enable_wait(enic->vdev);
 668
 669         /* Register and enable error interrupt */
 670         rte_intr_callback_register(&(enic->pdev->intr_handle),
 671                 enic_intr_handler, (void *)enic->rte_dev);
 672
 673         rte_intr_enable(&(enic->pdev->intr_handle));
 674         /* Unmask LSC interrupt */
 675         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 676
 677         return 0;
 678 }
 679
 680 int enic_alloc_intr_resources(struct enic *enic)
 681 {
 682         int err;
 683         unsigned int i;
 684
 685         dev_info(enic, "vNIC resources used:  "\
 686                 "wq %d rq %d cq %d intr %d\n",
 687                 enic->wq_count, enic_vnic_rq_count(enic),
 688                 enic->cq_count, enic->intr_count);
 689
 690         for (i = 0; i < enic->intr_count; i++) {
 691                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 692                 if (err) {
 693                         enic_free_vnic_resources(enic);
 694                         return err;
 695                 }
 696         }
 697         return 0;
 698 }
 699
 700 void enic_free_rq(void *rxq)
 701 {
 702         struct vnic_rq *rq_sop, *rq_data;
 703         struct enic *enic;
 704
 705         if (rxq == NULL)
 706                 return;
 707
 708         rq_sop = (struct vnic_rq *)rxq;
 709         enic = vnic_dev_priv(rq_sop->vdev);
 710         rq_data = &enic->rq[rq_sop->data_queue_idx];
 711
 712         if (rq_sop->free_mbufs) {
 713                 struct rte_mbuf **mb;
 714                 int i;
 715
 716                 mb = rq_sop->free_mbufs;
 717                 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
 718                      i < ENIC_RX_BURST_MAX; i++)
 719                         rte_pktmbuf_free(mb[i]);
 720                 rte_free(rq_sop->free_mbufs);
 721                 rq_sop->free_mbufs = NULL;
 722                 rq_sop->num_free_mbufs = 0;
 723         }
 724
 725         enic_rxmbuf_queue_release(enic, rq_sop);
 726         if (rq_data->in_use)
 727                 enic_rxmbuf_queue_release(enic, rq_data);
 728
 729         rte_free(rq_sop->mbuf_ring);
 730         if (rq_data->in_use)
 731                 rte_free(rq_data->mbuf_ring);
 732
 733         rq_sop->mbuf_ring = NULL;
 734         rq_data->mbuf_ring = NULL;
 735
 736         vnic_rq_free(rq_sop);
 737         if (rq_data->in_use)
 738                 vnic_rq_free(rq_data);
 739
 740         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 741
 742         rq_sop->in_use = 0;
 743         rq_data->in_use = 0;
 744 }
 745
 746 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 747 {
 748         struct rte_eth_dev_data *data = enic->dev_data;
 749         vnic_wq_enable(&enic->wq[queue_idx]);
 750         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 751 }
 752
 753 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 754 {
 755         struct rte_eth_dev_data *data = enic->dev_data;
 756         int ret;
 757
 758         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 759         if (ret)
 760                 return ret;
 761
 762         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 763         return 0;
 764 }
 765
 766 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 767 {
 768         struct rte_eth_dev_data *data = enic->dev_data;
 769         struct vnic_rq *rq_sop;
 770         struct vnic_rq *rq_data;
 771         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 772         rq_data = &enic->rq[rq_sop->data_queue_idx];
 773
 774         if (rq_data->in_use) {
 775                 vnic_rq_enable(rq_data);
 776                 enic_initial_post_rx(enic, rq_data);
 777         }
 778         rte_mb();
 779         vnic_rq_enable(rq_sop);
 780         enic_initial_post_rx(enic, rq_sop);
 781         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 782 }
 783
 784 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 785 {
 786         struct rte_eth_dev_data *data = enic->dev_data;
 787         int ret1 = 0, ret2 = 0;
 788         struct vnic_rq *rq_sop;
 789         struct vnic_rq *rq_data;
 790         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 791         rq_data = &enic->rq[rq_sop->data_queue_idx];
 792
 793         ret2 = vnic_rq_disable(rq_sop);
 794         rte_mb();
 795         if (rq_data->in_use)
 796                 ret1 = vnic_rq_disable(rq_data);
 797
 798         if (ret2)
 799                 return ret2;
 800         else if (ret1)
 801                 return ret1;
 802
 803         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 804         return 0;
 805 }
 806
 807 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 808         unsigned int socket_id, struct rte_mempool *mp,
 809         uint16_t nb_desc, uint16_t free_thresh)
 810 {
 811         struct enic_vf_representor *vf;
 812         int rc;
 813         uint16_t sop_queue_idx;
 814         uint16_t data_queue_idx;
 815         uint16_t cq_idx;
 816         struct vnic_rq *rq_sop;
 817         struct vnic_rq *rq_data;
 818         unsigned int mbuf_size, mbufs_per_pkt;
 819         unsigned int nb_sop_desc, nb_data_desc;
 820         uint16_t min_sop, max_sop, min_data, max_data;
 821         uint32_t max_rx_pktlen;
 822
 823         /*
 824          * Representor uses a reserved PF queue. Translate representor
 825          * queue number to PF queue number.
 826          */
 827         if (enic_is_vf_rep(enic)) {
 828                 RTE_ASSERT(queue_idx == 0);
 829                 vf = VF_ENIC_TO_VF_REP(enic);
 830                 sop_queue_idx = vf->pf_rq_sop_idx;
 831                 data_queue_idx = vf->pf_rq_data_idx;
 832                 enic = vf->pf;
 833                 queue_idx = sop_queue_idx;
 834         } else {
 835                 sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 836                 data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx, enic);
 837         }
 838         cq_idx = enic_cq_rq(enic, sop_queue_idx);
 839         rq_sop = &enic->rq[sop_queue_idx];
 840         rq_data = &enic->rq[data_queue_idx];
 841         rq_sop->is_sop = 1;
 842         rq_sop->data_queue_idx = data_queue_idx;
 843         rq_data->is_sop = 0;
 844         rq_data->data_queue_idx = 0;
 845         rq_sop->socket_id = socket_id;
 846         rq_sop->mp = mp;
 847         rq_data->socket_id = socket_id;
 848         rq_data->mp = mp;
 849         rq_sop->in_use = 1;
 850         rq_sop->rx_free_thresh = free_thresh;
 851         rq_data->rx_free_thresh = free_thresh;
 852         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 853                   free_thresh);
 854
 855         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 856                                RTE_PKTMBUF_HEADROOM);
 857         /* max_rx_pktlen includes the ethernet header and CRC. */
 858         max_rx_pktlen = enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu);
 859
 860         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 861             RTE_ETH_RX_OFFLOAD_SCATTER) {
 862                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 863                 /* ceil((max pkt len)/mbuf_size) */
 864                 mbufs_per_pkt = (max_rx_pktlen + mbuf_size - 1) / mbuf_size;
 865         } else {
 866                 dev_info(enic, "Scatter rx mode disabled\n");
 867                 mbufs_per_pkt = 1;
 868                 if (max_rx_pktlen > mbuf_size) {
 869                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 870                                     " larger than the mbuf size (%u), and"
 871                                     " scatter is disabled. Larger packets will"
 872                                     " be truncated.\n",
 873                                     max_rx_pktlen, mbuf_size);
 874                 }
 875         }
 876
 877         if (mbufs_per_pkt > 1) {
 878                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 879                 rq_sop->data_queue_enable = 1;
 880                 rq_data->in_use = 1;
 881                 /*
 882                  * HW does not directly support MTU. HW always
 883                  * receives packet sizes up to the "max" MTU.
 884                  * If not using scatter, we can achieve the effect of dropping
 885                  * larger packets by reducing the size of posted buffers.
 886                  * See enic_alloc_rx_queue_mbufs().
 887                  */
 888                 if (enic->rte_dev->data->mtu < enic->max_mtu) {
 889                         dev_warning(enic,
 890                                 "mtu is ignored when scatter rx mode is in use.\n");
 891                 }
 892         } else {
 893                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 894                          queue_idx);
 895                 rq_sop->data_queue_enable = 0;
 896                 rq_data->in_use = 0;
 897         }
 898
 899         /* number of descriptors have to be a multiple of 32 */
 900         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
 901         nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 902
 903         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 904         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 905
 906         if (mbufs_per_pkt > 1) {
 907                 min_sop = ENIC_RX_BURST_MAX;
 908                 max_sop = ((enic->config.rq_desc_count /
 909                             (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 910                 min_data = min_sop * (mbufs_per_pkt - 1);
 911                 max_data = enic->config.rq_desc_count;
 912         } else {
 913                 min_sop = ENIC_RX_BURST_MAX;
 914                 max_sop = enic->config.rq_desc_count;
 915                 min_data = 0;
 916                 max_data = 0;
 917         }
 918
 919         if (nb_desc < (min_sop + min_data)) {
 920                 dev_warning(enic,
 921                             "Number of rx descs too low, adjusting to minimum\n");
 922                 nb_sop_desc = min_sop;
 923                 nb_data_desc = min_data;
 924         } else if (nb_desc > (max_sop + max_data)) {
 925                 dev_warning(enic,
 926                             "Number of rx_descs too high, adjusting to maximum\n");
 927                 nb_sop_desc = max_sop;
 928                 nb_data_desc = max_data;
 929         }
 930         if (mbufs_per_pkt > 1) {
 931                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 932                          " rx descriptor range is %u to %u\n",
 933                          max_rx_pktlen, mbuf_size, min_sop + min_data,
 934                          max_sop + max_data);
 935         }
 936         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 937                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 938
 939         /* Allocate sop queue resources */
 940         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 941                 nb_sop_desc, sizeof(struct rq_enet_desc));
 942         if (rc) {
 943                 dev_err(enic, "error in allocation of sop rq\n");
 944                 goto err_exit;
 945         }
 946         nb_sop_desc = rq_sop->ring.desc_count;
 947
 948         if (rq_data->in_use) {
 949                 /* Allocate data queue resources */
 950                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 951                                    nb_data_desc,
 952                                    sizeof(struct rq_enet_desc));
 953                 if (rc) {
 954                         dev_err(enic, "error in allocation of data rq\n");
 955                         goto err_free_rq_sop;
 956                 }
 957                 nb_data_desc = rq_data->ring.desc_count;
 958         }
 959         /* Enable 64B CQ entry if requested */
 960         if (enic->cq64 && vnic_dev_set_cq_entry_size(enic->vdev,
 961                                 sop_queue_idx, VNIC_RQ_CQ_ENTRY_SIZE_64)) {
 962                 dev_err(enic, "failed to enable 64B CQ entry on sop rq\n");
 963                 goto err_free_rq_data;
 964         }
 965         if (rq_data->in_use && enic->cq64 &&
 966             vnic_dev_set_cq_entry_size(enic->vdev, data_queue_idx,
 967                 VNIC_RQ_CQ_ENTRY_SIZE_64)) {
 968                 dev_err(enic, "failed to enable 64B CQ entry on data rq\n");
 969                 goto err_free_rq_data;
 970         }
 971
 972         rc = vnic_cq_alloc(enic->vdev, &enic->cq[cq_idx], cq_idx,
 973                            socket_id, nb_sop_desc + nb_data_desc,
 974                            enic->cq64 ? sizeof(struct cq_enet_rq_desc_64) :
 975                            sizeof(struct cq_enet_rq_desc));
 976         if (rc) {
 977                 dev_err(enic, "error in allocation of cq for rq\n");
 978                 goto err_free_rq_data;
 979         }
 980
 981         /* Allocate the mbuf rings */
 982         rq_sop->mbuf_ring = (struct rte_mbuf **)
 983                 rte_zmalloc_socket("rq->mbuf_ring",
 984                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 985                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 986         if (rq_sop->mbuf_ring == NULL)
 987                 goto err_free_cq;
 988
 989         if (rq_data->in_use) {
 990                 rq_data->mbuf_ring = (struct rte_mbuf **)
 991                         rte_zmalloc_socket("rq->mbuf_ring",
 992                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 993                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 994                 if (rq_data->mbuf_ring == NULL)
 995                         goto err_free_sop_mbuf;
 996         }
 997
 998         rq_sop->free_mbufs = (struct rte_mbuf **)
 999                 rte_zmalloc_socket("rq->free_mbufs",
1000                                    sizeof(struct rte_mbuf *) *
1001                                    ENIC_RX_BURST_MAX,
1002                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
1003         if (rq_sop->free_mbufs == NULL)
1004                 goto err_free_data_mbuf;
1005         rq_sop->num_free_mbufs = 0;
1006
1007         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
1008
1009         return 0;
1010
1011 err_free_data_mbuf:
1012         rte_free(rq_data->mbuf_ring);
1013 err_free_sop_mbuf:
1014         rte_free(rq_sop->mbuf_ring);
1015 err_free_cq:
1016         /* cleanup on error */
1017         vnic_cq_free(&enic->cq[cq_idx]);
1018 err_free_rq_data:
1019         if (rq_data->in_use)
1020                 vnic_rq_free(rq_data);
1021 err_free_rq_sop:
1022         vnic_rq_free(rq_sop);
1023 err_exit:
1024         return -ENOMEM;
1025 }
1026
1027 void enic_free_wq(void *txq)
1028 {
1029         struct vnic_wq *wq;
1030         struct enic *enic;
1031
1032         if (txq == NULL)
1033                 return;
1034
1035         wq = (struct vnic_wq *)txq;
1036         enic = vnic_dev_priv(wq->vdev);
1037         rte_memzone_free(wq->cqmsg_rz);
1038         vnic_wq_free(wq);
1039         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1040 }
1041
1042 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1043         unsigned int socket_id, uint16_t nb_desc)
1044 {
1045         struct enic_vf_representor *vf;
1046         int err;
1047         struct vnic_wq *wq;
1048         unsigned int cq_index;
1049         char name[RTE_MEMZONE_NAMESIZE];
1050         static int instance;
1051
1052         /*
1053          * Representor uses a reserved PF queue. Translate representor
1054          * queue number to PF queue number.
1055          */
1056         if (enic_is_vf_rep(enic)) {
1057                 RTE_ASSERT(queue_idx == 0);
1058                 vf = VF_ENIC_TO_VF_REP(enic);
1059                 queue_idx = vf->pf_wq_idx;
1060                 cq_index = vf->pf_wq_cq_idx;
1061                 enic = vf->pf;
1062         } else {
1063                 cq_index = enic_cq_wq(enic, queue_idx);
1064         }
1065         wq = &enic->wq[queue_idx];
1066         wq->socket_id = socket_id;
1067         /*
1068          * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1069          * print an info message for diagnostics.
1070          */
1071         dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1072
1073         /* Allocate queue resources */
1074         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1075                 nb_desc,
1076                 sizeof(struct wq_enet_desc));
1077         if (err) {
1078                 dev_err(enic, "error in allocation of wq\n");
1079                 return err;
1080         }
1081
1082         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1083                 socket_id, nb_desc,
1084                 sizeof(struct cq_enet_wq_desc));
1085         if (err) {
1086                 vnic_wq_free(wq);
1087                 dev_err(enic, "error in allocation of cq for wq\n");
1088         }
1089
1090         /* setup up CQ message */
1091         snprintf((char *)name, sizeof(name),
1092                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1093                 instance++);
1094
1095         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1096                         sizeof(uint32_t), SOCKET_ID_ANY,
1097                         RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
1098         if (!wq->cqmsg_rz)
1099                 return -ENOMEM;
1100
1101         return err;
1102 }
1103
1104 int enic_disable(struct enic *enic)
1105 {
1106         unsigned int i;
1107         int err;
1108
1109         for (i = 0; i < enic->intr_count; i++) {
1110                 vnic_intr_mask(&enic->intr[i]);
1111                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1112         }
1113         enic_rxq_intr_deinit(enic);
1114         rte_intr_disable(&enic->pdev->intr_handle);
1115         rte_intr_callback_unregister(&enic->pdev->intr_handle,
1116                                      enic_intr_handler,
1117                                      (void *)enic->rte_dev);
1118
1119         vnic_dev_disable(enic->vdev);
1120
1121         enic_fm_destroy(enic);
1122
1123         if (!enic_is_sriov_vf(enic))
1124                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1125
1126         for (i = 0; i < enic->wq_count; i++) {
1127                 err = vnic_wq_disable(&enic->wq[i]);
1128                 if (err)
1129                         return err;
1130         }
1131         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1132                 if (enic->rq[i].in_use) {
1133                         err = vnic_rq_disable(&enic->rq[i]);
1134                         if (err)
1135                                 return err;
1136                 }
1137         }
1138
1139         /* If we were using interrupts, set the interrupt vector to -1
1140          * to disable interrupts.  We are not disabling link notifcations,
1141          * though, as we want the polling of link status to continue working.
1142          */
1143         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1144                 vnic_dev_notify_set(enic->vdev, -1);
1145
1146         vnic_dev_set_reset_flag(enic->vdev, 1);
1147
1148         for (i = 0; i < enic->wq_count; i++)
1149                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1150
1151         for (i = 0; i < enic_vnic_rq_count(enic); i++)
1152                 if (enic->rq[i].in_use)
1153                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1154         for (i = 0; i < enic->cq_count; i++)
1155                 vnic_cq_clean(&enic->cq[i]);
1156         for (i = 0; i < enic->intr_count; i++)
1157                 vnic_intr_clean(&enic->intr[i]);
1158
1159         return 0;
1160 }
1161
1162 static int enic_dev_wait(struct vnic_dev *vdev,
1163         int (*start)(struct vnic_dev *, int),
1164         int (*finished)(struct vnic_dev *, int *),
1165         int arg)
1166 {
1167         int done;
1168         int err;
1169         int i;
1170
1171         err = start(vdev, arg);
1172         if (err)
1173                 return err;
1174
1175         /* Wait for func to complete...2 seconds max */
1176         for (i = 0; i < 2000; i++) {
1177                 err = finished(vdev, &done);
1178                 if (err)
1179                         return err;
1180                 if (done)
1181                         return 0;
1182                 usleep(1000);
1183         }
1184         return -ETIMEDOUT;
1185 }
1186
1187 static int enic_dev_open(struct enic *enic)
1188 {
1189         int err;
1190         int flags = CMD_OPENF_IG_DESCCACHE;
1191
1192         err = enic_dev_wait(enic->vdev, vnic_dev_open,
1193                 vnic_dev_open_done, flags);
1194         if (err)
1195                 dev_err(enic_get_dev(enic),
1196                         "vNIC device open failed, err %d\n", err);
1197
1198         return err;
1199 }
1200
1201 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1202 {
1203         dma_addr_t rss_key_buf_pa;
1204         union vnic_rss_key *rss_key_buf_va = NULL;
1205         int err, i;
1206         uint8_t name[RTE_MEMZONE_NAMESIZE];
1207
1208         RTE_ASSERT(user_key != NULL);
1209         snprintf((char *)name, sizeof(name), "rss_key-%s", enic->bdf_name);
1210         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1211                 &rss_key_buf_pa, name);
1212         if (!rss_key_buf_va)
1213                 return -ENOMEM;
1214
1215         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1216                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1217
1218         err = enic_set_rss_key(enic,
1219                 rss_key_buf_pa,
1220                 sizeof(union vnic_rss_key));
1221
1222         /* Save for later queries */
1223         if (!err) {
1224                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1225                            sizeof(union vnic_rss_key));
1226         }
1227         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1228                 rss_key_buf_va, rss_key_buf_pa);
1229
1230         return err;
1231 }
1232
1233 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1234 {
1235         dma_addr_t rss_cpu_buf_pa;
1236         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1237         int err;
1238         uint8_t name[RTE_MEMZONE_NAMESIZE];
1239
1240         snprintf((char *)name, sizeof(name), "rss_cpu-%s", enic->bdf_name);
1241         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1242                 &rss_cpu_buf_pa, name);
1243         if (!rss_cpu_buf_va)
1244                 return -ENOMEM;
1245
1246         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1247
1248         err = enic_set_rss_cpu(enic,
1249                 rss_cpu_buf_pa,
1250                 sizeof(union vnic_rss_cpu));
1251
1252         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1253                 rss_cpu_buf_va, rss_cpu_buf_pa);
1254
1255         /* Save for later queries */
1256         if (!err)
1257                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1258         return err;
1259 }
1260
1261 static int enic_set_niccfg(struct enic *enic, uint8_t rss_default_cpu,
1262         uint8_t rss_hash_type, uint8_t rss_hash_bits, uint8_t rss_base_cpu,
1263         uint8_t rss_enable)
1264 {
1265         const uint8_t tso_ipid_split_en = 0;
1266         int err;
1267
1268         err = enic_set_nic_cfg(enic,
1269                 rss_default_cpu, rss_hash_type,
1270                 rss_hash_bits, rss_base_cpu,
1271                 rss_enable, tso_ipid_split_en,
1272                 enic->ig_vlan_strip_en);
1273
1274         return err;
1275 }
1276
1277 /* Initialize RSS with defaults, called from dev_configure */
1278 int enic_init_rss_nic_cfg(struct enic *enic)
1279 {
1280         static uint8_t default_rss_key[] = {
1281                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1282                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1283                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1284                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1285         };
1286         struct rte_eth_rss_conf rss_conf;
1287         union vnic_rss_cpu rss_cpu;
1288         int ret, i;
1289
1290         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1291         /*
1292          * If setting key for the first time, and the user gives us none, then
1293          * push the default key to NIC.
1294          */
1295         if (rss_conf.rss_key == NULL) {
1296                 rss_conf.rss_key = default_rss_key;
1297                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1298         }
1299         ret = enic_set_rss_conf(enic, &rss_conf);
1300         if (ret) {
1301                 dev_err(enic, "Failed to configure RSS\n");
1302                 return ret;
1303         }
1304         if (enic->rss_enable) {
1305                 /* If enabling RSS, use the default reta */
1306                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1307                         rss_cpu.cpu[i / 4].b[i % 4] =
1308                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1309                 }
1310                 ret = enic_set_rss_reta(enic, &rss_cpu);
1311                 if (ret)
1312                         dev_err(enic, "Failed to set RSS indirection table\n");
1313         }
1314         return ret;
1315 }
1316
1317 int enic_setup_finish(struct enic *enic)
1318 {
1319         enic_init_soft_stats(enic);
1320
1321         /* switchdev: enable promisc mode on PF */
1322         if (enic->switchdev_mode) {
1323                 vnic_dev_packet_filter(enic->vdev,
1324                                        0 /* directed  */,
1325                                        0 /* multicast */,
1326                                        0 /* broadcast */,
1327                                        1 /* promisc   */,
1328                                        0 /* allmulti  */);
1329                 enic->promisc = 1;
1330                 enic->allmulti = 0;
1331                 return 0;
1332         }
1333         /* Default conf */
1334         vnic_dev_packet_filter(enic->vdev,
1335                 1 /* directed  */,
1336                 1 /* multicast */,
1337                 1 /* broadcast */,
1338                 0 /* promisc   */,
1339                 1 /* allmulti  */);
1340
1341         enic->promisc = 0;
1342         enic->allmulti = 1;
1343
1344         return 0;
1345 }
1346
1347 static int enic_rss_conf_valid(struct enic *enic,
1348                                struct rte_eth_rss_conf *rss_conf)
1349 {
1350         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1351         if (enic->flow_type_rss_offloads == 0)
1352                 return 0;
1353         if (rss_conf->rss_key != NULL &&
1354             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1355                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1356                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1357                 return -EINVAL;
1358         }
1359         if (rss_conf->rss_hf != 0 &&
1360             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1361                 dev_err(enic, "Given rss_hf contains none of the supported"
1362                         " types\n");
1363                 return -EINVAL;
1364         }
1365         return 0;
1366 }
1367
1368 /* Set hash type and key according to rss_conf */
1369 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1370 {
1371         struct rte_eth_dev *eth_dev;
1372         uint64_t rss_hf;
1373         uint8_t rss_hash_type;
1374         uint8_t rss_enable;
1375         int ret;
1376
1377         RTE_ASSERT(rss_conf != NULL);
1378         ret = enic_rss_conf_valid(enic, rss_conf);
1379         if (ret) {
1380                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1381                 return ret;
1382         }
1383
1384         eth_dev = enic->rte_dev;
1385         rss_hash_type = 0;
1386         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1387         if (enic->rq_count > 1 &&
1388             (eth_dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) &&
1389             rss_hf != 0) {
1390                 rss_enable = 1;
1391                 if (rss_hf & (RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
1392                               RTE_ETH_RSS_NONFRAG_IPV4_OTHER))
1393                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1394                 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
1395                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1396                 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) {
1397                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1398                         if (enic->udp_rss_weak) {
1399                                 /*
1400                                  * 'TCP' is not a typo. The "weak" version of
1401                                  * UDP RSS requires both the TCP and UDP bits
1402                                  * be set. It does enable TCP RSS as well.
1403                                  */
1404                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1405                         }
1406                 }
1407                 if (rss_hf & (RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_IPV6_EX |
1408                               RTE_ETH_RSS_FRAG_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_OTHER))
1409                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1410                 if (rss_hf & (RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_IPV6_TCP_EX))
1411                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1412                 if (rss_hf & (RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_IPV6_UDP_EX)) {
1413                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1414                         if (enic->udp_rss_weak)
1415                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1416                 }
1417         } else {
1418                 rss_enable = 0;
1419                 rss_hf = 0;
1420         }
1421
1422         /* Set the hash key if provided */
1423         if (rss_enable && rss_conf->rss_key) {
1424                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1425                 if (ret) {
1426                         dev_err(enic, "Failed to set RSS key\n");
1427                         return ret;
1428                 }
1429         }
1430
1431         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1432                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1433                               rss_enable);
1434         if (!ret) {
1435                 enic->rss_hf = rss_hf;
1436                 enic->rss_hash_type = rss_hash_type;
1437                 enic->rss_enable = rss_enable;
1438         } else {
1439                 dev_err(enic, "Failed to update RSS configurations."
1440                         " hash=0x%x\n", rss_hash_type);
1441         }
1442         return ret;
1443 }
1444
1445 int enic_set_vlan_strip(struct enic *enic)
1446 {
1447         /*
1448          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1449          * together. So, re-do niccfg, preserving the current RSS settings.
1450          */
1451         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1452                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1453                                enic->rss_enable);
1454 }
1455
1456 int enic_add_packet_filter(struct enic *enic)
1457 {
1458         /* switchdev ignores packet filters */
1459         if (enic->switchdev_mode) {
1460                 ENICPMD_LOG(DEBUG, " switchdev: ignore packet filter");
1461                 return 0;
1462         }
1463         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1464         return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1465                 enic->promisc, enic->allmulti);
1466 }
1467
1468 int enic_get_link_status(struct enic *enic)
1469 {
1470         return vnic_dev_link_status(enic->vdev);
1471 }
1472
1473 static void enic_dev_deinit(struct enic *enic)
1474 {
1475         /* stop link status checking */
1476         vnic_dev_notify_unset(enic->vdev);
1477
1478         /* mac_addrs is freed by rte_eth_dev_release_port() */
1479         rte_free(enic->cq);
1480         rte_free(enic->intr);
1481         rte_free(enic->rq);
1482         rte_free(enic->wq);
1483 }
1484
1485
1486 int enic_set_vnic_res(struct enic *enic)
1487 {
1488         struct rte_eth_dev *eth_dev = enic->rte_dev;
1489         int rc = 0;
1490         unsigned int required_rq, required_wq, required_cq, required_intr;
1491
1492         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1493         required_rq = eth_dev->data->nb_rx_queues * 2;
1494         required_wq = eth_dev->data->nb_tx_queues;
1495         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1496         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1497         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1498                 required_intr += eth_dev->data->nb_rx_queues;
1499         }
1500         ENICPMD_LOG(DEBUG, "Required queues for PF: rq %u wq %u cq %u",
1501                     required_rq, required_wq, required_cq);
1502         if (enic->vf_required_rq) {
1503                 /* Queues needed for VF representors */
1504                 required_rq += enic->vf_required_rq;
1505                 required_wq += enic->vf_required_wq;
1506                 required_cq += enic->vf_required_cq;
1507                 ENICPMD_LOG(DEBUG, "Required queues for VF representors: rq %u wq %u cq %u",
1508                             enic->vf_required_rq, enic->vf_required_wq,
1509                             enic->vf_required_cq);
1510         }
1511
1512         if (enic->conf_rq_count < required_rq) {
1513                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1514                         eth_dev->data->nb_rx_queues,
1515                         required_rq, enic->conf_rq_count);
1516                 rc = -EINVAL;
1517         }
1518         if (enic->conf_wq_count < required_wq) {
1519                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1520                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1521                 rc = -EINVAL;
1522         }
1523
1524         if (enic->conf_cq_count < required_cq) {
1525                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1526                         required_cq, enic->conf_cq_count);
1527                 rc = -EINVAL;
1528         }
1529         if (enic->conf_intr_count < required_intr) {
1530                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1531                         " interrupts. Required:%u, Configured:%u\n",
1532                         required_intr, enic->conf_intr_count);
1533                 rc = -EINVAL;
1534         }
1535
1536         if (rc == 0) {
1537                 enic->rq_count = eth_dev->data->nb_rx_queues;
1538                 enic->wq_count = eth_dev->data->nb_tx_queues;
1539                 enic->cq_count = enic->rq_count + enic->wq_count;
1540                 enic->intr_count = required_intr;
1541         }
1542
1543         return rc;
1544 }
1545
1546 /* Initialize the completion queue for an RQ */
1547 static int
1548 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1549 {
1550         struct vnic_rq *sop_rq, *data_rq;
1551         unsigned int cq_idx;
1552         int rc = 0;
1553
1554         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1555         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx, enic)];
1556         cq_idx = enic_cq_rq(enic, rq_idx);
1557
1558         vnic_cq_clean(&enic->cq[cq_idx]);
1559         vnic_cq_init(&enic->cq[cq_idx],
1560                      0 /* flow_control_enable */,
1561                      1 /* color_enable */,
1562                      0 /* cq_head */,
1563                      0 /* cq_tail */,
1564                      1 /* cq_tail_color */,
1565                      0 /* interrupt_enable */,
1566                      1 /* cq_entry_enable */,
1567                      0 /* cq_message_enable */,
1568                      0 /* interrupt offset */,
1569                      0 /* cq_message_addr */);
1570
1571
1572         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1573                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1574                            sop_rq->ring.desc_count - 1, 1, 0);
1575         if (data_rq->in_use) {
1576                 vnic_rq_init_start(data_rq,
1577                                    enic_cq_rq(enic,
1578                                    enic_rte_rq_idx_to_data_idx(rq_idx, enic)),
1579                                    0, data_rq->ring.desc_count - 1, 1, 0);
1580         }
1581
1582         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1583         if (rc)
1584                 return rc;
1585
1586         if (data_rq->in_use) {
1587                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1588                 if (rc) {
1589                         enic_rxmbuf_queue_release(enic, sop_rq);
1590                         return rc;
1591                 }
1592         }
1593
1594         return 0;
1595 }
1596
1597 /* The Cisco NIC can send and receive packets up to a max packet size
1598  * determined by the NIC type and firmware. There is also an MTU
1599  * configured into the NIC via the CIMC/UCSM management interface
1600  * which can be overridden by this function (up to the max packet size).
1601  * Depending on the network setup, doing so may cause packet drops
1602  * and unexpected behavior.
1603  */
1604 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1605 {
1606         unsigned int rq_idx;
1607         struct vnic_rq *rq;
1608         int rc = 0;
1609         uint16_t old_mtu;       /* previous setting */
1610         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1611         struct rte_eth_dev *eth_dev = enic->rte_dev;
1612
1613         old_mtu = eth_dev->data->mtu;
1614         config_mtu = enic->config.mtu;
1615
1616         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1617                 return -E_RTE_SECONDARY;
1618
1619         if (new_mtu > enic->max_mtu) {
1620                 dev_err(enic,
1621                         "MTU not updated: requested (%u) greater than max (%u)\n",
1622                         new_mtu, enic->max_mtu);
1623                 return -EINVAL;
1624         }
1625         if (new_mtu < ENIC_MIN_MTU) {
1626                 dev_info(enic,
1627                         "MTU not updated: requested (%u) less than min (%u)\n",
1628                         new_mtu, ENIC_MIN_MTU);
1629                 return -EINVAL;
1630         }
1631         if (new_mtu > config_mtu)
1632                 dev_warning(enic,
1633                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1634                         new_mtu, config_mtu);
1635
1636         /*
1637          * If the device has not started (enic_enable), nothing to do.
1638          * Later, enic_enable() will set up RQs reflecting the new maximum
1639          * packet length.
1640          */
1641         if (!eth_dev->data->dev_started)
1642                 goto set_mtu_done;
1643
1644         /*
1645          * The device has started, re-do RQs on the fly. In the process, we
1646          * pick up the new maximum packet length.
1647          *
1648          * Some applications rely on the ability to change MTU without stopping
1649          * the device. So keep this behavior for now.
1650          */
1651         rte_spinlock_lock(&enic->mtu_lock);
1652
1653         /* Stop traffic on all RQs */
1654         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1655                 rq = &enic->rq[rq_idx];
1656                 if (rq->is_sop && rq->in_use) {
1657                         rc = enic_stop_rq(enic,
1658                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1659                         if (rc) {
1660                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1661                                 goto set_mtu_done;
1662                         }
1663                 }
1664         }
1665
1666         /* replace Rx function with a no-op to avoid getting stale pkts */
1667         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1668         rte_mb();
1669
1670         /* Allow time for threads to exit the real Rx function. */
1671         usleep(100000);
1672
1673         /* now it is safe to reconfigure the RQs */
1674
1675
1676         /* free and reallocate RQs with the new MTU */
1677         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1678                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1679                 if (!rq->in_use)
1680                         continue;
1681
1682                 enic_free_rq(rq);
1683                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1684                                    rq->tot_nb_desc, rq->rx_free_thresh);
1685                 if (rc) {
1686                         dev_err(enic,
1687                                 "Fatal MTU alloc error- No traffic will pass\n");
1688                         goto set_mtu_done;
1689                 }
1690
1691                 rc = enic_reinit_rq(enic, rq_idx);
1692                 if (rc) {
1693                         dev_err(enic,
1694                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1695                         goto set_mtu_done;
1696                 }
1697         }
1698
1699         /* put back the real receive function */
1700         rte_mb();
1701         enic_pick_rx_handler(eth_dev);
1702         rte_mb();
1703
1704         /* restart Rx traffic */
1705         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1706                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1707                 if (rq->is_sop && rq->in_use)
1708                         enic_start_rq(enic, rq_idx);
1709         }
1710
1711 set_mtu_done:
1712         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1713         rte_spinlock_unlock(&enic->mtu_lock);
1714         return rc;
1715 }
1716
1717 static void
1718 enic_disable_overlay_offload(struct enic *enic)
1719 {
1720         /*
1721          * Disabling fails if the feature is provisioned but
1722          * not enabled. So ignore result and do not log error.
1723          */
1724         if (enic->vxlan) {
1725                 vnic_dev_overlay_offload_ctrl(enic->vdev,
1726                         OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE);
1727         }
1728         if (enic->geneve) {
1729                 vnic_dev_overlay_offload_ctrl(enic->vdev,
1730                         OVERLAY_FEATURE_GENEVE, OVERLAY_OFFLOAD_DISABLE);
1731         }
1732 }
1733
1734 static int
1735 enic_enable_overlay_offload(struct enic *enic)
1736 {
1737         if (enic->vxlan && vnic_dev_overlay_offload_ctrl(enic->vdev,
1738                         OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) != 0) {
1739                 dev_err(NULL, "failed to enable VXLAN offload\n");
1740                 return -EINVAL;
1741         }
1742         if (enic->geneve && vnic_dev_overlay_offload_ctrl(enic->vdev,
1743                         OVERLAY_FEATURE_GENEVE, OVERLAY_OFFLOAD_ENABLE) != 0) {
1744                 dev_err(NULL, "failed to enable Geneve offload\n");
1745                 return -EINVAL;
1746         }
1747         enic->tx_offload_capa |=
1748                 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1749                 (enic->geneve ? RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO : 0) |
1750                 (enic->vxlan ? RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO : 0);
1751         enic->tx_offload_mask |=
1752                 RTE_MBUF_F_TX_OUTER_IPV6 |
1753                 RTE_MBUF_F_TX_OUTER_IPV4 |
1754                 RTE_MBUF_F_TX_OUTER_IP_CKSUM |
1755                 RTE_MBUF_F_TX_TUNNEL_MASK;
1756         enic->overlay_offload = true;
1757
1758         if (enic->vxlan && enic->geneve)
1759                 dev_info(NULL, "Overlay offload is enabled (VxLAN, Geneve)\n");
1760         else if (enic->vxlan)
1761                 dev_info(NULL, "Overlay offload is enabled (VxLAN)\n");
1762         else
1763                 dev_info(NULL, "Overlay offload is enabled (Geneve)\n");
1764
1765         return 0;
1766 }
1767
1768 static int
1769 enic_reset_overlay_port(struct enic *enic)
1770 {
1771         if (enic->vxlan) {
1772                 enic->vxlan_port = RTE_VXLAN_DEFAULT_PORT;
1773                 /*
1774                  * Reset the vxlan port to the default, as the NIC firmware
1775                  * does not reset it automatically and keeps the old setting.
1776                  */
1777                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1778                                                  OVERLAY_CFG_VXLAN_PORT_UPDATE,
1779                                                  RTE_VXLAN_DEFAULT_PORT)) {
1780                         dev_err(enic, "failed to update vxlan port\n");
1781                         return -EINVAL;
1782                 }
1783         }
1784         if (enic->geneve) {
1785                 enic->geneve_port = RTE_GENEVE_DEFAULT_PORT;
1786                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1787                                                  OVERLAY_CFG_GENEVE_PORT_UPDATE,
1788                                                  RTE_GENEVE_DEFAULT_PORT)) {
1789                         dev_err(enic, "failed to update vxlan port\n");
1790                         return -EINVAL;
1791                 }
1792         }
1793         return 0;
1794 }
1795
1796 static int enic_dev_init(struct enic *enic)
1797 {
1798         int err;
1799         struct rte_eth_dev *eth_dev = enic->rte_dev;
1800
1801         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1802
1803         /* Get vNIC configuration
1804         */
1805         err = enic_get_vnic_config(enic);
1806         if (err) {
1807                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1808                 return err;
1809         }
1810
1811         /* Get available resource counts */
1812         enic_get_res_counts(enic);
1813         if (enic->conf_rq_count == 1) {
1814                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1815                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1816                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1817                 return -EINVAL;
1818         }
1819         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1820         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1821                                enic->conf_cq_count, 8);
1822         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1823                                  enic->conf_intr_count, 8);
1824         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1825                                enic->conf_rq_count, 8);
1826         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1827                                enic->conf_wq_count, 8);
1828         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1829                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1830                 return -1;
1831         }
1832         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1833                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1834                 return -1;
1835         }
1836         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1837                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1838                 return -1;
1839         }
1840         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1841                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1842                 return -1;
1843         }
1844
1845         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1846                                         sizeof(struct rte_ether_addr) *
1847                                         ENIC_UNICAST_PERFECT_FILTERS, 0);
1848         if (!eth_dev->data->mac_addrs) {
1849                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1850                 return -1;
1851         }
1852         rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1853                         eth_dev->data->mac_addrs);
1854
1855         vnic_dev_set_reset_flag(enic->vdev, 0);
1856
1857         LIST_INIT(&enic->flows);
1858
1859         /* set up link status checking */
1860         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1861
1862         enic->overlay_offload = false;
1863         /*
1864          * First, explicitly disable overlay offload as the setting is
1865          * sticky, and resetting vNIC may not disable it.
1866          */
1867         enic_disable_overlay_offload(enic);
1868         /* Then, enable overlay offload according to vNIC flags */
1869         if (!enic->disable_overlay && (enic->vxlan || enic->geneve)) {
1870                 err = enic_enable_overlay_offload(enic);
1871                 if (err) {
1872                         dev_info(NULL, "failed to enable overlay offload\n");
1873                         return err;
1874                 }
1875         }
1876         /*
1877          * Reset the vxlan/geneve port if HW parsing is available. It
1878          * is always enabled regardless of overlay offload
1879          * enable/disable.
1880          */
1881         err = enic_reset_overlay_port(enic);
1882         if (err)
1883                 return err;
1884
1885         if (enic_fm_init(enic))
1886                 dev_warning(enic, "Init of flowman failed.\n");
1887         return 0;
1888 }
1889
1890 static void lock_devcmd(void *priv)
1891 {
1892         struct enic *enic = priv;
1893
1894         rte_spinlock_lock(&enic->devcmd_lock);
1895 }
1896
1897 static void unlock_devcmd(void *priv)
1898 {
1899         struct enic *enic = priv;
1900
1901         rte_spinlock_unlock(&enic->devcmd_lock);
1902 }
1903
1904 int enic_probe(struct enic *enic)
1905 {
1906         struct rte_pci_device *pdev = enic->pdev;
1907         int err = -1;
1908
1909         dev_debug(enic, "Initializing ENIC PMD\n");
1910
1911         /* if this is a secondary process the hardware is already initialized */
1912         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1913                 return 0;
1914
1915         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1916         enic->bar0.len = pdev->mem_resource[0].len;
1917
1918         /* Register vNIC device */
1919         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1920         if (!enic->vdev) {
1921                 dev_err(enic, "vNIC registration failed, aborting\n");
1922                 goto err_out;
1923         }
1924
1925         LIST_INIT(&enic->memzone_list);
1926         rte_spinlock_init(&enic->memzone_list_lock);
1927
1928         vnic_register_cbacks(enic->vdev,
1929                 enic_alloc_consistent,
1930                 enic_free_consistent);
1931
1932         /*
1933          * Allocate the consistent memory for stats upfront so both primary and
1934          * secondary processes can dump stats.
1935          */
1936         err = vnic_dev_alloc_stats_mem(enic->vdev);
1937         if (err) {
1938                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1939                 goto err_out_unregister;
1940         }
1941         /* Issue device open to get device in known state */
1942         err = enic_dev_open(enic);
1943         if (err) {
1944                 dev_err(enic, "vNIC dev open failed, aborting\n");
1945                 goto err_out_unregister;
1946         }
1947
1948         /* Set ingress vlan rewrite mode before vnic initialization */
1949         dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1950                   enic->ig_vlan_rewrite_mode);
1951         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1952                 enic->ig_vlan_rewrite_mode);
1953         if (err) {
1954                 dev_err(enic,
1955                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1956                 goto err_out_dev_close;
1957         }
1958
1959         /* Issue device init to initialize the vnic-to-switch link.
1960          * We'll start with carrier off and wait for link UP
1961          * notification later to turn on carrier.  We don't need
1962          * to wait here for the vnic-to-switch link initialization
1963          * to complete; link UP notification is the indication that
1964          * the process is complete.
1965          */
1966
1967         err = vnic_dev_init(enic->vdev, 0);
1968         if (err) {
1969                 dev_err(enic, "vNIC dev init failed, aborting\n");
1970                 goto err_out_dev_close;
1971         }
1972
1973         err = enic_dev_init(enic);
1974         if (err) {
1975                 dev_err(enic, "Device initialization failed, aborting\n");
1976                 goto err_out_dev_close;
1977         }
1978
1979         /* Use a PF spinlock to serialize devcmd from PF and VF representors */
1980         if (enic->switchdev_mode) {
1981                 rte_spinlock_init(&enic->devcmd_lock);
1982                 vnic_register_lock(enic->vdev, lock_devcmd, unlock_devcmd);
1983         }
1984         return 0;
1985
1986 err_out_dev_close:
1987         vnic_dev_close(enic->vdev);
1988 err_out_unregister:
1989         vnic_dev_unregister(enic->vdev);
1990 err_out:
1991         return err;
1992 }
1993
1994 void enic_remove(struct enic *enic)
1995 {
1996         enic_dev_deinit(enic);
1997         vnic_dev_close(enic->vdev);
1998         vnic_dev_unregister(enic->vdev);
1999 }