drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11
  12 #include <rte_pci.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_ethdev_driver.h>
  19
  20 #include "enic_compat.h"
  21 #include "enic.h"
  22 #include "wq_enet_desc.h"
  23 #include "rq_enet_desc.h"
  24 #include "cq_enet_desc.h"
  25 #include "vnic_enet.h"
  26 #include "vnic_dev.h"
  27 #include "vnic_wq.h"
  28 #include "vnic_rq.h"
  29 #include "vnic_cq.h"
  30 #include "vnic_intr.h"
  31 #include "vnic_nic.h"
  32
  33 static inline int enic_is_sriov_vf(struct enic *enic)
  34 {
  35         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  36 }
  37
  38 static int is_zero_addr(uint8_t *addr)
  39 {
  40         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  41 }
  42
  43 static int is_mcast_addr(uint8_t *addr)
  44 {
  45         return addr[0] & 1;
  46 }
  47
  48 static int is_eth_addr_valid(uint8_t *addr)
  49 {
  50         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  51 }
  52
  53 static void
  54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  55 {
  56         uint16_t i;
  57
  58         if (!rq || !rq->mbuf_ring) {
  59                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  60                 return;
  61         }
  62
  63         for (i = 0; i < rq->ring.desc_count; i++) {
  64                 if (rq->mbuf_ring[i]) {
  65                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  66                         rq->mbuf_ring[i] = NULL;
  67                 }
  68         }
  69 }
  70
  71 static void enic_free_wq_buf(struct rte_mbuf **buf)
  72 {
  73         struct rte_mbuf *mbuf = *buf;
  74
  75         rte_pktmbuf_free_seg(mbuf);
  76         *buf = NULL;
  77 }
  78
  79 static void enic_log_q_error(struct enic *enic)
  80 {
  81         unsigned int i;
  82         uint32_t error_status;
  83
  84         for (i = 0; i < enic->wq_count; i++) {
  85                 error_status = vnic_wq_error_status(&enic->wq[i]);
  86                 if (error_status)
  87                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  88                                 error_status);
  89         }
  90
  91         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  92                 if (!enic->rq[i].in_use)
  93                         continue;
  94                 error_status = vnic_rq_error_status(&enic->rq[i]);
  95                 if (error_status)
  96                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  97                                 error_status);
  98         }
  99 }
 100
 101 static void enic_clear_soft_stats(struct enic *enic)
 102 {
 103         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 104         rte_atomic64_clear(&soft_stats->rx_nombuf);
 105         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 106         rte_atomic64_clear(&soft_stats->tx_oversized);
 107 }
 108
 109 static void enic_init_soft_stats(struct enic *enic)
 110 {
 111         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 112         rte_atomic64_init(&soft_stats->rx_nombuf);
 113         rte_atomic64_init(&soft_stats->rx_packet_errors);
 114         rte_atomic64_init(&soft_stats->tx_oversized);
 115         enic_clear_soft_stats(enic);
 116 }
 117
 118 int enic_dev_stats_clear(struct enic *enic)
 119 {
 120         int ret;
 121
 122         ret = vnic_dev_stats_clear(enic->vdev);
 123         if (ret != 0) {
 124                 dev_err(enic, "Error in clearing stats\n");
 125                 return ret;
 126         }
 127         enic_clear_soft_stats(enic);
 128
 129         return 0;
 130 }
 131
 132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 133 {
 134         struct vnic_stats *stats;
 135         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 136         int64_t rx_truncated;
 137         uint64_t rx_packet_errors;
 138         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 139
 140         if (ret) {
 141                 dev_err(enic, "Error in getting stats\n");
 142                 return ret;
 143         }
 144
 145         /* The number of truncated packets can only be calculated by
 146          * subtracting a hardware counter from error packets received by
 147          * the driver. Note: this causes transient inaccuracies in the
 148          * ipackets count. Also, the length of truncated packets are
 149          * counted in ibytes even though truncated packets are dropped
 150          * which can make ibytes be slightly higher than it should be.
 151          */
 152         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 153         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 154
 155         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 156         r_stats->opackets = stats->tx.tx_frames_ok;
 157
 158         r_stats->ibytes = stats->rx.rx_bytes_ok;
 159         r_stats->obytes = stats->tx.tx_bytes_ok;
 160
 161         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 162         r_stats->oerrors = stats->tx.tx_errors
 163                            + rte_atomic64_read(&soft_stats->tx_oversized);
 164
 165         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 166
 167         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 168         return 0;
 169 }
 170
 171 int enic_del_mac_address(struct enic *enic, int mac_index)
 172 {
 173         struct rte_eth_dev *eth_dev = enic->rte_dev;
 174         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 175
 176         return vnic_dev_del_addr(enic->vdev, mac_addr);
 177 }
 178
 179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 180 {
 181         int err;
 182
 183         if (!is_eth_addr_valid(mac_addr)) {
 184                 dev_err(enic, "invalid mac address\n");
 185                 return -EINVAL;
 186         }
 187
 188         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 189         if (err)
 190                 dev_err(enic, "add mac addr failed\n");
 191         return err;
 192 }
 193
 194 static void
 195 enic_free_rq_buf(struct rte_mbuf **mbuf)
 196 {
 197         if (*mbuf == NULL)
 198                 return;
 199
 200         rte_pktmbuf_free(*mbuf);
 201         *mbuf = NULL;
 202 }
 203
 204 void enic_init_vnic_resources(struct enic *enic)
 205 {
 206         unsigned int error_interrupt_enable = 1;
 207         unsigned int error_interrupt_offset = 0;
 208         unsigned int rxq_interrupt_enable = 0;
 209         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 210         unsigned int index = 0;
 211         unsigned int cq_idx;
 212         struct vnic_rq *data_rq;
 213
 214         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 215                 rxq_interrupt_enable = 1;
 216
 217         for (index = 0; index < enic->rq_count; index++) {
 218                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 219
 220                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 221                         cq_idx,
 222                         error_interrupt_enable,
 223                         error_interrupt_offset);
 224
 225                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 226                 if (data_rq->in_use)
 227                         vnic_rq_init(data_rq,
 228                                      cq_idx,
 229                                      error_interrupt_enable,
 230                                      error_interrupt_offset);
 231
 232                 vnic_cq_init(&enic->cq[cq_idx],
 233                         0 /* flow_control_enable */,
 234                         1 /* color_enable */,
 235                         0 /* cq_head */,
 236                         0 /* cq_tail */,
 237                         1 /* cq_tail_color */,
 238                         rxq_interrupt_enable,
 239                         1 /* cq_entry_enable */,
 240                         0 /* cq_message_enable */,
 241                         rxq_interrupt_offset,
 242                         0 /* cq_message_addr */);
 243                 if (rxq_interrupt_enable)
 244                         rxq_interrupt_offset++;
 245         }
 246
 247         for (index = 0; index < enic->wq_count; index++) {
 248                 vnic_wq_init(&enic->wq[index],
 249                         enic_cq_wq(enic, index),
 250                         error_interrupt_enable,
 251                         error_interrupt_offset);
 252                 /* Compute unsupported ol flags for enic_prep_pkts() */
 253                 enic->wq[index].tx_offload_notsup_mask =
 254                         PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 255
 256                 cq_idx = enic_cq_wq(enic, index);
 257                 vnic_cq_init(&enic->cq[cq_idx],
 258                         0 /* flow_control_enable */,
 259                         1 /* color_enable */,
 260                         0 /* cq_head */,
 261                         0 /* cq_tail */,
 262                         1 /* cq_tail_color */,
 263                         0 /* interrupt_enable */,
 264                         0 /* cq_entry_enable */,
 265                         1 /* cq_message_enable */,
 266                         0 /* interrupt offset */,
 267                         (uint64_t)enic->wq[index].cqmsg_rz->iova);
 268         }
 269
 270         for (index = 0; index < enic->intr_count; index++) {
 271                 vnic_intr_init(&enic->intr[index],
 272                                enic->config.intr_timer_usec,
 273                                enic->config.intr_timer_type,
 274                                /*mask_on_assertion*/1);
 275         }
 276 }
 277
 278
 279 static int
 280 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 281 {
 282         struct rte_mbuf *mb;
 283         struct rq_enet_desc *rqd = rq->ring.descs;
 284         unsigned i;
 285         dma_addr_t dma_addr;
 286         uint32_t max_rx_pkt_len;
 287         uint16_t rq_buf_len;
 288
 289         if (!rq->in_use)
 290                 return 0;
 291
 292         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 293                   rq->ring.desc_count);
 294
 295         /*
 296          * If *not* using scatter and the mbuf size is greater than the
 297          * requested max packet size (max_rx_pkt_len), then reduce the
 298          * posted buffer size to max_rx_pkt_len. HW still receives packets
 299          * larger than max_rx_pkt_len, but they will be truncated, which we
 300          * drop in the rx handler. Not ideal, but better than returning
 301          * large packets when the user is not expecting them.
 302          */
 303         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 304         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 305         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 306                 rq_buf_len = max_rx_pkt_len;
 307         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 308                 mb = rte_mbuf_raw_alloc(rq->mp);
 309                 if (mb == NULL) {
 310                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 311                         (unsigned)rq->index);
 312                         return -ENOMEM;
 313                 }
 314
 315                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 316                 dma_addr = (dma_addr_t)(mb->buf_iova
 317                            + RTE_PKTMBUF_HEADROOM);
 318                 rq_enet_desc_enc(rqd, dma_addr,
 319                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 320                                 : RQ_ENET_TYPE_NOT_SOP),
 321                                 rq_buf_len);
 322                 rq->mbuf_ring[i] = mb;
 323         }
 324         /*
 325          * Do not post the buffers to the NIC until we enable the RQ via
 326          * enic_start_rq().
 327          */
 328         rq->need_initial_post = true;
 329         /* Initialize fetch index while RQ is disabled */
 330         iowrite32(0, &rq->ctrl->fetch_index);
 331         return 0;
 332 }
 333
 334 /*
 335  * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
 336  * allocated the buffers and filled the RQ descriptor ring. Just need to push
 337  * the post index to the NIC.
 338  */
 339 static void
 340 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 341 {
 342         if (!rq->in_use || !rq->need_initial_post)
 343                 return;
 344
 345         /* make sure all prior writes are complete before doing the PIO write */
 346         rte_rmb();
 347
 348         /* Post all but the last buffer to VIC. */
 349         rq->posted_index = rq->ring.desc_count - 1;
 350
 351         rq->rx_nb_hold = 0;
 352
 353         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 354                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 355         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 356         rte_rmb();
 357         rq->need_initial_post = false;
 358 }
 359
 360 void *
 361 enic_alloc_consistent(void *priv, size_t size,
 362         dma_addr_t *dma_handle, uint8_t *name)
 363 {
 364         void *vaddr;
 365         const struct rte_memzone *rz;
 366         *dma_handle = 0;
 367         struct enic *enic = (struct enic *)priv;
 368         struct enic_memzone_entry *mze;
 369
 370         rz = rte_memzone_reserve_aligned((const char *)name, size,
 371                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
 372         if (!rz) {
 373                 pr_err("%s : Failed to allocate memory requested for %s\n",
 374                         __func__, name);
 375                 return NULL;
 376         }
 377
 378         vaddr = rz->addr;
 379         *dma_handle = (dma_addr_t)rz->iova;
 380
 381         mze = rte_malloc("enic memzone entry",
 382                          sizeof(struct enic_memzone_entry), 0);
 383
 384         if (!mze) {
 385                 pr_err("%s : Failed to allocate memory for memzone list\n",
 386                        __func__);
 387                 rte_memzone_free(rz);
 388                 return NULL;
 389         }
 390
 391         mze->rz = rz;
 392
 393         rte_spinlock_lock(&enic->memzone_list_lock);
 394         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 395         rte_spinlock_unlock(&enic->memzone_list_lock);
 396
 397         return vaddr;
 398 }
 399
 400 void
 401 enic_free_consistent(void *priv,
 402                      __rte_unused size_t size,
 403                      void *vaddr,
 404                      dma_addr_t dma_handle)
 405 {
 406         struct enic_memzone_entry *mze;
 407         struct enic *enic = (struct enic *)priv;
 408
 409         rte_spinlock_lock(&enic->memzone_list_lock);
 410         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 411                 if (mze->rz->addr == vaddr &&
 412                     mze->rz->iova == dma_handle)
 413                         break;
 414         }
 415         if (mze == NULL) {
 416                 rte_spinlock_unlock(&enic->memzone_list_lock);
 417                 dev_warning(enic,
 418                             "Tried to free memory, but couldn't find it in the memzone list\n");
 419                 return;
 420         }
 421         LIST_REMOVE(mze, entries);
 422         rte_spinlock_unlock(&enic->memzone_list_lock);
 423         rte_memzone_free(mze->rz);
 424         rte_free(mze);
 425 }
 426
 427 int enic_link_update(struct rte_eth_dev *eth_dev)
 428 {
 429         struct enic *enic = pmd_priv(eth_dev);
 430         struct rte_eth_link link;
 431
 432         memset(&link, 0, sizeof(link));
 433         link.link_status = enic_get_link_status(enic);
 434         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 435         link.link_speed = vnic_dev_port_speed(enic->vdev);
 436
 437         return rte_eth_linkstatus_set(eth_dev, &link);
 438 }
 439
 440 static void
 441 enic_intr_handler(void *arg)
 442 {
 443         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 444         struct enic *enic = pmd_priv(dev);
 445
 446         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 447
 448         enic_link_update(dev);
 449         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 450         enic_log_q_error(enic);
 451         /* Re-enable irq in case of INTx */
 452         rte_intr_ack(&enic->pdev->intr_handle);
 453 }
 454
 455 static int enic_rxq_intr_init(struct enic *enic)
 456 {
 457         struct rte_intr_handle *intr_handle;
 458         uint32_t rxq_intr_count, i;
 459         int err;
 460
 461         intr_handle = enic->rte_dev->intr_handle;
 462         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 463                 return 0;
 464         /*
 465          * Rx queue interrupts only work when we have MSI-X interrupts,
 466          * one per queue. Sharing one interrupt is technically
 467          * possible with VIC, but it is not worth the complications it brings.
 468          */
 469         if (!rte_intr_cap_multiple(intr_handle)) {
 470                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 471                         " (vfio-pci driver)\n");
 472                 return -ENOTSUP;
 473         }
 474         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 475         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 476         if (err) {
 477                 dev_err(enic, "Failed to enable event fds for Rx queue"
 478                         " interrupts\n");
 479                 return err;
 480         }
 481         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 482                                             rxq_intr_count * sizeof(int), 0);
 483         if (intr_handle->intr_vec == NULL) {
 484                 dev_err(enic, "Failed to allocate intr_vec\n");
 485                 return -ENOMEM;
 486         }
 487         for (i = 0; i < rxq_intr_count; i++)
 488                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 489         return 0;
 490 }
 491
 492 static void enic_rxq_intr_deinit(struct enic *enic)
 493 {
 494         struct rte_intr_handle *intr_handle;
 495
 496         intr_handle = enic->rte_dev->intr_handle;
 497         rte_intr_efd_disable(intr_handle);
 498         if (intr_handle->intr_vec != NULL) {
 499                 rte_free(intr_handle->intr_vec);
 500                 intr_handle->intr_vec = NULL;
 501         }
 502 }
 503
 504 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 505 {
 506         struct wq_enet_desc *desc;
 507         struct vnic_wq *wq;
 508         unsigned int i;
 509
 510         /*
 511          * Fill WQ descriptor fields that never change. Every descriptor is
 512          * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
 513          * descriptors (i.e. request one completion update every 32 packets).
 514          */
 515         wq = &enic->wq[queue_idx];
 516         desc = (struct wq_enet_desc *)wq->ring.descs;
 517         for (i = 0; i < wq->ring.desc_count; i++, desc++) {
 518                 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
 519                 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
 520                         desc->header_length_flags |=
 521                                 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
 522         }
 523 }
 524
 525 /*
 526  * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
 527  * used when that file is not compiled.
 528  */
 529 __rte_weak bool
 530 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
 531 {
 532         return false;
 533 }
 534
 535 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
 536 {
 537         struct enic *enic = pmd_priv(eth_dev);
 538
 539         /*
 540          * Preference order:
 541          * 1. The vectorized handler if possible and requested.
 542          * 2. The non-scatter, simplified handler if scatter Rx is not used.
 543          * 3. The default handler as a fallback.
 544          */
 545         if (enic_use_vector_rx_handler(eth_dev))
 546                 return;
 547         if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 548                 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
 549                 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
 550         } else {
 551                 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
 552                 eth_dev->rx_pkt_burst = &enic_recv_pkts;
 553         }
 554 }
 555
 556 /* Secondary process uses this to set the Tx handler */
 557 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
 558 {
 559         struct enic *enic = pmd_priv(eth_dev);
 560
 561         if (enic->use_simple_tx_handler) {
 562                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 563                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 564         } else {
 565                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 566                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 567         }
 568 }
 569
 570 int enic_enable(struct enic *enic)
 571 {
 572         unsigned int index;
 573         int err;
 574         struct rte_eth_dev *eth_dev = enic->rte_dev;
 575         uint64_t simple_tx_offloads;
 576         uintptr_t p;
 577
 578         if (enic->enable_avx2_rx) {
 579                 struct rte_mbuf mb_def = { .buf_addr = 0 };
 580
 581                 /*
 582                  * mbuf_initializer contains const-after-init fields of
 583                  * receive mbufs (i.e. 64 bits of fields from rearm_data).
 584                  * It is currently used by the vectorized handler.
 585                  */
 586                 mb_def.nb_segs = 1;
 587                 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 588                 mb_def.port = enic->port_id;
 589                 rte_mbuf_refcnt_set(&mb_def, 1);
 590                 rte_compiler_barrier();
 591                 p = (uintptr_t)&mb_def.rearm_data;
 592                 enic->mbuf_initializer = *(uint64_t *)p;
 593         }
 594
 595         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 596         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 597
 598         /* vnic notification of link status has already been turned on in
 599          * enic_dev_init() which is called during probe time.  Here we are
 600          * just turning on interrupt vector 0 if needed.
 601          */
 602         if (eth_dev->data->dev_conf.intr_conf.lsc)
 603                 vnic_dev_notify_set(enic->vdev, 0);
 604
 605         err = enic_rxq_intr_init(enic);
 606         if (err)
 607                 return err;
 608         if (enic_clsf_init(enic))
 609                 dev_warning(enic, "Init of hash table for clsf failed."\
 610                         "Flow director feature will not work\n");
 611
 612         if (enic_fm_init(enic))
 613                 dev_warning(enic, "Init of flowman failed.\n");
 614
 615         for (index = 0; index < enic->rq_count; index++) {
 616                 err = enic_alloc_rx_queue_mbufs(enic,
 617                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 618                 if (err) {
 619                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 620                         return err;
 621                 }
 622                 err = enic_alloc_rx_queue_mbufs(enic,
 623                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 624                 if (err) {
 625                         /* release the allocated mbufs for the sop rq*/
 626                         enic_rxmbuf_queue_release(enic,
 627                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 628
 629                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 630                         return err;
 631                 }
 632         }
 633
 634         /*
 635          * Use the simple TX handler if possible. Only checksum offloads
 636          * and vlan insertion are supported.
 637          */
 638         simple_tx_offloads = enic->tx_offload_capa &
 639                 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 640                  DEV_TX_OFFLOAD_VLAN_INSERT |
 641                  DEV_TX_OFFLOAD_IPV4_CKSUM |
 642                  DEV_TX_OFFLOAD_UDP_CKSUM |
 643                  DEV_TX_OFFLOAD_TCP_CKSUM);
 644         if ((eth_dev->data->dev_conf.txmode.offloads &
 645              ~simple_tx_offloads) == 0) {
 646                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 647                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 648                 for (index = 0; index < enic->wq_count; index++)
 649                         enic_prep_wq_for_simple_tx(enic, index);
 650                 enic->use_simple_tx_handler = 1;
 651         } else {
 652                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 653                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 654         }
 655
 656         enic_pick_rx_handler(eth_dev);
 657
 658         for (index = 0; index < enic->wq_count; index++)
 659                 enic_start_wq(enic, index);
 660         for (index = 0; index < enic->rq_count; index++)
 661                 enic_start_rq(enic, index);
 662
 663         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 664
 665         vnic_dev_enable_wait(enic->vdev);
 666
 667         /* Register and enable error interrupt */
 668         rte_intr_callback_register(&(enic->pdev->intr_handle),
 669                 enic_intr_handler, (void *)enic->rte_dev);
 670
 671         rte_intr_enable(&(enic->pdev->intr_handle));
 672         /* Unmask LSC interrupt */
 673         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 674
 675         return 0;
 676 }
 677
 678 int enic_alloc_intr_resources(struct enic *enic)
 679 {
 680         int err;
 681         unsigned int i;
 682
 683         dev_info(enic, "vNIC resources used:  "\
 684                 "wq %d rq %d cq %d intr %d\n",
 685                 enic->wq_count, enic_vnic_rq_count(enic),
 686                 enic->cq_count, enic->intr_count);
 687
 688         for (i = 0; i < enic->intr_count; i++) {
 689                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 690                 if (err) {
 691                         enic_free_vnic_resources(enic);
 692                         return err;
 693                 }
 694         }
 695         return 0;
 696 }
 697
 698 void enic_free_rq(void *rxq)
 699 {
 700         struct vnic_rq *rq_sop, *rq_data;
 701         struct enic *enic;
 702
 703         if (rxq == NULL)
 704                 return;
 705
 706         rq_sop = (struct vnic_rq *)rxq;
 707         enic = vnic_dev_priv(rq_sop->vdev);
 708         rq_data = &enic->rq[rq_sop->data_queue_idx];
 709
 710         if (rq_sop->free_mbufs) {
 711                 struct rte_mbuf **mb;
 712                 int i;
 713
 714                 mb = rq_sop->free_mbufs;
 715                 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
 716                      i < ENIC_RX_BURST_MAX; i++)
 717                         rte_pktmbuf_free(mb[i]);
 718                 rte_free(rq_sop->free_mbufs);
 719                 rq_sop->free_mbufs = NULL;
 720                 rq_sop->num_free_mbufs = 0;
 721         }
 722
 723         enic_rxmbuf_queue_release(enic, rq_sop);
 724         if (rq_data->in_use)
 725                 enic_rxmbuf_queue_release(enic, rq_data);
 726
 727         rte_free(rq_sop->mbuf_ring);
 728         if (rq_data->in_use)
 729                 rte_free(rq_data->mbuf_ring);
 730
 731         rq_sop->mbuf_ring = NULL;
 732         rq_data->mbuf_ring = NULL;
 733
 734         vnic_rq_free(rq_sop);
 735         if (rq_data->in_use)
 736                 vnic_rq_free(rq_data);
 737
 738         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 739
 740         rq_sop->in_use = 0;
 741         rq_data->in_use = 0;
 742 }
 743
 744 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 745 {
 746         struct rte_eth_dev_data *data = enic->dev_data;
 747         vnic_wq_enable(&enic->wq[queue_idx]);
 748         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 749 }
 750
 751 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 752 {
 753         struct rte_eth_dev_data *data = enic->dev_data;
 754         int ret;
 755
 756         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 757         if (ret)
 758                 return ret;
 759
 760         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 761         return 0;
 762 }
 763
 764 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 765 {
 766         struct rte_eth_dev_data *data = enic->dev_data;
 767         struct vnic_rq *rq_sop;
 768         struct vnic_rq *rq_data;
 769         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 770         rq_data = &enic->rq[rq_sop->data_queue_idx];
 771
 772         if (rq_data->in_use) {
 773                 vnic_rq_enable(rq_data);
 774                 enic_initial_post_rx(enic, rq_data);
 775         }
 776         rte_mb();
 777         vnic_rq_enable(rq_sop);
 778         enic_initial_post_rx(enic, rq_sop);
 779         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 780 }
 781
 782 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 783 {
 784         struct rte_eth_dev_data *data = enic->dev_data;
 785         int ret1 = 0, ret2 = 0;
 786         struct vnic_rq *rq_sop;
 787         struct vnic_rq *rq_data;
 788         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 789         rq_data = &enic->rq[rq_sop->data_queue_idx];
 790
 791         ret2 = vnic_rq_disable(rq_sop);
 792         rte_mb();
 793         if (rq_data->in_use)
 794                 ret1 = vnic_rq_disable(rq_data);
 795
 796         if (ret2)
 797                 return ret2;
 798         else if (ret1)
 799                 return ret1;
 800
 801         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 802         return 0;
 803 }
 804
 805 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 806         unsigned int socket_id, struct rte_mempool *mp,
 807         uint16_t nb_desc, uint16_t free_thresh)
 808 {
 809         int rc;
 810         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 811         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 812         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 813         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 814         unsigned int mbuf_size, mbufs_per_pkt;
 815         unsigned int nb_sop_desc, nb_data_desc;
 816         uint16_t min_sop, max_sop, min_data, max_data;
 817         uint32_t max_rx_pkt_len;
 818
 819         rq_sop->is_sop = 1;
 820         rq_sop->data_queue_idx = data_queue_idx;
 821         rq_data->is_sop = 0;
 822         rq_data->data_queue_idx = 0;
 823         rq_sop->socket_id = socket_id;
 824         rq_sop->mp = mp;
 825         rq_data->socket_id = socket_id;
 826         rq_data->mp = mp;
 827         rq_sop->in_use = 1;
 828         rq_sop->rx_free_thresh = free_thresh;
 829         rq_data->rx_free_thresh = free_thresh;
 830         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 831                   free_thresh);
 832
 833         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 834                                RTE_PKTMBUF_HEADROOM);
 835         /* max_rx_pkt_len includes the ethernet header and CRC. */
 836         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 837
 838         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 839             DEV_RX_OFFLOAD_SCATTER) {
 840                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 841                 /* ceil((max pkt len)/mbuf_size) */
 842                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 843         } else {
 844                 dev_info(enic, "Scatter rx mode disabled\n");
 845                 mbufs_per_pkt = 1;
 846                 if (max_rx_pkt_len > mbuf_size) {
 847                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 848                                     " larger than the mbuf size (%u), and"
 849                                     " scatter is disabled. Larger packets will"
 850                                     " be truncated.\n",
 851                                     max_rx_pkt_len, mbuf_size);
 852                 }
 853         }
 854
 855         if (mbufs_per_pkt > 1) {
 856                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 857                 rq_sop->data_queue_enable = 1;
 858                 rq_data->in_use = 1;
 859                 /*
 860                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 861                  * receives packet sizes up to the "max" MTU.
 862                  * If not using scatter, we can achieve the effect of dropping
 863                  * larger packets by reducing the size of posted buffers.
 864                  * See enic_alloc_rx_queue_mbufs().
 865                  */
 866                 if (max_rx_pkt_len <
 867                     enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
 868                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 869                                     " when scatter rx mode is in use.\n");
 870                 }
 871         } else {
 872                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 873                          queue_idx);
 874                 rq_sop->data_queue_enable = 0;
 875                 rq_data->in_use = 0;
 876         }
 877
 878         /* number of descriptors have to be a multiple of 32 */
 879         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
 880         nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 881
 882         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 883         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 884
 885         if (mbufs_per_pkt > 1) {
 886                 min_sop = ENIC_RX_BURST_MAX;
 887                 max_sop = ((enic->config.rq_desc_count /
 888                             (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 889                 min_data = min_sop * (mbufs_per_pkt - 1);
 890                 max_data = enic->config.rq_desc_count;
 891         } else {
 892                 min_sop = ENIC_RX_BURST_MAX;
 893                 max_sop = enic->config.rq_desc_count;
 894                 min_data = 0;
 895                 max_data = 0;
 896         }
 897
 898         if (nb_desc < (min_sop + min_data)) {
 899                 dev_warning(enic,
 900                             "Number of rx descs too low, adjusting to minimum\n");
 901                 nb_sop_desc = min_sop;
 902                 nb_data_desc = min_data;
 903         } else if (nb_desc > (max_sop + max_data)) {
 904                 dev_warning(enic,
 905                             "Number of rx_descs too high, adjusting to maximum\n");
 906                 nb_sop_desc = max_sop;
 907                 nb_data_desc = max_data;
 908         }
 909         if (mbufs_per_pkt > 1) {
 910                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 911                          " rx descriptor range is %u to %u\n",
 912                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 913                          max_sop + max_data);
 914         }
 915         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 916                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 917
 918         /* Allocate sop queue resources */
 919         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 920                 nb_sop_desc, sizeof(struct rq_enet_desc));
 921         if (rc) {
 922                 dev_err(enic, "error in allocation of sop rq\n");
 923                 goto err_exit;
 924         }
 925         nb_sop_desc = rq_sop->ring.desc_count;
 926
 927         if (rq_data->in_use) {
 928                 /* Allocate data queue resources */
 929                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 930                                    nb_data_desc,
 931                                    sizeof(struct rq_enet_desc));
 932                 if (rc) {
 933                         dev_err(enic, "error in allocation of data rq\n");
 934                         goto err_free_rq_sop;
 935                 }
 936                 nb_data_desc = rq_data->ring.desc_count;
 937         }
 938         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 939                            socket_id, nb_sop_desc + nb_data_desc,
 940                            sizeof(struct cq_enet_rq_desc));
 941         if (rc) {
 942                 dev_err(enic, "error in allocation of cq for rq\n");
 943                 goto err_free_rq_data;
 944         }
 945
 946         /* Allocate the mbuf rings */
 947         rq_sop->mbuf_ring = (struct rte_mbuf **)
 948                 rte_zmalloc_socket("rq->mbuf_ring",
 949                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 950                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 951         if (rq_sop->mbuf_ring == NULL)
 952                 goto err_free_cq;
 953
 954         if (rq_data->in_use) {
 955                 rq_data->mbuf_ring = (struct rte_mbuf **)
 956                         rte_zmalloc_socket("rq->mbuf_ring",
 957                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 958                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 959                 if (rq_data->mbuf_ring == NULL)
 960                         goto err_free_sop_mbuf;
 961         }
 962
 963         rq_sop->free_mbufs = (struct rte_mbuf **)
 964                 rte_zmalloc_socket("rq->free_mbufs",
 965                                    sizeof(struct rte_mbuf *) *
 966                                    ENIC_RX_BURST_MAX,
 967                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 968         if (rq_sop->free_mbufs == NULL)
 969                 goto err_free_data_mbuf;
 970         rq_sop->num_free_mbufs = 0;
 971
 972         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 973
 974         return 0;
 975
 976 err_free_data_mbuf:
 977         rte_free(rq_data->mbuf_ring);
 978 err_free_sop_mbuf:
 979         rte_free(rq_sop->mbuf_ring);
 980 err_free_cq:
 981         /* cleanup on error */
 982         vnic_cq_free(&enic->cq[queue_idx]);
 983 err_free_rq_data:
 984         if (rq_data->in_use)
 985                 vnic_rq_free(rq_data);
 986 err_free_rq_sop:
 987         vnic_rq_free(rq_sop);
 988 err_exit:
 989         return -ENOMEM;
 990 }
 991
 992 void enic_free_wq(void *txq)
 993 {
 994         struct vnic_wq *wq;
 995         struct enic *enic;
 996
 997         if (txq == NULL)
 998                 return;
 999
1000         wq = (struct vnic_wq *)txq;
1001         enic = vnic_dev_priv(wq->vdev);
1002         rte_memzone_free(wq->cqmsg_rz);
1003         vnic_wq_free(wq);
1004         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1005 }
1006
1007 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1008         unsigned int socket_id, uint16_t nb_desc)
1009 {
1010         int err;
1011         struct vnic_wq *wq = &enic->wq[queue_idx];
1012         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
1013         char name[RTE_MEMZONE_NAMESIZE];
1014         static int instance;
1015
1016         wq->socket_id = socket_id;
1017         /*
1018          * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1019          * print an info message for diagnostics.
1020          */
1021         dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1022
1023         /* Allocate queue resources */
1024         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1025                 nb_desc,
1026                 sizeof(struct wq_enet_desc));
1027         if (err) {
1028                 dev_err(enic, "error in allocation of wq\n");
1029                 return err;
1030         }
1031
1032         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1033                 socket_id, nb_desc,
1034                 sizeof(struct cq_enet_wq_desc));
1035         if (err) {
1036                 vnic_wq_free(wq);
1037                 dev_err(enic, "error in allocation of cq for wq\n");
1038         }
1039
1040         /* setup up CQ message */
1041         snprintf((char *)name, sizeof(name),
1042                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1043                 instance++);
1044
1045         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1046                         sizeof(uint32_t), SOCKET_ID_ANY,
1047                         RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
1048         if (!wq->cqmsg_rz)
1049                 return -ENOMEM;
1050
1051         return err;
1052 }
1053
1054 int enic_disable(struct enic *enic)
1055 {
1056         unsigned int i;
1057         int err;
1058
1059         for (i = 0; i < enic->intr_count; i++) {
1060                 vnic_intr_mask(&enic->intr[i]);
1061                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1062         }
1063         enic_rxq_intr_deinit(enic);
1064         rte_intr_disable(&enic->pdev->intr_handle);
1065         rte_intr_callback_unregister(&enic->pdev->intr_handle,
1066                                      enic_intr_handler,
1067                                      (void *)enic->rte_dev);
1068
1069         vnic_dev_disable(enic->vdev);
1070
1071         enic_clsf_destroy(enic);
1072         enic_fm_destroy(enic);
1073
1074         if (!enic_is_sriov_vf(enic))
1075                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1076
1077         for (i = 0; i < enic->wq_count; i++) {
1078                 err = vnic_wq_disable(&enic->wq[i]);
1079                 if (err)
1080                         return err;
1081         }
1082         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1083                 if (enic->rq[i].in_use) {
1084                         err = vnic_rq_disable(&enic->rq[i]);
1085                         if (err)
1086                                 return err;
1087                 }
1088         }
1089
1090         /* If we were using interrupts, set the interrupt vector to -1
1091          * to disable interrupts.  We are not disabling link notifcations,
1092          * though, as we want the polling of link status to continue working.
1093          */
1094         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1095                 vnic_dev_notify_set(enic->vdev, -1);
1096
1097         vnic_dev_set_reset_flag(enic->vdev, 1);
1098
1099         for (i = 0; i < enic->wq_count; i++)
1100                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1101
1102         for (i = 0; i < enic_vnic_rq_count(enic); i++)
1103                 if (enic->rq[i].in_use)
1104                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1105         for (i = 0; i < enic->cq_count; i++)
1106                 vnic_cq_clean(&enic->cq[i]);
1107         for (i = 0; i < enic->intr_count; i++)
1108                 vnic_intr_clean(&enic->intr[i]);
1109
1110         return 0;
1111 }
1112
1113 static int enic_dev_wait(struct vnic_dev *vdev,
1114         int (*start)(struct vnic_dev *, int),
1115         int (*finished)(struct vnic_dev *, int *),
1116         int arg)
1117 {
1118         int done;
1119         int err;
1120         int i;
1121
1122         err = start(vdev, arg);
1123         if (err)
1124                 return err;
1125
1126         /* Wait for func to complete...2 seconds max */
1127         for (i = 0; i < 2000; i++) {
1128                 err = finished(vdev, &done);
1129                 if (err)
1130                         return err;
1131                 if (done)
1132                         return 0;
1133                 usleep(1000);
1134         }
1135         return -ETIMEDOUT;
1136 }
1137
1138 static int enic_dev_open(struct enic *enic)
1139 {
1140         int err;
1141         int flags = CMD_OPENF_IG_DESCCACHE;
1142
1143         err = enic_dev_wait(enic->vdev, vnic_dev_open,
1144                 vnic_dev_open_done, flags);
1145         if (err)
1146                 dev_err(enic_get_dev(enic),
1147                         "vNIC device open failed, err %d\n", err);
1148
1149         return err;
1150 }
1151
1152 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1153 {
1154         dma_addr_t rss_key_buf_pa;
1155         union vnic_rss_key *rss_key_buf_va = NULL;
1156         int err, i;
1157         uint8_t name[RTE_MEMZONE_NAMESIZE];
1158
1159         RTE_ASSERT(user_key != NULL);
1160         snprintf((char *)name, sizeof(name), "rss_key-%s", enic->bdf_name);
1161         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1162                 &rss_key_buf_pa, name);
1163         if (!rss_key_buf_va)
1164                 return -ENOMEM;
1165
1166         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1167                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1168
1169         err = enic_set_rss_key(enic,
1170                 rss_key_buf_pa,
1171                 sizeof(union vnic_rss_key));
1172
1173         /* Save for later queries */
1174         if (!err) {
1175                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1176                            sizeof(union vnic_rss_key));
1177         }
1178         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1179                 rss_key_buf_va, rss_key_buf_pa);
1180
1181         return err;
1182 }
1183
1184 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1185 {
1186         dma_addr_t rss_cpu_buf_pa;
1187         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1188         int err;
1189         uint8_t name[RTE_MEMZONE_NAMESIZE];
1190
1191         snprintf((char *)name, sizeof(name), "rss_cpu-%s", enic->bdf_name);
1192         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1193                 &rss_cpu_buf_pa, name);
1194         if (!rss_cpu_buf_va)
1195                 return -ENOMEM;
1196
1197         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1198
1199         err = enic_set_rss_cpu(enic,
1200                 rss_cpu_buf_pa,
1201                 sizeof(union vnic_rss_cpu));
1202
1203         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1204                 rss_cpu_buf_va, rss_cpu_buf_pa);
1205
1206         /* Save for later queries */
1207         if (!err)
1208                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1209         return err;
1210 }
1211
1212 static int enic_set_niccfg(struct enic *enic, uint8_t rss_default_cpu,
1213         uint8_t rss_hash_type, uint8_t rss_hash_bits, uint8_t rss_base_cpu,
1214         uint8_t rss_enable)
1215 {
1216         const uint8_t tso_ipid_split_en = 0;
1217         int err;
1218
1219         err = enic_set_nic_cfg(enic,
1220                 rss_default_cpu, rss_hash_type,
1221                 rss_hash_bits, rss_base_cpu,
1222                 rss_enable, tso_ipid_split_en,
1223                 enic->ig_vlan_strip_en);
1224
1225         return err;
1226 }
1227
1228 /* Initialize RSS with defaults, called from dev_configure */
1229 int enic_init_rss_nic_cfg(struct enic *enic)
1230 {
1231         static uint8_t default_rss_key[] = {
1232                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1233                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1234                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1235                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1236         };
1237         struct rte_eth_rss_conf rss_conf;
1238         union vnic_rss_cpu rss_cpu;
1239         int ret, i;
1240
1241         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1242         /*
1243          * If setting key for the first time, and the user gives us none, then
1244          * push the default key to NIC.
1245          */
1246         if (rss_conf.rss_key == NULL) {
1247                 rss_conf.rss_key = default_rss_key;
1248                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1249         }
1250         ret = enic_set_rss_conf(enic, &rss_conf);
1251         if (ret) {
1252                 dev_err(enic, "Failed to configure RSS\n");
1253                 return ret;
1254         }
1255         if (enic->rss_enable) {
1256                 /* If enabling RSS, use the default reta */
1257                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1258                         rss_cpu.cpu[i / 4].b[i % 4] =
1259                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1260                 }
1261                 ret = enic_set_rss_reta(enic, &rss_cpu);
1262                 if (ret)
1263                         dev_err(enic, "Failed to set RSS indirection table\n");
1264         }
1265         return ret;
1266 }
1267
1268 int enic_setup_finish(struct enic *enic)
1269 {
1270         enic_init_soft_stats(enic);
1271
1272         /* Default conf */
1273         vnic_dev_packet_filter(enic->vdev,
1274                 1 /* directed  */,
1275                 1 /* multicast */,
1276                 1 /* broadcast */,
1277                 0 /* promisc   */,
1278                 1 /* allmulti  */);
1279
1280         enic->promisc = 0;
1281         enic->allmulti = 1;
1282
1283         return 0;
1284 }
1285
1286 static int enic_rss_conf_valid(struct enic *enic,
1287                                struct rte_eth_rss_conf *rss_conf)
1288 {
1289         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1290         if (enic->flow_type_rss_offloads == 0)
1291                 return 0;
1292         if (rss_conf->rss_key != NULL &&
1293             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1294                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1295                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1296                 return -EINVAL;
1297         }
1298         if (rss_conf->rss_hf != 0 &&
1299             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1300                 dev_err(enic, "Given rss_hf contains none of the supported"
1301                         " types\n");
1302                 return -EINVAL;
1303         }
1304         return 0;
1305 }
1306
1307 /* Set hash type and key according to rss_conf */
1308 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1309 {
1310         struct rte_eth_dev *eth_dev;
1311         uint64_t rss_hf;
1312         uint8_t rss_hash_type;
1313         uint8_t rss_enable;
1314         int ret;
1315
1316         RTE_ASSERT(rss_conf != NULL);
1317         ret = enic_rss_conf_valid(enic, rss_conf);
1318         if (ret) {
1319                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1320                 return ret;
1321         }
1322
1323         eth_dev = enic->rte_dev;
1324         rss_hash_type = 0;
1325         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1326         if (enic->rq_count > 1 &&
1327             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1328             rss_hf != 0) {
1329                 rss_enable = 1;
1330                 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1331                               ETH_RSS_NONFRAG_IPV4_OTHER))
1332                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1333                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1334                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1335                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1336                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1337                         if (enic->udp_rss_weak) {
1338                                 /*
1339                                  * 'TCP' is not a typo. The "weak" version of
1340                                  * UDP RSS requires both the TCP and UDP bits
1341                                  * be set. It does enable TCP RSS as well.
1342                                  */
1343                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1344                         }
1345                 }
1346                 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1347                               ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1348                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1349                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1350                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1351                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1352                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1353                         if (enic->udp_rss_weak)
1354                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1355                 }
1356         } else {
1357                 rss_enable = 0;
1358                 rss_hf = 0;
1359         }
1360
1361         /* Set the hash key if provided */
1362         if (rss_enable && rss_conf->rss_key) {
1363                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1364                 if (ret) {
1365                         dev_err(enic, "Failed to set RSS key\n");
1366                         return ret;
1367                 }
1368         }
1369
1370         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1371                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1372                               rss_enable);
1373         if (!ret) {
1374                 enic->rss_hf = rss_hf;
1375                 enic->rss_hash_type = rss_hash_type;
1376                 enic->rss_enable = rss_enable;
1377         } else {
1378                 dev_err(enic, "Failed to update RSS configurations."
1379                         " hash=0x%x\n", rss_hash_type);
1380         }
1381         return ret;
1382 }
1383
1384 int enic_set_vlan_strip(struct enic *enic)
1385 {
1386         /*
1387          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1388          * together. So, re-do niccfg, preserving the current RSS settings.
1389          */
1390         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1391                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1392                                enic->rss_enable);
1393 }
1394
1395 int enic_add_packet_filter(struct enic *enic)
1396 {
1397         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1398         return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1399                 enic->promisc, enic->allmulti);
1400 }
1401
1402 int enic_get_link_status(struct enic *enic)
1403 {
1404         return vnic_dev_link_status(enic->vdev);
1405 }
1406
1407 static void enic_dev_deinit(struct enic *enic)
1408 {
1409         /* stop link status checking */
1410         vnic_dev_notify_unset(enic->vdev);
1411
1412         /* mac_addrs is freed by rte_eth_dev_release_port() */
1413         rte_free(enic->cq);
1414         rte_free(enic->intr);
1415         rte_free(enic->rq);
1416         rte_free(enic->wq);
1417 }
1418
1419
1420 int enic_set_vnic_res(struct enic *enic)
1421 {
1422         struct rte_eth_dev *eth_dev = enic->rte_dev;
1423         int rc = 0;
1424         unsigned int required_rq, required_wq, required_cq, required_intr;
1425
1426         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1427         required_rq = eth_dev->data->nb_rx_queues * 2;
1428         required_wq = eth_dev->data->nb_tx_queues;
1429         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1430         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1431         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1432                 required_intr += eth_dev->data->nb_rx_queues;
1433         }
1434
1435         if (enic->conf_rq_count < required_rq) {
1436                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1437                         eth_dev->data->nb_rx_queues,
1438                         required_rq, enic->conf_rq_count);
1439                 rc = -EINVAL;
1440         }
1441         if (enic->conf_wq_count < required_wq) {
1442                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1443                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1444                 rc = -EINVAL;
1445         }
1446
1447         if (enic->conf_cq_count < required_cq) {
1448                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1449                         required_cq, enic->conf_cq_count);
1450                 rc = -EINVAL;
1451         }
1452         if (enic->conf_intr_count < required_intr) {
1453                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1454                         " interrupts. Required:%u, Configured:%u\n",
1455                         required_intr, enic->conf_intr_count);
1456                 rc = -EINVAL;
1457         }
1458
1459         if (rc == 0) {
1460                 enic->rq_count = eth_dev->data->nb_rx_queues;
1461                 enic->wq_count = eth_dev->data->nb_tx_queues;
1462                 enic->cq_count = enic->rq_count + enic->wq_count;
1463                 enic->intr_count = required_intr;
1464         }
1465
1466         return rc;
1467 }
1468
1469 /* Initialize the completion queue for an RQ */
1470 static int
1471 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1472 {
1473         struct vnic_rq *sop_rq, *data_rq;
1474         unsigned int cq_idx;
1475         int rc = 0;
1476
1477         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1478         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1479         cq_idx = rq_idx;
1480
1481         vnic_cq_clean(&enic->cq[cq_idx]);
1482         vnic_cq_init(&enic->cq[cq_idx],
1483                      0 /* flow_control_enable */,
1484                      1 /* color_enable */,
1485                      0 /* cq_head */,
1486                      0 /* cq_tail */,
1487                      1 /* cq_tail_color */,
1488                      0 /* interrupt_enable */,
1489                      1 /* cq_entry_enable */,
1490                      0 /* cq_message_enable */,
1491                      0 /* interrupt offset */,
1492                      0 /* cq_message_addr */);
1493
1494
1495         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1496                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1497                            sop_rq->ring.desc_count - 1, 1, 0);
1498         if (data_rq->in_use) {
1499                 vnic_rq_init_start(data_rq,
1500                                    enic_cq_rq(enic,
1501                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1502                                    data_rq->ring.desc_count - 1, 1, 0);
1503         }
1504
1505         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1506         if (rc)
1507                 return rc;
1508
1509         if (data_rq->in_use) {
1510                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1511                 if (rc) {
1512                         enic_rxmbuf_queue_release(enic, sop_rq);
1513                         return rc;
1514                 }
1515         }
1516
1517         return 0;
1518 }
1519
1520 /* The Cisco NIC can send and receive packets up to a max packet size
1521  * determined by the NIC type and firmware. There is also an MTU
1522  * configured into the NIC via the CIMC/UCSM management interface
1523  * which can be overridden by this function (up to the max packet size).
1524  * Depending on the network setup, doing so may cause packet drops
1525  * and unexpected behavior.
1526  */
1527 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1528 {
1529         unsigned int rq_idx;
1530         struct vnic_rq *rq;
1531         int rc = 0;
1532         uint16_t old_mtu;       /* previous setting */
1533         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1534         struct rte_eth_dev *eth_dev = enic->rte_dev;
1535
1536         old_mtu = eth_dev->data->mtu;
1537         config_mtu = enic->config.mtu;
1538
1539         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1540                 return -E_RTE_SECONDARY;
1541
1542         if (new_mtu > enic->max_mtu) {
1543                 dev_err(enic,
1544                         "MTU not updated: requested (%u) greater than max (%u)\n",
1545                         new_mtu, enic->max_mtu);
1546                 return -EINVAL;
1547         }
1548         if (new_mtu < ENIC_MIN_MTU) {
1549                 dev_info(enic,
1550                         "MTU not updated: requested (%u) less than min (%u)\n",
1551                         new_mtu, ENIC_MIN_MTU);
1552                 return -EINVAL;
1553         }
1554         if (new_mtu > config_mtu)
1555                 dev_warning(enic,
1556                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1557                         new_mtu, config_mtu);
1558
1559         /* Update the MTU and maximum packet length */
1560         eth_dev->data->mtu = new_mtu;
1561         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1562                 enic_mtu_to_max_rx_pktlen(new_mtu);
1563
1564         /*
1565          * If the device has not started (enic_enable), nothing to do.
1566          * Later, enic_enable() will set up RQs reflecting the new maximum
1567          * packet length.
1568          */
1569         if (!eth_dev->data->dev_started)
1570                 goto set_mtu_done;
1571
1572         /*
1573          * The device has started, re-do RQs on the fly. In the process, we
1574          * pick up the new maximum packet length.
1575          *
1576          * Some applications rely on the ability to change MTU without stopping
1577          * the device. So keep this behavior for now.
1578          */
1579         rte_spinlock_lock(&enic->mtu_lock);
1580
1581         /* Stop traffic on all RQs */
1582         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1583                 rq = &enic->rq[rq_idx];
1584                 if (rq->is_sop && rq->in_use) {
1585                         rc = enic_stop_rq(enic,
1586                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1587                         if (rc) {
1588                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1589                                 goto set_mtu_done;
1590                         }
1591                 }
1592         }
1593
1594         /* replace Rx function with a no-op to avoid getting stale pkts */
1595         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1596         rte_mb();
1597
1598         /* Allow time for threads to exit the real Rx function. */
1599         usleep(100000);
1600
1601         /* now it is safe to reconfigure the RQs */
1602
1603
1604         /* free and reallocate RQs with the new MTU */
1605         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1606                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1607                 if (!rq->in_use)
1608                         continue;
1609
1610                 enic_free_rq(rq);
1611                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1612                                    rq->tot_nb_desc, rq->rx_free_thresh);
1613                 if (rc) {
1614                         dev_err(enic,
1615                                 "Fatal MTU alloc error- No traffic will pass\n");
1616                         goto set_mtu_done;
1617                 }
1618
1619                 rc = enic_reinit_rq(enic, rq_idx);
1620                 if (rc) {
1621                         dev_err(enic,
1622                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1623                         goto set_mtu_done;
1624                 }
1625         }
1626
1627         /* put back the real receive function */
1628         rte_mb();
1629         enic_pick_rx_handler(eth_dev);
1630         rte_mb();
1631
1632         /* restart Rx traffic */
1633         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1634                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1635                 if (rq->is_sop && rq->in_use)
1636                         enic_start_rq(enic, rq_idx);
1637         }
1638
1639 set_mtu_done:
1640         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1641         rte_spinlock_unlock(&enic->mtu_lock);
1642         return rc;
1643 }
1644
1645 static int enic_dev_init(struct enic *enic)
1646 {
1647         int err;
1648         struct rte_eth_dev *eth_dev = enic->rte_dev;
1649
1650         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1651
1652         /* Get vNIC configuration
1653         */
1654         err = enic_get_vnic_config(enic);
1655         if (err) {
1656                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1657                 return err;
1658         }
1659
1660         /* Get available resource counts */
1661         enic_get_res_counts(enic);
1662         if (enic->conf_rq_count == 1) {
1663                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1664                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1665                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1666                 return -EINVAL;
1667         }
1668         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1669         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1670                                enic->conf_cq_count, 8);
1671         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1672                                  enic->conf_intr_count, 8);
1673         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1674                                enic->conf_rq_count, 8);
1675         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1676                                enic->conf_wq_count, 8);
1677         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1678                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1679                 return -1;
1680         }
1681         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1682                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1683                 return -1;
1684         }
1685         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1686                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1687                 return -1;
1688         }
1689         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1690                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1691                 return -1;
1692         }
1693
1694         /* Get the supported filters */
1695         enic_fdir_info(enic);
1696
1697         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1698                                         sizeof(struct rte_ether_addr) *
1699                                         ENIC_UNICAST_PERFECT_FILTERS, 0);
1700         if (!eth_dev->data->mac_addrs) {
1701                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1702                 return -1;
1703         }
1704         rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1705                         eth_dev->data->mac_addrs);
1706
1707         vnic_dev_set_reset_flag(enic->vdev, 0);
1708
1709         LIST_INIT(&enic->flows);
1710
1711         /* set up link status checking */
1712         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1713
1714         /*
1715          * When Geneve with options offload is available, always disable it
1716          * first as it can interfere with user flow rules.
1717          */
1718         if (enic->geneve_opt_avail) {
1719                 /*
1720                  * Disabling fails if the feature is provisioned but
1721                  * not enabled. So ignore result and do not log error.
1722                  */
1723                 vnic_dev_overlay_offload_ctrl(enic->vdev,
1724                         OVERLAY_FEATURE_GENEVE,
1725                         OVERLAY_OFFLOAD_DISABLE);
1726         }
1727         enic->overlay_offload = false;
1728         if (enic->disable_overlay && enic->vxlan) {
1729                 /*
1730                  * Explicitly disable overlay offload as the setting is
1731                  * sticky, and resetting vNIC does not disable it.
1732                  */
1733                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1734                                                   OVERLAY_FEATURE_VXLAN,
1735                                                   OVERLAY_OFFLOAD_DISABLE)) {
1736                         dev_err(enic, "failed to disable overlay offload\n");
1737                 } else {
1738                         dev_info(enic, "Overlay offload is disabled\n");
1739                 }
1740         }
1741         if (!enic->disable_overlay && enic->vxlan &&
1742             /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1743             vnic_dev_overlay_offload_ctrl(enic->vdev,
1744                                           OVERLAY_FEATURE_VXLAN,
1745                                           OVERLAY_OFFLOAD_ENABLE) == 0) {
1746                 enic->tx_offload_capa |=
1747                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1748                         DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1749                         DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1750                 enic->tx_offload_mask |=
1751                         PKT_TX_OUTER_IPV6 |
1752                         PKT_TX_OUTER_IPV4 |
1753                         PKT_TX_OUTER_IP_CKSUM |
1754                         PKT_TX_TUNNEL_MASK;
1755                 enic->overlay_offload = true;
1756                 dev_info(enic, "Overlay offload is enabled\n");
1757         }
1758         /* Geneve with options offload requires overlay offload */
1759         if (enic->overlay_offload && enic->geneve_opt_avail &&
1760             enic->geneve_opt_request) {
1761                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1762                                 OVERLAY_FEATURE_GENEVE,
1763                                 OVERLAY_OFFLOAD_ENABLE)) {
1764                         dev_err(enic, "failed to enable geneve+option\n");
1765                 } else {
1766                         enic->geneve_opt_enabled = 1;
1767                         dev_info(enic, "Geneve with options is enabled\n");
1768                 }
1769         }
1770         /*
1771          * Reset the vxlan port if HW vxlan parsing is available. It
1772          * is always enabled regardless of overlay offload
1773          * enable/disable.
1774          */
1775         if (enic->vxlan) {
1776                 enic->vxlan_port = RTE_VXLAN_DEFAULT_PORT;
1777                 /*
1778                  * Reset the vxlan port to the default, as the NIC firmware
1779                  * does not reset it automatically and keeps the old setting.
1780                  */
1781                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1782                                                  OVERLAY_CFG_VXLAN_PORT_UPDATE,
1783                                                  RTE_VXLAN_DEFAULT_PORT)) {
1784                         dev_err(enic, "failed to update vxlan port\n");
1785                         return -EINVAL;
1786                 }
1787         }
1788
1789         return 0;
1790
1791 }
1792
1793 int enic_probe(struct enic *enic)
1794 {
1795         struct rte_pci_device *pdev = enic->pdev;
1796         int err = -1;
1797
1798         dev_debug(enic, "Initializing ENIC PMD\n");
1799
1800         /* if this is a secondary process the hardware is already initialized */
1801         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1802                 return 0;
1803
1804         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1805         enic->bar0.len = pdev->mem_resource[0].len;
1806
1807         /* Register vNIC device */
1808         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1809         if (!enic->vdev) {
1810                 dev_err(enic, "vNIC registration failed, aborting\n");
1811                 goto err_out;
1812         }
1813
1814         LIST_INIT(&enic->memzone_list);
1815         rte_spinlock_init(&enic->memzone_list_lock);
1816
1817         vnic_register_cbacks(enic->vdev,
1818                 enic_alloc_consistent,
1819                 enic_free_consistent);
1820
1821         /*
1822          * Allocate the consistent memory for stats upfront so both primary and
1823          * secondary processes can dump stats.
1824          */
1825         err = vnic_dev_alloc_stats_mem(enic->vdev);
1826         if (err) {
1827                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1828                 goto err_out_unregister;
1829         }
1830         /* Issue device open to get device in known state */
1831         err = enic_dev_open(enic);
1832         if (err) {
1833                 dev_err(enic, "vNIC dev open failed, aborting\n");
1834                 goto err_out_unregister;
1835         }
1836
1837         /* Set ingress vlan rewrite mode before vnic initialization */
1838         dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1839                   enic->ig_vlan_rewrite_mode);
1840         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1841                 enic->ig_vlan_rewrite_mode);
1842         if (err) {
1843                 dev_err(enic,
1844                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1845                 goto err_out_dev_close;
1846         }
1847
1848         /* Issue device init to initialize the vnic-to-switch link.
1849          * We'll start with carrier off and wait for link UP
1850          * notification later to turn on carrier.  We don't need
1851          * to wait here for the vnic-to-switch link initialization
1852          * to complete; link UP notification is the indication that
1853          * the process is complete.
1854          */
1855
1856         err = vnic_dev_init(enic->vdev, 0);
1857         if (err) {
1858                 dev_err(enic, "vNIC dev init failed, aborting\n");
1859                 goto err_out_dev_close;
1860         }
1861
1862         err = enic_dev_init(enic);
1863         if (err) {
1864                 dev_err(enic, "Device initialization failed, aborting\n");
1865                 goto err_out_dev_close;
1866         }
1867
1868         return 0;
1869
1870 err_out_dev_close:
1871         vnic_dev_close(enic->vdev);
1872 err_out_unregister:
1873         vnic_dev_unregister(enic->vdev);
1874 err_out:
1875         return err;
1876 }
1877
1878 void enic_remove(struct enic *enic)
1879 {
1880         enic_dev_deinit(enic);
1881         vnic_dev_close(enic->vdev);
1882         vnic_dev_unregister(enic->vdev);
1883 }