drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11
  12 #include <rte_pci.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_ethdev_driver.h>
  19
  20 #include "enic_compat.h"
  21 #include "enic.h"
  22 #include "wq_enet_desc.h"
  23 #include "rq_enet_desc.h"
  24 #include "cq_enet_desc.h"
  25 #include "vnic_enet.h"
  26 #include "vnic_dev.h"
  27 #include "vnic_wq.h"
  28 #include "vnic_rq.h"
  29 #include "vnic_cq.h"
  30 #include "vnic_intr.h"
  31 #include "vnic_nic.h"
  32
  33 static inline int enic_is_sriov_vf(struct enic *enic)
  34 {
  35         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  36 }
  37
  38 static int is_zero_addr(uint8_t *addr)
  39 {
  40         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  41 }
  42
  43 static int is_mcast_addr(uint8_t *addr)
  44 {
  45         return addr[0] & 1;
  46 }
  47
  48 static int is_eth_addr_valid(uint8_t *addr)
  49 {
  50         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  51 }
  52
  53 static void
  54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  55 {
  56         uint16_t i;
  57
  58         if (!rq || !rq->mbuf_ring) {
  59                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  60                 return;
  61         }
  62
  63         for (i = 0; i < rq->ring.desc_count; i++) {
  64                 if (rq->mbuf_ring[i]) {
  65                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  66                         rq->mbuf_ring[i] = NULL;
  67                 }
  68         }
  69 }
  70
  71 static void enic_free_wq_buf(struct rte_mbuf **buf)
  72 {
  73         struct rte_mbuf *mbuf = *buf;
  74
  75         rte_pktmbuf_free_seg(mbuf);
  76         *buf = NULL;
  77 }
  78
  79 static void enic_log_q_error(struct enic *enic)
  80 {
  81         unsigned int i;
  82         u32 error_status;
  83
  84         for (i = 0; i < enic->wq_count; i++) {
  85                 error_status = vnic_wq_error_status(&enic->wq[i]);
  86                 if (error_status)
  87                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  88                                 error_status);
  89         }
  90
  91         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  92                 if (!enic->rq[i].in_use)
  93                         continue;
  94                 error_status = vnic_rq_error_status(&enic->rq[i]);
  95                 if (error_status)
  96                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  97                                 error_status);
  98         }
  99 }
 100
 101 static void enic_clear_soft_stats(struct enic *enic)
 102 {
 103         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 104         rte_atomic64_clear(&soft_stats->rx_nombuf);
 105         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 106         rte_atomic64_clear(&soft_stats->tx_oversized);
 107 }
 108
 109 static void enic_init_soft_stats(struct enic *enic)
 110 {
 111         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 112         rte_atomic64_init(&soft_stats->rx_nombuf);
 113         rte_atomic64_init(&soft_stats->rx_packet_errors);
 114         rte_atomic64_init(&soft_stats->tx_oversized);
 115         enic_clear_soft_stats(enic);
 116 }
 117
 118 int enic_dev_stats_clear(struct enic *enic)
 119 {
 120         int ret;
 121
 122         ret = vnic_dev_stats_clear(enic->vdev);
 123         if (ret != 0) {
 124                 dev_err(enic, "Error in clearing stats\n");
 125                 return ret;
 126         }
 127         enic_clear_soft_stats(enic);
 128
 129         return 0;
 130 }
 131
 132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 133 {
 134         struct vnic_stats *stats;
 135         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 136         int64_t rx_truncated;
 137         uint64_t rx_packet_errors;
 138         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 139
 140         if (ret) {
 141                 dev_err(enic, "Error in getting stats\n");
 142                 return ret;
 143         }
 144
 145         /* The number of truncated packets can only be calculated by
 146          * subtracting a hardware counter from error packets received by
 147          * the driver. Note: this causes transient inaccuracies in the
 148          * ipackets count. Also, the length of truncated packets are
 149          * counted in ibytes even though truncated packets are dropped
 150          * which can make ibytes be slightly higher than it should be.
 151          */
 152         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 153         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 154
 155         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 156         r_stats->opackets = stats->tx.tx_frames_ok;
 157
 158         r_stats->ibytes = stats->rx.rx_bytes_ok;
 159         r_stats->obytes = stats->tx.tx_bytes_ok;
 160
 161         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 162         r_stats->oerrors = stats->tx.tx_errors
 163                            + rte_atomic64_read(&soft_stats->tx_oversized);
 164
 165         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 166
 167         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 168         return 0;
 169 }
 170
 171 int enic_del_mac_address(struct enic *enic, int mac_index)
 172 {
 173         struct rte_eth_dev *eth_dev = enic->rte_dev;
 174         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 175
 176         return vnic_dev_del_addr(enic->vdev, mac_addr);
 177 }
 178
 179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 180 {
 181         int err;
 182
 183         if (!is_eth_addr_valid(mac_addr)) {
 184                 dev_err(enic, "invalid mac address\n");
 185                 return -EINVAL;
 186         }
 187
 188         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 189         if (err)
 190                 dev_err(enic, "add mac addr failed\n");
 191         return err;
 192 }
 193
 194 static void
 195 enic_free_rq_buf(struct rte_mbuf **mbuf)
 196 {
 197         if (*mbuf == NULL)
 198                 return;
 199
 200         rte_pktmbuf_free(*mbuf);
 201         *mbuf = NULL;
 202 }
 203
 204 void enic_init_vnic_resources(struct enic *enic)
 205 {
 206         unsigned int error_interrupt_enable = 1;
 207         unsigned int error_interrupt_offset = 0;
 208         unsigned int rxq_interrupt_enable = 0;
 209         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 210         unsigned int index = 0;
 211         unsigned int cq_idx;
 212         struct vnic_rq *data_rq;
 213
 214         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 215                 rxq_interrupt_enable = 1;
 216
 217         for (index = 0; index < enic->rq_count; index++) {
 218                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 219
 220                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 221                         cq_idx,
 222                         error_interrupt_enable,
 223                         error_interrupt_offset);
 224
 225                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 226                 if (data_rq->in_use)
 227                         vnic_rq_init(data_rq,
 228                                      cq_idx,
 229                                      error_interrupt_enable,
 230                                      error_interrupt_offset);
 231
 232                 vnic_cq_init(&enic->cq[cq_idx],
 233                         0 /* flow_control_enable */,
 234                         1 /* color_enable */,
 235                         0 /* cq_head */,
 236                         0 /* cq_tail */,
 237                         1 /* cq_tail_color */,
 238                         rxq_interrupt_enable,
 239                         1 /* cq_entry_enable */,
 240                         0 /* cq_message_enable */,
 241                         rxq_interrupt_offset,
 242                         0 /* cq_message_addr */);
 243                 if (rxq_interrupt_enable)
 244                         rxq_interrupt_offset++;
 245         }
 246
 247         for (index = 0; index < enic->wq_count; index++) {
 248                 vnic_wq_init(&enic->wq[index],
 249                         enic_cq_wq(enic, index),
 250                         error_interrupt_enable,
 251                         error_interrupt_offset);
 252                 /* Compute unsupported ol flags for enic_prep_pkts() */
 253                 enic->wq[index].tx_offload_notsup_mask =
 254                         PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 255
 256                 cq_idx = enic_cq_wq(enic, index);
 257                 vnic_cq_init(&enic->cq[cq_idx],
 258                         0 /* flow_control_enable */,
 259                         1 /* color_enable */,
 260                         0 /* cq_head */,
 261                         0 /* cq_tail */,
 262                         1 /* cq_tail_color */,
 263                         0 /* interrupt_enable */,
 264                         0 /* cq_entry_enable */,
 265                         1 /* cq_message_enable */,
 266                         0 /* interrupt offset */,
 267                         (u64)enic->wq[index].cqmsg_rz->iova);
 268         }
 269
 270         for (index = 0; index < enic->intr_count; index++) {
 271                 vnic_intr_init(&enic->intr[index],
 272                                enic->config.intr_timer_usec,
 273                                enic->config.intr_timer_type,
 274                                /*mask_on_assertion*/1);
 275         }
 276 }
 277
 278
 279 static int
 280 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 281 {
 282         struct rte_mbuf *mb;
 283         struct rq_enet_desc *rqd = rq->ring.descs;
 284         unsigned i;
 285         dma_addr_t dma_addr;
 286         uint32_t max_rx_pkt_len;
 287         uint16_t rq_buf_len;
 288
 289         if (!rq->in_use)
 290                 return 0;
 291
 292         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 293                   rq->ring.desc_count);
 294
 295         /*
 296          * If *not* using scatter and the mbuf size is greater than the
 297          * requested max packet size (max_rx_pkt_len), then reduce the
 298          * posted buffer size to max_rx_pkt_len. HW still receives packets
 299          * larger than max_rx_pkt_len, but they will be truncated, which we
 300          * drop in the rx handler. Not ideal, but better than returning
 301          * large packets when the user is not expecting them.
 302          */
 303         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 304         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 305         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 306                 rq_buf_len = max_rx_pkt_len;
 307         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 308                 mb = rte_mbuf_raw_alloc(rq->mp);
 309                 if (mb == NULL) {
 310                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 311                         (unsigned)rq->index);
 312                         return -ENOMEM;
 313                 }
 314
 315                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 316                 dma_addr = (dma_addr_t)(mb->buf_iova
 317                            + RTE_PKTMBUF_HEADROOM);
 318                 rq_enet_desc_enc(rqd, dma_addr,
 319                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 320                                 : RQ_ENET_TYPE_NOT_SOP),
 321                                 rq_buf_len);
 322                 rq->mbuf_ring[i] = mb;
 323         }
 324         /*
 325          * Do not post the buffers to the NIC until we enable the RQ via
 326          * enic_start_rq().
 327          */
 328         rq->need_initial_post = true;
 329         /* Initialize fetch index while RQ is disabled */
 330         iowrite32(0, &rq->ctrl->fetch_index);
 331         return 0;
 332 }
 333
 334 /*
 335  * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
 336  * allocated the buffers and filled the RQ descriptor ring. Just need to push
 337  * the post index to the NIC.
 338  */
 339 static void
 340 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 341 {
 342         if (!rq->in_use || !rq->need_initial_post)
 343                 return;
 344
 345         /* make sure all prior writes are complete before doing the PIO write */
 346         rte_rmb();
 347
 348         /* Post all but the last buffer to VIC. */
 349         rq->posted_index = rq->ring.desc_count - 1;
 350
 351         rq->rx_nb_hold = 0;
 352
 353         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 354                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 355         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 356         rte_rmb();
 357         rq->need_initial_post = false;
 358 }
 359
 360 void *
 361 enic_alloc_consistent(void *priv, size_t size,
 362         dma_addr_t *dma_handle, u8 *name)
 363 {
 364         void *vaddr;
 365         const struct rte_memzone *rz;
 366         *dma_handle = 0;
 367         struct enic *enic = (struct enic *)priv;
 368         struct enic_memzone_entry *mze;
 369
 370         rz = rte_memzone_reserve_aligned((const char *)name, size,
 371                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 372         if (!rz) {
 373                 pr_err("%s : Failed to allocate memory requested for %s\n",
 374                         __func__, name);
 375                 return NULL;
 376         }
 377
 378         vaddr = rz->addr;
 379         *dma_handle = (dma_addr_t)rz->iova;
 380
 381         mze = rte_malloc("enic memzone entry",
 382                          sizeof(struct enic_memzone_entry), 0);
 383
 384         if (!mze) {
 385                 pr_err("%s : Failed to allocate memory for memzone list\n",
 386                        __func__);
 387                 rte_memzone_free(rz);
 388                 return NULL;
 389         }
 390
 391         mze->rz = rz;
 392
 393         rte_spinlock_lock(&enic->memzone_list_lock);
 394         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 395         rte_spinlock_unlock(&enic->memzone_list_lock);
 396
 397         return vaddr;
 398 }
 399
 400 void
 401 enic_free_consistent(void *priv,
 402                      __rte_unused size_t size,
 403                      void *vaddr,
 404                      dma_addr_t dma_handle)
 405 {
 406         struct enic_memzone_entry *mze;
 407         struct enic *enic = (struct enic *)priv;
 408
 409         rte_spinlock_lock(&enic->memzone_list_lock);
 410         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 411                 if (mze->rz->addr == vaddr &&
 412                     mze->rz->iova == dma_handle)
 413                         break;
 414         }
 415         if (mze == NULL) {
 416                 rte_spinlock_unlock(&enic->memzone_list_lock);
 417                 dev_warning(enic,
 418                             "Tried to free memory, but couldn't find it in the memzone list\n");
 419                 return;
 420         }
 421         LIST_REMOVE(mze, entries);
 422         rte_spinlock_unlock(&enic->memzone_list_lock);
 423         rte_memzone_free(mze->rz);
 424         rte_free(mze);
 425 }
 426
 427 int enic_link_update(struct rte_eth_dev *eth_dev)
 428 {
 429         struct enic *enic = pmd_priv(eth_dev);
 430         struct rte_eth_link link;
 431
 432         memset(&link, 0, sizeof(link));
 433         link.link_status = enic_get_link_status(enic);
 434         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 435         link.link_speed = vnic_dev_port_speed(enic->vdev);
 436
 437         return rte_eth_linkstatus_set(eth_dev, &link);
 438 }
 439
 440 static void
 441 enic_intr_handler(void *arg)
 442 {
 443         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 444         struct enic *enic = pmd_priv(dev);
 445
 446         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 447
 448         enic_link_update(dev);
 449         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 450         enic_log_q_error(enic);
 451         /* Re-enable irq in case of INTx */
 452         rte_intr_ack(&enic->pdev->intr_handle);
 453 }
 454
 455 static int enic_rxq_intr_init(struct enic *enic)
 456 {
 457         struct rte_intr_handle *intr_handle;
 458         uint32_t rxq_intr_count, i;
 459         int err;
 460
 461         intr_handle = enic->rte_dev->intr_handle;
 462         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 463                 return 0;
 464         /*
 465          * Rx queue interrupts only work when we have MSI-X interrupts,
 466          * one per queue. Sharing one interrupt is technically
 467          * possible with VIC, but it is not worth the complications it brings.
 468          */
 469         if (!rte_intr_cap_multiple(intr_handle)) {
 470                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 471                         " (vfio-pci driver)\n");
 472                 return -ENOTSUP;
 473         }
 474         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 475         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 476         if (err) {
 477                 dev_err(enic, "Failed to enable event fds for Rx queue"
 478                         " interrupts\n");
 479                 return err;
 480         }
 481         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 482                                             rxq_intr_count * sizeof(int), 0);
 483         if (intr_handle->intr_vec == NULL) {
 484                 dev_err(enic, "Failed to allocate intr_vec\n");
 485                 return -ENOMEM;
 486         }
 487         for (i = 0; i < rxq_intr_count; i++)
 488                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 489         return 0;
 490 }
 491
 492 static void enic_rxq_intr_deinit(struct enic *enic)
 493 {
 494         struct rte_intr_handle *intr_handle;
 495
 496         intr_handle = enic->rte_dev->intr_handle;
 497         rte_intr_efd_disable(intr_handle);
 498         if (intr_handle->intr_vec != NULL) {
 499                 rte_free(intr_handle->intr_vec);
 500                 intr_handle->intr_vec = NULL;
 501         }
 502 }
 503
 504 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 505 {
 506         struct wq_enet_desc *desc;
 507         struct vnic_wq *wq;
 508         unsigned int i;
 509
 510         /*
 511          * Fill WQ descriptor fields that never change. Every descriptor is
 512          * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
 513          * descriptors (i.e. request one completion update every 32 packets).
 514          */
 515         wq = &enic->wq[queue_idx];
 516         desc = (struct wq_enet_desc *)wq->ring.descs;
 517         for (i = 0; i < wq->ring.desc_count; i++, desc++) {
 518                 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
 519                 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
 520                         desc->header_length_flags |=
 521                                 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
 522         }
 523 }
 524
 525 /*
 526  * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
 527  * used when that file is not compiled.
 528  */
 529 __rte_weak bool
 530 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
 531 {
 532         return false;
 533 }
 534
 535 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
 536 {
 537         struct enic *enic = pmd_priv(eth_dev);
 538
 539         /*
 540          * Preference order:
 541          * 1. The vectorized handler if possible and requested.
 542          * 2. The non-scatter, simplified handler if scatter Rx is not used.
 543          * 3. The default handler as a fallback.
 544          */
 545         if (enic_use_vector_rx_handler(eth_dev))
 546                 return;
 547         if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 548                 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
 549                 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
 550         } else {
 551                 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
 552                 eth_dev->rx_pkt_burst = &enic_recv_pkts;
 553         }
 554 }
 555
 556 /* Secondary process uses this to set the Tx handler */
 557 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
 558 {
 559         struct enic *enic = pmd_priv(eth_dev);
 560
 561         if (enic->use_simple_tx_handler) {
 562                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 563                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 564         } else {
 565                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 566                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 567         }
 568 }
 569
 570 int enic_enable(struct enic *enic)
 571 {
 572         unsigned int index;
 573         int err;
 574         struct rte_eth_dev *eth_dev = enic->rte_dev;
 575         uint64_t simple_tx_offloads;
 576         uintptr_t p;
 577
 578         if (enic->enable_avx2_rx) {
 579                 struct rte_mbuf mb_def = { .buf_addr = 0 };
 580
 581                 /*
 582                  * mbuf_initializer contains const-after-init fields of
 583                  * receive mbufs (i.e. 64 bits of fields from rearm_data).
 584                  * It is currently used by the vectorized handler.
 585                  */
 586                 mb_def.nb_segs = 1;
 587                 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 588                 mb_def.port = enic->port_id;
 589                 rte_mbuf_refcnt_set(&mb_def, 1);
 590                 rte_compiler_barrier();
 591                 p = (uintptr_t)&mb_def.rearm_data;
 592                 enic->mbuf_initializer = *(uint64_t *)p;
 593         }
 594
 595         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 596         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 597
 598         /* vnic notification of link status has already been turned on in
 599          * enic_dev_init() which is called during probe time.  Here we are
 600          * just turning on interrupt vector 0 if needed.
 601          */
 602         if (eth_dev->data->dev_conf.intr_conf.lsc)
 603                 vnic_dev_notify_set(enic->vdev, 0);
 604
 605         err = enic_rxq_intr_init(enic);
 606         if (err)
 607                 return err;
 608         if (enic_clsf_init(enic))
 609                 dev_warning(enic, "Init of hash table for clsf failed."\
 610                         "Flow director feature will not work\n");
 611
 612         if (enic_fm_init(enic))
 613                 dev_warning(enic, "Init of flowman failed.\n");
 614
 615         for (index = 0; index < enic->rq_count; index++) {
 616                 err = enic_alloc_rx_queue_mbufs(enic,
 617                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 618                 if (err) {
 619                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 620                         return err;
 621                 }
 622                 err = enic_alloc_rx_queue_mbufs(enic,
 623                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 624                 if (err) {
 625                         /* release the allocated mbufs for the sop rq*/
 626                         enic_rxmbuf_queue_release(enic,
 627                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 628
 629                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 630                         return err;
 631                 }
 632         }
 633
 634         /*
 635          * Use the simple TX handler if possible. Only checksum offloads
 636          * and vlan insertion are supported.
 637          */
 638         simple_tx_offloads = enic->tx_offload_capa &
 639                 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 640                  DEV_TX_OFFLOAD_VLAN_INSERT |
 641                  DEV_TX_OFFLOAD_IPV4_CKSUM |
 642                  DEV_TX_OFFLOAD_UDP_CKSUM |
 643                  DEV_TX_OFFLOAD_TCP_CKSUM);
 644         if ((eth_dev->data->dev_conf.txmode.offloads &
 645              ~simple_tx_offloads) == 0) {
 646                 ENICPMD_LOG(DEBUG, " use the simple tx handler");
 647                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 648                 for (index = 0; index < enic->wq_count; index++)
 649                         enic_prep_wq_for_simple_tx(enic, index);
 650                 enic->use_simple_tx_handler = 1;
 651         } else {
 652                 ENICPMD_LOG(DEBUG, " use the default tx handler");
 653                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 654         }
 655
 656         enic_pick_rx_handler(eth_dev);
 657
 658         for (index = 0; index < enic->wq_count; index++)
 659                 enic_start_wq(enic, index);
 660         for (index = 0; index < enic->rq_count; index++)
 661                 enic_start_rq(enic, index);
 662
 663         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 664
 665         vnic_dev_enable_wait(enic->vdev);
 666
 667         /* Register and enable error interrupt */
 668         rte_intr_callback_register(&(enic->pdev->intr_handle),
 669                 enic_intr_handler, (void *)enic->rte_dev);
 670
 671         rte_intr_enable(&(enic->pdev->intr_handle));
 672         /* Unmask LSC interrupt */
 673         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 674
 675         return 0;
 676 }
 677
 678 int enic_alloc_intr_resources(struct enic *enic)
 679 {
 680         int err;
 681         unsigned int i;
 682
 683         dev_info(enic, "vNIC resources used:  "\
 684                 "wq %d rq %d cq %d intr %d\n",
 685                 enic->wq_count, enic_vnic_rq_count(enic),
 686                 enic->cq_count, enic->intr_count);
 687
 688         for (i = 0; i < enic->intr_count; i++) {
 689                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 690                 if (err) {
 691                         enic_free_vnic_resources(enic);
 692                         return err;
 693                 }
 694         }
 695         return 0;
 696 }
 697
 698 void enic_free_rq(void *rxq)
 699 {
 700         struct vnic_rq *rq_sop, *rq_data;
 701         struct enic *enic;
 702
 703         if (rxq == NULL)
 704                 return;
 705
 706         rq_sop = (struct vnic_rq *)rxq;
 707         enic = vnic_dev_priv(rq_sop->vdev);
 708         rq_data = &enic->rq[rq_sop->data_queue_idx];
 709
 710         if (rq_sop->free_mbufs) {
 711                 struct rte_mbuf **mb;
 712                 int i;
 713
 714                 mb = rq_sop->free_mbufs;
 715                 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
 716                      i < ENIC_RX_BURST_MAX; i++)
 717                         rte_pktmbuf_free(mb[i]);
 718                 rte_free(rq_sop->free_mbufs);
 719                 rq_sop->free_mbufs = NULL;
 720                 rq_sop->num_free_mbufs = 0;
 721         }
 722
 723         enic_rxmbuf_queue_release(enic, rq_sop);
 724         if (rq_data->in_use)
 725                 enic_rxmbuf_queue_release(enic, rq_data);
 726
 727         rte_free(rq_sop->mbuf_ring);
 728         if (rq_data->in_use)
 729                 rte_free(rq_data->mbuf_ring);
 730
 731         rq_sop->mbuf_ring = NULL;
 732         rq_data->mbuf_ring = NULL;
 733
 734         vnic_rq_free(rq_sop);
 735         if (rq_data->in_use)
 736                 vnic_rq_free(rq_data);
 737
 738         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 739
 740         rq_sop->in_use = 0;
 741         rq_data->in_use = 0;
 742 }
 743
 744 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 745 {
 746         struct rte_eth_dev_data *data = enic->dev_data;
 747         vnic_wq_enable(&enic->wq[queue_idx]);
 748         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 749 }
 750
 751 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 752 {
 753         struct rte_eth_dev_data *data = enic->dev_data;
 754         int ret;
 755
 756         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 757         if (ret)
 758                 return ret;
 759
 760         data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 761         return 0;
 762 }
 763
 764 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 765 {
 766         struct rte_eth_dev_data *data = enic->dev_data;
 767         struct vnic_rq *rq_sop;
 768         struct vnic_rq *rq_data;
 769         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 770         rq_data = &enic->rq[rq_sop->data_queue_idx];
 771
 772         if (rq_data->in_use) {
 773                 vnic_rq_enable(rq_data);
 774                 enic_initial_post_rx(enic, rq_data);
 775         }
 776         rte_mb();
 777         vnic_rq_enable(rq_sop);
 778         enic_initial_post_rx(enic, rq_sop);
 779         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 780 }
 781
 782 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 783 {
 784         struct rte_eth_dev_data *data = enic->dev_data;
 785         int ret1 = 0, ret2 = 0;
 786         struct vnic_rq *rq_sop;
 787         struct vnic_rq *rq_data;
 788         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 789         rq_data = &enic->rq[rq_sop->data_queue_idx];
 790
 791         ret2 = vnic_rq_disable(rq_sop);
 792         rte_mb();
 793         if (rq_data->in_use)
 794                 ret1 = vnic_rq_disable(rq_data);
 795
 796         if (ret2)
 797                 return ret2;
 798         else if (ret1)
 799                 return ret1;
 800
 801         data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 802         return 0;
 803 }
 804
 805 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 806         unsigned int socket_id, struct rte_mempool *mp,
 807         uint16_t nb_desc, uint16_t free_thresh)
 808 {
 809         int rc;
 810         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 811         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 812         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 813         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 814         unsigned int mbuf_size, mbufs_per_pkt;
 815         unsigned int nb_sop_desc, nb_data_desc;
 816         uint16_t min_sop, max_sop, min_data, max_data;
 817         uint32_t max_rx_pkt_len;
 818
 819         rq_sop->is_sop = 1;
 820         rq_sop->data_queue_idx = data_queue_idx;
 821         rq_data->is_sop = 0;
 822         rq_data->data_queue_idx = 0;
 823         rq_sop->socket_id = socket_id;
 824         rq_sop->mp = mp;
 825         rq_data->socket_id = socket_id;
 826         rq_data->mp = mp;
 827         rq_sop->in_use = 1;
 828         rq_sop->rx_free_thresh = free_thresh;
 829         rq_data->rx_free_thresh = free_thresh;
 830         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 831                   free_thresh);
 832
 833         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 834                                RTE_PKTMBUF_HEADROOM);
 835         /* max_rx_pkt_len includes the ethernet header and CRC. */
 836         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 837
 838         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 839             DEV_RX_OFFLOAD_SCATTER) {
 840                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 841                 /* ceil((max pkt len)/mbuf_size) */
 842                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 843         } else {
 844                 dev_info(enic, "Scatter rx mode disabled\n");
 845                 mbufs_per_pkt = 1;
 846                 if (max_rx_pkt_len > mbuf_size) {
 847                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 848                                     " larger than the mbuf size (%u), and"
 849                                     " scatter is disabled. Larger packets will"
 850                                     " be truncated.\n",
 851                                     max_rx_pkt_len, mbuf_size);
 852                 }
 853         }
 854
 855         if (mbufs_per_pkt > 1) {
 856                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 857                 rq_sop->data_queue_enable = 1;
 858                 rq_data->in_use = 1;
 859                 /*
 860                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 861                  * receives packet sizes up to the "max" MTU.
 862                  * If not using scatter, we can achieve the effect of dropping
 863                  * larger packets by reducing the size of posted buffers.
 864                  * See enic_alloc_rx_queue_mbufs().
 865                  */
 866                 if (max_rx_pkt_len <
 867                     enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
 868                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 869                                     " when scatter rx mode is in use.\n");
 870                 }
 871         } else {
 872                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 873                          queue_idx);
 874                 rq_sop->data_queue_enable = 0;
 875                 rq_data->in_use = 0;
 876         }
 877
 878         /* number of descriptors have to be a multiple of 32 */
 879         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
 880         nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 881
 882         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 883         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 884
 885         if (mbufs_per_pkt > 1) {
 886                 min_sop = ENIC_RX_BURST_MAX;
 887                 max_sop = ((enic->config.rq_desc_count /
 888                             (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 889                 min_data = min_sop * (mbufs_per_pkt - 1);
 890                 max_data = enic->config.rq_desc_count;
 891         } else {
 892                 min_sop = ENIC_RX_BURST_MAX;
 893                 max_sop = enic->config.rq_desc_count;
 894                 min_data = 0;
 895                 max_data = 0;
 896         }
 897
 898         if (nb_desc < (min_sop + min_data)) {
 899                 dev_warning(enic,
 900                             "Number of rx descs too low, adjusting to minimum\n");
 901                 nb_sop_desc = min_sop;
 902                 nb_data_desc = min_data;
 903         } else if (nb_desc > (max_sop + max_data)) {
 904                 dev_warning(enic,
 905                             "Number of rx_descs too high, adjusting to maximum\n");
 906                 nb_sop_desc = max_sop;
 907                 nb_data_desc = max_data;
 908         }
 909         if (mbufs_per_pkt > 1) {
 910                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 911                          " rx descriptor range is %u to %u\n",
 912                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 913                          max_sop + max_data);
 914         }
 915         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 916                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 917
 918         /* Allocate sop queue resources */
 919         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 920                 nb_sop_desc, sizeof(struct rq_enet_desc));
 921         if (rc) {
 922                 dev_err(enic, "error in allocation of sop rq\n");
 923                 goto err_exit;
 924         }
 925         nb_sop_desc = rq_sop->ring.desc_count;
 926
 927         if (rq_data->in_use) {
 928                 /* Allocate data queue resources */
 929                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 930                                    nb_data_desc,
 931                                    sizeof(struct rq_enet_desc));
 932                 if (rc) {
 933                         dev_err(enic, "error in allocation of data rq\n");
 934                         goto err_free_rq_sop;
 935                 }
 936                 nb_data_desc = rq_data->ring.desc_count;
 937         }
 938         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 939                            socket_id, nb_sop_desc + nb_data_desc,
 940                            sizeof(struct cq_enet_rq_desc));
 941         if (rc) {
 942                 dev_err(enic, "error in allocation of cq for rq\n");
 943                 goto err_free_rq_data;
 944         }
 945
 946         /* Allocate the mbuf rings */
 947         rq_sop->mbuf_ring = (struct rte_mbuf **)
 948                 rte_zmalloc_socket("rq->mbuf_ring",
 949                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 950                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 951         if (rq_sop->mbuf_ring == NULL)
 952                 goto err_free_cq;
 953
 954         if (rq_data->in_use) {
 955                 rq_data->mbuf_ring = (struct rte_mbuf **)
 956                         rte_zmalloc_socket("rq->mbuf_ring",
 957                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 958                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 959                 if (rq_data->mbuf_ring == NULL)
 960                         goto err_free_sop_mbuf;
 961         }
 962
 963         rq_sop->free_mbufs = (struct rte_mbuf **)
 964                 rte_zmalloc_socket("rq->free_mbufs",
 965                                    sizeof(struct rte_mbuf *) *
 966                                    ENIC_RX_BURST_MAX,
 967                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 968         if (rq_sop->free_mbufs == NULL)
 969                 goto err_free_data_mbuf;
 970         rq_sop->num_free_mbufs = 0;
 971
 972         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 973
 974         return 0;
 975
 976 err_free_data_mbuf:
 977         rte_free(rq_data->mbuf_ring);
 978 err_free_sop_mbuf:
 979         rte_free(rq_sop->mbuf_ring);
 980 err_free_cq:
 981         /* cleanup on error */
 982         vnic_cq_free(&enic->cq[queue_idx]);
 983 err_free_rq_data:
 984         if (rq_data->in_use)
 985                 vnic_rq_free(rq_data);
 986 err_free_rq_sop:
 987         vnic_rq_free(rq_sop);
 988 err_exit:
 989         return -ENOMEM;
 990 }
 991
 992 void enic_free_wq(void *txq)
 993 {
 994         struct vnic_wq *wq;
 995         struct enic *enic;
 996
 997         if (txq == NULL)
 998                 return;
 999
1000         wq = (struct vnic_wq *)txq;
1001         enic = vnic_dev_priv(wq->vdev);
1002         rte_memzone_free(wq->cqmsg_rz);
1003         vnic_wq_free(wq);
1004         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1005 }
1006
1007 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1008         unsigned int socket_id, uint16_t nb_desc)
1009 {
1010         int err;
1011         struct vnic_wq *wq = &enic->wq[queue_idx];
1012         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
1013         char name[RTE_MEMZONE_NAMESIZE];
1014         static int instance;
1015
1016         wq->socket_id = socket_id;
1017         /*
1018          * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1019          * print an info message for diagnostics.
1020          */
1021         dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1022
1023         /* Allocate queue resources */
1024         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1025                 nb_desc,
1026                 sizeof(struct wq_enet_desc));
1027         if (err) {
1028                 dev_err(enic, "error in allocation of wq\n");
1029                 return err;
1030         }
1031
1032         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1033                 socket_id, nb_desc,
1034                 sizeof(struct cq_enet_wq_desc));
1035         if (err) {
1036                 vnic_wq_free(wq);
1037                 dev_err(enic, "error in allocation of cq for wq\n");
1038         }
1039
1040         /* setup up CQ message */
1041         snprintf((char *)name, sizeof(name),
1042                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1043                 instance++);
1044
1045         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1046                         sizeof(uint32_t), SOCKET_ID_ANY,
1047                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1048         if (!wq->cqmsg_rz)
1049                 return -ENOMEM;
1050
1051         return err;
1052 }
1053
1054 int enic_disable(struct enic *enic)
1055 {
1056         unsigned int i;
1057         int err;
1058
1059         for (i = 0; i < enic->intr_count; i++) {
1060                 vnic_intr_mask(&enic->intr[i]);
1061                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1062         }
1063         enic_rxq_intr_deinit(enic);
1064         rte_intr_disable(&enic->pdev->intr_handle);
1065         rte_intr_callback_unregister(&enic->pdev->intr_handle,
1066                                      enic_intr_handler,
1067                                      (void *)enic->rte_dev);
1068
1069         vnic_dev_disable(enic->vdev);
1070
1071         enic_clsf_destroy(enic);
1072         enic_fm_destroy(enic);
1073
1074         if (!enic_is_sriov_vf(enic))
1075                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1076
1077         for (i = 0; i < enic->wq_count; i++) {
1078                 err = vnic_wq_disable(&enic->wq[i]);
1079                 if (err)
1080                         return err;
1081         }
1082         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1083                 if (enic->rq[i].in_use) {
1084                         err = vnic_rq_disable(&enic->rq[i]);
1085                         if (err)
1086                                 return err;
1087                 }
1088         }
1089
1090         /* If we were using interrupts, set the interrupt vector to -1
1091          * to disable interrupts.  We are not disabling link notifcations,
1092          * though, as we want the polling of link status to continue working.
1093          */
1094         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1095                 vnic_dev_notify_set(enic->vdev, -1);
1096
1097         vnic_dev_set_reset_flag(enic->vdev, 1);
1098
1099         for (i = 0; i < enic->wq_count; i++)
1100                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1101
1102         for (i = 0; i < enic_vnic_rq_count(enic); i++)
1103                 if (enic->rq[i].in_use)
1104                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1105         for (i = 0; i < enic->cq_count; i++)
1106                 vnic_cq_clean(&enic->cq[i]);
1107         for (i = 0; i < enic->intr_count; i++)
1108                 vnic_intr_clean(&enic->intr[i]);
1109
1110         return 0;
1111 }
1112
1113 static int enic_dev_wait(struct vnic_dev *vdev,
1114         int (*start)(struct vnic_dev *, int),
1115         int (*finished)(struct vnic_dev *, int *),
1116         int arg)
1117 {
1118         int done;
1119         int err;
1120         int i;
1121
1122         err = start(vdev, arg);
1123         if (err)
1124                 return err;
1125
1126         /* Wait for func to complete...2 seconds max */
1127         for (i = 0; i < 2000; i++) {
1128                 err = finished(vdev, &done);
1129                 if (err)
1130                         return err;
1131                 if (done)
1132                         return 0;
1133                 usleep(1000);
1134         }
1135         return -ETIMEDOUT;
1136 }
1137
1138 static int enic_dev_open(struct enic *enic)
1139 {
1140         int err;
1141         int flags = CMD_OPENF_IG_DESCCACHE;
1142
1143         err = enic_dev_wait(enic->vdev, vnic_dev_open,
1144                 vnic_dev_open_done, flags);
1145         if (err)
1146                 dev_err(enic_get_dev(enic),
1147                         "vNIC device open failed, err %d\n", err);
1148
1149         return err;
1150 }
1151
1152 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1153 {
1154         dma_addr_t rss_key_buf_pa;
1155         union vnic_rss_key *rss_key_buf_va = NULL;
1156         int err, i;
1157         u8 name[RTE_MEMZONE_NAMESIZE];
1158
1159         RTE_ASSERT(user_key != NULL);
1160         snprintf((char *)name, sizeof(name), "rss_key-%s", enic->bdf_name);
1161         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1162                 &rss_key_buf_pa, name);
1163         if (!rss_key_buf_va)
1164                 return -ENOMEM;
1165
1166         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1167                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1168
1169         err = enic_set_rss_key(enic,
1170                 rss_key_buf_pa,
1171                 sizeof(union vnic_rss_key));
1172
1173         /* Save for later queries */
1174         if (!err) {
1175                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1176                            sizeof(union vnic_rss_key));
1177         }
1178         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1179                 rss_key_buf_va, rss_key_buf_pa);
1180
1181         return err;
1182 }
1183
1184 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1185 {
1186         dma_addr_t rss_cpu_buf_pa;
1187         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1188         int err;
1189         u8 name[RTE_MEMZONE_NAMESIZE];
1190
1191         snprintf((char *)name, sizeof(name), "rss_cpu-%s", enic->bdf_name);
1192         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1193                 &rss_cpu_buf_pa, name);
1194         if (!rss_cpu_buf_va)
1195                 return -ENOMEM;
1196
1197         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1198
1199         err = enic_set_rss_cpu(enic,
1200                 rss_cpu_buf_pa,
1201                 sizeof(union vnic_rss_cpu));
1202
1203         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1204                 rss_cpu_buf_va, rss_cpu_buf_pa);
1205
1206         /* Save for later queries */
1207         if (!err)
1208                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1209         return err;
1210 }
1211
1212 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1213         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1214 {
1215         const u8 tso_ipid_split_en = 0;
1216         int err;
1217
1218         err = enic_set_nic_cfg(enic,
1219                 rss_default_cpu, rss_hash_type,
1220                 rss_hash_bits, rss_base_cpu,
1221                 rss_enable, tso_ipid_split_en,
1222                 enic->ig_vlan_strip_en);
1223
1224         return err;
1225 }
1226
1227 /* Initialize RSS with defaults, called from dev_configure */
1228 int enic_init_rss_nic_cfg(struct enic *enic)
1229 {
1230         static uint8_t default_rss_key[] = {
1231                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1232                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1233                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1234                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1235         };
1236         struct rte_eth_rss_conf rss_conf;
1237         union vnic_rss_cpu rss_cpu;
1238         int ret, i;
1239
1240         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1241         /*
1242          * If setting key for the first time, and the user gives us none, then
1243          * push the default key to NIC.
1244          */
1245         if (rss_conf.rss_key == NULL) {
1246                 rss_conf.rss_key = default_rss_key;
1247                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1248         }
1249         ret = enic_set_rss_conf(enic, &rss_conf);
1250         if (ret) {
1251                 dev_err(enic, "Failed to configure RSS\n");
1252                 return ret;
1253         }
1254         if (enic->rss_enable) {
1255                 /* If enabling RSS, use the default reta */
1256                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1257                         rss_cpu.cpu[i / 4].b[i % 4] =
1258                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1259                 }
1260                 ret = enic_set_rss_reta(enic, &rss_cpu);
1261                 if (ret)
1262                         dev_err(enic, "Failed to set RSS indirection table\n");
1263         }
1264         return ret;
1265 }
1266
1267 int enic_setup_finish(struct enic *enic)
1268 {
1269         enic_init_soft_stats(enic);
1270
1271         /* Default conf */
1272         vnic_dev_packet_filter(enic->vdev,
1273                 1 /* directed  */,
1274                 1 /* multicast */,
1275                 1 /* broadcast */,
1276                 0 /* promisc   */,
1277                 1 /* allmulti  */);
1278
1279         enic->promisc = 0;
1280         enic->allmulti = 1;
1281
1282         return 0;
1283 }
1284
1285 static int enic_rss_conf_valid(struct enic *enic,
1286                                struct rte_eth_rss_conf *rss_conf)
1287 {
1288         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1289         if (enic->flow_type_rss_offloads == 0)
1290                 return 0;
1291         if (rss_conf->rss_key != NULL &&
1292             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1293                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1294                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1295                 return -EINVAL;
1296         }
1297         if (rss_conf->rss_hf != 0 &&
1298             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1299                 dev_err(enic, "Given rss_hf contains none of the supported"
1300                         " types\n");
1301                 return -EINVAL;
1302         }
1303         return 0;
1304 }
1305
1306 /* Set hash type and key according to rss_conf */
1307 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1308 {
1309         struct rte_eth_dev *eth_dev;
1310         uint64_t rss_hf;
1311         u8 rss_hash_type;
1312         u8 rss_enable;
1313         int ret;
1314
1315         RTE_ASSERT(rss_conf != NULL);
1316         ret = enic_rss_conf_valid(enic, rss_conf);
1317         if (ret) {
1318                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1319                 return ret;
1320         }
1321
1322         eth_dev = enic->rte_dev;
1323         rss_hash_type = 0;
1324         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1325         if (enic->rq_count > 1 &&
1326             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1327             rss_hf != 0) {
1328                 rss_enable = 1;
1329                 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1330                               ETH_RSS_NONFRAG_IPV4_OTHER))
1331                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1332                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1333                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1334                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1335                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1336                         if (enic->udp_rss_weak) {
1337                                 /*
1338                                  * 'TCP' is not a typo. The "weak" version of
1339                                  * UDP RSS requires both the TCP and UDP bits
1340                                  * be set. It does enable TCP RSS as well.
1341                                  */
1342                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1343                         }
1344                 }
1345                 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1346                               ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1347                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1348                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1349                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1350                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1351                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1352                         if (enic->udp_rss_weak)
1353                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1354                 }
1355         } else {
1356                 rss_enable = 0;
1357                 rss_hf = 0;
1358         }
1359
1360         /* Set the hash key if provided */
1361         if (rss_enable && rss_conf->rss_key) {
1362                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1363                 if (ret) {
1364                         dev_err(enic, "Failed to set RSS key\n");
1365                         return ret;
1366                 }
1367         }
1368
1369         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1370                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1371                               rss_enable);
1372         if (!ret) {
1373                 enic->rss_hf = rss_hf;
1374                 enic->rss_hash_type = rss_hash_type;
1375                 enic->rss_enable = rss_enable;
1376         } else {
1377                 dev_err(enic, "Failed to update RSS configurations."
1378                         " hash=0x%x\n", rss_hash_type);
1379         }
1380         return ret;
1381 }
1382
1383 int enic_set_vlan_strip(struct enic *enic)
1384 {
1385         /*
1386          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1387          * together. So, re-do niccfg, preserving the current RSS settings.
1388          */
1389         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1390                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1391                                enic->rss_enable);
1392 }
1393
1394 int enic_add_packet_filter(struct enic *enic)
1395 {
1396         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1397         return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1398                 enic->promisc, enic->allmulti);
1399 }
1400
1401 int enic_get_link_status(struct enic *enic)
1402 {
1403         return vnic_dev_link_status(enic->vdev);
1404 }
1405
1406 static void enic_dev_deinit(struct enic *enic)
1407 {
1408         /* stop link status checking */
1409         vnic_dev_notify_unset(enic->vdev);
1410
1411         /* mac_addrs is freed by rte_eth_dev_release_port() */
1412         rte_free(enic->cq);
1413         rte_free(enic->intr);
1414         rte_free(enic->rq);
1415         rte_free(enic->wq);
1416 }
1417
1418
1419 int enic_set_vnic_res(struct enic *enic)
1420 {
1421         struct rte_eth_dev *eth_dev = enic->rte_dev;
1422         int rc = 0;
1423         unsigned int required_rq, required_wq, required_cq, required_intr;
1424
1425         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1426         required_rq = eth_dev->data->nb_rx_queues * 2;
1427         required_wq = eth_dev->data->nb_tx_queues;
1428         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1429         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1430         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1431                 required_intr += eth_dev->data->nb_rx_queues;
1432         }
1433
1434         if (enic->conf_rq_count < required_rq) {
1435                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1436                         eth_dev->data->nb_rx_queues,
1437                         required_rq, enic->conf_rq_count);
1438                 rc = -EINVAL;
1439         }
1440         if (enic->conf_wq_count < required_wq) {
1441                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1442                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1443                 rc = -EINVAL;
1444         }
1445
1446         if (enic->conf_cq_count < required_cq) {
1447                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1448                         required_cq, enic->conf_cq_count);
1449                 rc = -EINVAL;
1450         }
1451         if (enic->conf_intr_count < required_intr) {
1452                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1453                         " interrupts. Required:%u, Configured:%u\n",
1454                         required_intr, enic->conf_intr_count);
1455                 rc = -EINVAL;
1456         }
1457
1458         if (rc == 0) {
1459                 enic->rq_count = eth_dev->data->nb_rx_queues;
1460                 enic->wq_count = eth_dev->data->nb_tx_queues;
1461                 enic->cq_count = enic->rq_count + enic->wq_count;
1462                 enic->intr_count = required_intr;
1463         }
1464
1465         return rc;
1466 }
1467
1468 /* Initialize the completion queue for an RQ */
1469 static int
1470 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1471 {
1472         struct vnic_rq *sop_rq, *data_rq;
1473         unsigned int cq_idx;
1474         int rc = 0;
1475
1476         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1477         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1478         cq_idx = rq_idx;
1479
1480         vnic_cq_clean(&enic->cq[cq_idx]);
1481         vnic_cq_init(&enic->cq[cq_idx],
1482                      0 /* flow_control_enable */,
1483                      1 /* color_enable */,
1484                      0 /* cq_head */,
1485                      0 /* cq_tail */,
1486                      1 /* cq_tail_color */,
1487                      0 /* interrupt_enable */,
1488                      1 /* cq_entry_enable */,
1489                      0 /* cq_message_enable */,
1490                      0 /* interrupt offset */,
1491                      0 /* cq_message_addr */);
1492
1493
1494         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1495                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1496                            sop_rq->ring.desc_count - 1, 1, 0);
1497         if (data_rq->in_use) {
1498                 vnic_rq_init_start(data_rq,
1499                                    enic_cq_rq(enic,
1500                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1501                                    data_rq->ring.desc_count - 1, 1, 0);
1502         }
1503
1504         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1505         if (rc)
1506                 return rc;
1507
1508         if (data_rq->in_use) {
1509                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1510                 if (rc) {
1511                         enic_rxmbuf_queue_release(enic, sop_rq);
1512                         return rc;
1513                 }
1514         }
1515
1516         return 0;
1517 }
1518
1519 /* The Cisco NIC can send and receive packets up to a max packet size
1520  * determined by the NIC type and firmware. There is also an MTU
1521  * configured into the NIC via the CIMC/UCSM management interface
1522  * which can be overridden by this function (up to the max packet size).
1523  * Depending on the network setup, doing so may cause packet drops
1524  * and unexpected behavior.
1525  */
1526 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1527 {
1528         unsigned int rq_idx;
1529         struct vnic_rq *rq;
1530         int rc = 0;
1531         uint16_t old_mtu;       /* previous setting */
1532         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1533         struct rte_eth_dev *eth_dev = enic->rte_dev;
1534
1535         old_mtu = eth_dev->data->mtu;
1536         config_mtu = enic->config.mtu;
1537
1538         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1539                 return -E_RTE_SECONDARY;
1540
1541         if (new_mtu > enic->max_mtu) {
1542                 dev_err(enic,
1543                         "MTU not updated: requested (%u) greater than max (%u)\n",
1544                         new_mtu, enic->max_mtu);
1545                 return -EINVAL;
1546         }
1547         if (new_mtu < ENIC_MIN_MTU) {
1548                 dev_info(enic,
1549                         "MTU not updated: requested (%u) less than min (%u)\n",
1550                         new_mtu, ENIC_MIN_MTU);
1551                 return -EINVAL;
1552         }
1553         if (new_mtu > config_mtu)
1554                 dev_warning(enic,
1555                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1556                         new_mtu, config_mtu);
1557
1558         /* Update the MTU and maximum packet length */
1559         eth_dev->data->mtu = new_mtu;
1560         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1561                 enic_mtu_to_max_rx_pktlen(new_mtu);
1562
1563         /*
1564          * If the device has not started (enic_enable), nothing to do.
1565          * Later, enic_enable() will set up RQs reflecting the new maximum
1566          * packet length.
1567          */
1568         if (!eth_dev->data->dev_started)
1569                 goto set_mtu_done;
1570
1571         /*
1572          * The device has started, re-do RQs on the fly. In the process, we
1573          * pick up the new maximum packet length.
1574          *
1575          * Some applications rely on the ability to change MTU without stopping
1576          * the device. So keep this behavior for now.
1577          */
1578         rte_spinlock_lock(&enic->mtu_lock);
1579
1580         /* Stop traffic on all RQs */
1581         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1582                 rq = &enic->rq[rq_idx];
1583                 if (rq->is_sop && rq->in_use) {
1584                         rc = enic_stop_rq(enic,
1585                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1586                         if (rc) {
1587                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1588                                 goto set_mtu_done;
1589                         }
1590                 }
1591         }
1592
1593         /* replace Rx function with a no-op to avoid getting stale pkts */
1594         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1595         rte_mb();
1596
1597         /* Allow time for threads to exit the real Rx function. */
1598         usleep(100000);
1599
1600         /* now it is safe to reconfigure the RQs */
1601
1602
1603         /* free and reallocate RQs with the new MTU */
1604         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1605                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1606                 if (!rq->in_use)
1607                         continue;
1608
1609                 enic_free_rq(rq);
1610                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1611                                    rq->tot_nb_desc, rq->rx_free_thresh);
1612                 if (rc) {
1613                         dev_err(enic,
1614                                 "Fatal MTU alloc error- No traffic will pass\n");
1615                         goto set_mtu_done;
1616                 }
1617
1618                 rc = enic_reinit_rq(enic, rq_idx);
1619                 if (rc) {
1620                         dev_err(enic,
1621                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1622                         goto set_mtu_done;
1623                 }
1624         }
1625
1626         /* put back the real receive function */
1627         rte_mb();
1628         enic_pick_rx_handler(eth_dev);
1629         rte_mb();
1630
1631         /* restart Rx traffic */
1632         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1633                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1634                 if (rq->is_sop && rq->in_use)
1635                         enic_start_rq(enic, rq_idx);
1636         }
1637
1638 set_mtu_done:
1639         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1640         rte_spinlock_unlock(&enic->mtu_lock);
1641         return rc;
1642 }
1643
1644 static int enic_dev_init(struct enic *enic)
1645 {
1646         int err;
1647         struct rte_eth_dev *eth_dev = enic->rte_dev;
1648
1649         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1650
1651         /* Get vNIC configuration
1652         */
1653         err = enic_get_vnic_config(enic);
1654         if (err) {
1655                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1656                 return err;
1657         }
1658
1659         /* Get available resource counts */
1660         enic_get_res_counts(enic);
1661         if (enic->conf_rq_count == 1) {
1662                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1663                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1664                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1665                 return -EINVAL;
1666         }
1667         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1668         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1669                                enic->conf_cq_count, 8);
1670         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1671                                  enic->conf_intr_count, 8);
1672         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1673                                enic->conf_rq_count, 8);
1674         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1675                                enic->conf_wq_count, 8);
1676         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1677                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1678                 return -1;
1679         }
1680         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1681                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1682                 return -1;
1683         }
1684         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1685                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1686                 return -1;
1687         }
1688         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1689                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1690                 return -1;
1691         }
1692
1693         /* Get the supported filters */
1694         enic_fdir_info(enic);
1695
1696         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1697                                         sizeof(struct rte_ether_addr) *
1698                                         ENIC_UNICAST_PERFECT_FILTERS, 0);
1699         if (!eth_dev->data->mac_addrs) {
1700                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1701                 return -1;
1702         }
1703         rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1704                         eth_dev->data->mac_addrs);
1705
1706         vnic_dev_set_reset_flag(enic->vdev, 0);
1707
1708         LIST_INIT(&enic->flows);
1709
1710         /* set up link status checking */
1711         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1712
1713         /*
1714          * When Geneve with options offload is available, always disable it
1715          * first as it can interfere with user flow rules.
1716          */
1717         if (enic->geneve_opt_avail) {
1718                 /*
1719                  * Disabling fails if the feature is provisioned but
1720                  * not enabled. So ignore result and do not log error.
1721                  */
1722                 vnic_dev_overlay_offload_ctrl(enic->vdev,
1723                         OVERLAY_FEATURE_GENEVE,
1724                         OVERLAY_OFFLOAD_DISABLE);
1725         }
1726         enic->overlay_offload = false;
1727         if (enic->disable_overlay && enic->vxlan) {
1728                 /*
1729                  * Explicitly disable overlay offload as the setting is
1730                  * sticky, and resetting vNIC does not disable it.
1731                  */
1732                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1733                                                   OVERLAY_FEATURE_VXLAN,
1734                                                   OVERLAY_OFFLOAD_DISABLE)) {
1735                         dev_err(enic, "failed to disable overlay offload\n");
1736                 } else {
1737                         dev_info(enic, "Overlay offload is disabled\n");
1738                 }
1739         }
1740         if (!enic->disable_overlay && enic->vxlan &&
1741             /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1742             vnic_dev_overlay_offload_ctrl(enic->vdev,
1743                                           OVERLAY_FEATURE_VXLAN,
1744                                           OVERLAY_OFFLOAD_ENABLE) == 0) {
1745                 enic->tx_offload_capa |=
1746                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1747                         DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1748                         DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1749                 enic->tx_offload_mask |=
1750                         PKT_TX_OUTER_IPV6 |
1751                         PKT_TX_OUTER_IPV4 |
1752                         PKT_TX_OUTER_IP_CKSUM |
1753                         PKT_TX_TUNNEL_MASK;
1754                 enic->overlay_offload = true;
1755                 dev_info(enic, "Overlay offload is enabled\n");
1756         }
1757         /* Geneve with options offload requires overlay offload */
1758         if (enic->overlay_offload && enic->geneve_opt_avail &&
1759             enic->geneve_opt_request) {
1760                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1761                                 OVERLAY_FEATURE_GENEVE,
1762                                 OVERLAY_OFFLOAD_ENABLE)) {
1763                         dev_err(enic, "failed to enable geneve+option\n");
1764                 } else {
1765                         enic->geneve_opt_enabled = 1;
1766                         dev_info(enic, "Geneve with options is enabled\n");
1767                 }
1768         }
1769         /*
1770          * Reset the vxlan port if HW vxlan parsing is available. It
1771          * is always enabled regardless of overlay offload
1772          * enable/disable.
1773          */
1774         if (enic->vxlan) {
1775                 enic->vxlan_port = RTE_VXLAN_DEFAULT_PORT;
1776                 /*
1777                  * Reset the vxlan port to the default, as the NIC firmware
1778                  * does not reset it automatically and keeps the old setting.
1779                  */
1780                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1781                                                  OVERLAY_CFG_VXLAN_PORT_UPDATE,
1782                                                  RTE_VXLAN_DEFAULT_PORT)) {
1783                         dev_err(enic, "failed to update vxlan port\n");
1784                         return -EINVAL;
1785                 }
1786         }
1787
1788         return 0;
1789
1790 }
1791
1792 int enic_probe(struct enic *enic)
1793 {
1794         struct rte_pci_device *pdev = enic->pdev;
1795         int err = -1;
1796
1797         dev_debug(enic, "Initializing ENIC PMD\n");
1798
1799         /* if this is a secondary process the hardware is already initialized */
1800         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1801                 return 0;
1802
1803         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1804         enic->bar0.len = pdev->mem_resource[0].len;
1805
1806         /* Register vNIC device */
1807         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1808         if (!enic->vdev) {
1809                 dev_err(enic, "vNIC registration failed, aborting\n");
1810                 goto err_out;
1811         }
1812
1813         LIST_INIT(&enic->memzone_list);
1814         rte_spinlock_init(&enic->memzone_list_lock);
1815
1816         vnic_register_cbacks(enic->vdev,
1817                 enic_alloc_consistent,
1818                 enic_free_consistent);
1819
1820         /*
1821          * Allocate the consistent memory for stats upfront so both primary and
1822          * secondary processes can dump stats.
1823          */
1824         err = vnic_dev_alloc_stats_mem(enic->vdev);
1825         if (err) {
1826                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1827                 goto err_out_unregister;
1828         }
1829         /* Issue device open to get device in known state */
1830         err = enic_dev_open(enic);
1831         if (err) {
1832                 dev_err(enic, "vNIC dev open failed, aborting\n");
1833                 goto err_out_unregister;
1834         }
1835
1836         /* Set ingress vlan rewrite mode before vnic initialization */
1837         dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1838                   enic->ig_vlan_rewrite_mode);
1839         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1840                 enic->ig_vlan_rewrite_mode);
1841         if (err) {
1842                 dev_err(enic,
1843                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1844                 goto err_out_dev_close;
1845         }
1846
1847         /* Issue device init to initialize the vnic-to-switch link.
1848          * We'll start with carrier off and wait for link UP
1849          * notification later to turn on carrier.  We don't need
1850          * to wait here for the vnic-to-switch link initialization
1851          * to complete; link UP notification is the indication that
1852          * the process is complete.
1853          */
1854
1855         err = vnic_dev_init(enic->vdev, 0);
1856         if (err) {
1857                 dev_err(enic, "vNIC dev init failed, aborting\n");
1858                 goto err_out_dev_close;
1859         }
1860
1861         err = enic_dev_init(enic);
1862         if (err) {
1863                 dev_err(enic, "Device initialization failed, aborting\n");
1864                 goto err_out_dev_close;
1865         }
1866
1867         return 0;
1868
1869 err_out_dev_close:
1870         vnic_dev_close(enic->vdev);
1871 err_out_unregister:
1872         vnic_dev_unregister(enic->vdev);
1873 err_out:
1874         return err;
1875 }
1876
1877 void enic_remove(struct enic *enic)
1878 {
1879         enic_dev_deinit(enic);
1880         vnic_dev_close(enic->vdev);
1881         vnic_dev_unregister(enic->vdev);
1882 }