drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11 #include <libgen.h>
  12
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_ethdev_driver.h>
  20
  21 #include "enic_compat.h"
  22 #include "enic.h"
  23 #include "wq_enet_desc.h"
  24 #include "rq_enet_desc.h"
  25 #include "cq_enet_desc.h"
  26 #include "vnic_enet.h"
  27 #include "vnic_dev.h"
  28 #include "vnic_wq.h"
  29 #include "vnic_rq.h"
  30 #include "vnic_cq.h"
  31 #include "vnic_intr.h"
  32 #include "vnic_nic.h"
  33
  34 static inline int enic_is_sriov_vf(struct enic *enic)
  35 {
  36         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  37 }
  38
  39 static int is_zero_addr(uint8_t *addr)
  40 {
  41         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  42 }
  43
  44 static int is_mcast_addr(uint8_t *addr)
  45 {
  46         return addr[0] & 1;
  47 }
  48
  49 static int is_eth_addr_valid(uint8_t *addr)
  50 {
  51         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  52 }
  53
  54 static void
  55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  56 {
  57         uint16_t i;
  58
  59         if (!rq || !rq->mbuf_ring) {
  60                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  61                 return;
  62         }
  63
  64         for (i = 0; i < rq->ring.desc_count; i++) {
  65                 if (rq->mbuf_ring[i]) {
  66                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  67                         rq->mbuf_ring[i] = NULL;
  68                 }
  69         }
  70 }
  71
  72 static void enic_free_wq_buf(struct vnic_wq_buf *buf)
  73 {
  74         struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
  75
  76         rte_pktmbuf_free_seg(mbuf);
  77         buf->mb = NULL;
  78 }
  79
  80 static void enic_log_q_error(struct enic *enic)
  81 {
  82         unsigned int i;
  83         u32 error_status;
  84
  85         for (i = 0; i < enic->wq_count; i++) {
  86                 error_status = vnic_wq_error_status(&enic->wq[i]);
  87                 if (error_status)
  88                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  89                                 error_status);
  90         }
  91
  92         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  93                 if (!enic->rq[i].in_use)
  94                         continue;
  95                 error_status = vnic_rq_error_status(&enic->rq[i]);
  96                 if (error_status)
  97                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  98                                 error_status);
  99         }
 100 }
 101
 102 static void enic_clear_soft_stats(struct enic *enic)
 103 {
 104         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 105         rte_atomic64_clear(&soft_stats->rx_nombuf);
 106         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 107         rte_atomic64_clear(&soft_stats->tx_oversized);
 108 }
 109
 110 static void enic_init_soft_stats(struct enic *enic)
 111 {
 112         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 113         rte_atomic64_init(&soft_stats->rx_nombuf);
 114         rte_atomic64_init(&soft_stats->rx_packet_errors);
 115         rte_atomic64_init(&soft_stats->tx_oversized);
 116         enic_clear_soft_stats(enic);
 117 }
 118
 119 void enic_dev_stats_clear(struct enic *enic)
 120 {
 121         if (vnic_dev_stats_clear(enic->vdev))
 122                 dev_err(enic, "Error in clearing stats\n");
 123         enic_clear_soft_stats(enic);
 124 }
 125
 126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 127 {
 128         struct vnic_stats *stats;
 129         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 130         int64_t rx_truncated;
 131         uint64_t rx_packet_errors;
 132         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 133
 134         if (ret) {
 135                 dev_err(enic, "Error in getting stats\n");
 136                 return ret;
 137         }
 138
 139         /* The number of truncated packets can only be calculated by
 140          * subtracting a hardware counter from error packets received by
 141          * the driver. Note: this causes transient inaccuracies in the
 142          * ipackets count. Also, the length of truncated packets are
 143          * counted in ibytes even though truncated packets are dropped
 144          * which can make ibytes be slightly higher than it should be.
 145          */
 146         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 147         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 148
 149         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 150         r_stats->opackets = stats->tx.tx_frames_ok;
 151
 152         r_stats->ibytes = stats->rx.rx_bytes_ok;
 153         r_stats->obytes = stats->tx.tx_bytes_ok;
 154
 155         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 156         r_stats->oerrors = stats->tx.tx_errors
 157                            + rte_atomic64_read(&soft_stats->tx_oversized);
 158
 159         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 160
 161         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 162         return 0;
 163 }
 164
 165 void enic_del_mac_address(struct enic *enic, int mac_index)
 166 {
 167         struct rte_eth_dev *eth_dev = enic->rte_dev;
 168         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 169
 170         if (vnic_dev_del_addr(enic->vdev, mac_addr))
 171                 dev_err(enic, "del mac addr failed\n");
 172 }
 173
 174 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 175 {
 176         int err;
 177
 178         if (!is_eth_addr_valid(mac_addr)) {
 179                 dev_err(enic, "invalid mac address\n");
 180                 return -EINVAL;
 181         }
 182
 183         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 184         if (err)
 185                 dev_err(enic, "add mac addr failed\n");
 186         return err;
 187 }
 188
 189 static void
 190 enic_free_rq_buf(struct rte_mbuf **mbuf)
 191 {
 192         if (*mbuf == NULL)
 193                 return;
 194
 195         rte_pktmbuf_free(*mbuf);
 196         *mbuf = NULL;
 197 }
 198
 199 void enic_init_vnic_resources(struct enic *enic)
 200 {
 201         unsigned int error_interrupt_enable = 1;
 202         unsigned int error_interrupt_offset = 0;
 203         unsigned int rxq_interrupt_enable = 0;
 204         unsigned int rxq_interrupt_offset;
 205         unsigned int index = 0;
 206         unsigned int cq_idx;
 207         struct vnic_rq *data_rq;
 208
 209         if (enic->rte_dev->data->dev_conf.intr_conf.rxq) {
 210                 rxq_interrupt_enable = 1;
 211                 rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 212         }
 213         for (index = 0; index < enic->rq_count; index++) {
 214                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 215
 216                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 217                         cq_idx,
 218                         error_interrupt_enable,
 219                         error_interrupt_offset);
 220
 221                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 222                 if (data_rq->in_use)
 223                         vnic_rq_init(data_rq,
 224                                      cq_idx,
 225                                      error_interrupt_enable,
 226                                      error_interrupt_offset);
 227
 228                 vnic_cq_init(&enic->cq[cq_idx],
 229                         0 /* flow_control_enable */,
 230                         1 /* color_enable */,
 231                         0 /* cq_head */,
 232                         0 /* cq_tail */,
 233                         1 /* cq_tail_color */,
 234                         rxq_interrupt_enable,
 235                         1 /* cq_entry_enable */,
 236                         0 /* cq_message_enable */,
 237                         rxq_interrupt_offset,
 238                         0 /* cq_message_addr */);
 239                 if (rxq_interrupt_enable)
 240                         rxq_interrupt_offset++;
 241         }
 242
 243         for (index = 0; index < enic->wq_count; index++) {
 244                 vnic_wq_init(&enic->wq[index],
 245                         enic_cq_wq(enic, index),
 246                         error_interrupt_enable,
 247                         error_interrupt_offset);
 248
 249                 cq_idx = enic_cq_wq(enic, index);
 250                 vnic_cq_init(&enic->cq[cq_idx],
 251                         0 /* flow_control_enable */,
 252                         1 /* color_enable */,
 253                         0 /* cq_head */,
 254                         0 /* cq_tail */,
 255                         1 /* cq_tail_color */,
 256                         0 /* interrupt_enable */,
 257                         0 /* cq_entry_enable */,
 258                         1 /* cq_message_enable */,
 259                         0 /* interrupt offset */,
 260                         (u64)enic->wq[index].cqmsg_rz->iova);
 261         }
 262
 263         for (index = 0; index < enic->intr_count; index++) {
 264                 vnic_intr_init(&enic->intr[index],
 265                                enic->config.intr_timer_usec,
 266                                enic->config.intr_timer_type,
 267                                /*mask_on_assertion*/1);
 268         }
 269 }
 270
 271
 272 static int
 273 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 274 {
 275         struct rte_mbuf *mb;
 276         struct rq_enet_desc *rqd = rq->ring.descs;
 277         unsigned i;
 278         dma_addr_t dma_addr;
 279         uint32_t max_rx_pkt_len;
 280         uint16_t rq_buf_len;
 281
 282         if (!rq->in_use)
 283                 return 0;
 284
 285         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 286                   rq->ring.desc_count);
 287
 288         /*
 289          * If *not* using scatter and the mbuf size is smaller than the
 290          * requested max packet size (max_rx_pkt_len), then reduce the
 291          * posted buffer size to max_rx_pkt_len. HW still receives packets
 292          * larger than max_rx_pkt_len, but they will be truncated, which we
 293          * drop in the rx handler. Not ideal, but better than returning
 294          * large packets when the user is not expecting them.
 295          */
 296         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 297         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 298         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 299                 rq_buf_len = max_rx_pkt_len;
 300         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 301                 mb = rte_mbuf_raw_alloc(rq->mp);
 302                 if (mb == NULL) {
 303                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 304                         (unsigned)rq->index);
 305                         return -ENOMEM;
 306                 }
 307
 308                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 309                 dma_addr = (dma_addr_t)(mb->buf_iova
 310                            + RTE_PKTMBUF_HEADROOM);
 311                 rq_enet_desc_enc(rqd, dma_addr,
 312                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 313                                 : RQ_ENET_TYPE_NOT_SOP),
 314                                 rq_buf_len);
 315                 rq->mbuf_ring[i] = mb;
 316         }
 317
 318         /* make sure all prior writes are complete before doing the PIO write */
 319         rte_rmb();
 320
 321         /* Post all but the last buffer to VIC. */
 322         rq->posted_index = rq->ring.desc_count - 1;
 323
 324         rq->rx_nb_hold = 0;
 325
 326         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 327                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 328         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 329         iowrite32(0, &rq->ctrl->fetch_index);
 330         rte_rmb();
 331
 332         return 0;
 333
 334 }
 335
 336 static void *
 337 enic_alloc_consistent(void *priv, size_t size,
 338         dma_addr_t *dma_handle, u8 *name)
 339 {
 340         void *vaddr;
 341         const struct rte_memzone *rz;
 342         *dma_handle = 0;
 343         struct enic *enic = (struct enic *)priv;
 344         struct enic_memzone_entry *mze;
 345
 346         rz = rte_memzone_reserve_aligned((const char *)name, size,
 347                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 348         if (!rz) {
 349                 pr_err("%s : Failed to allocate memory requested for %s\n",
 350                         __func__, name);
 351                 return NULL;
 352         }
 353
 354         vaddr = rz->addr;
 355         *dma_handle = (dma_addr_t)rz->iova;
 356
 357         mze = rte_malloc("enic memzone entry",
 358                          sizeof(struct enic_memzone_entry), 0);
 359
 360         if (!mze) {
 361                 pr_err("%s : Failed to allocate memory for memzone list\n",
 362                        __func__);
 363                 rte_memzone_free(rz);
 364                 return NULL;
 365         }
 366
 367         mze->rz = rz;
 368
 369         rte_spinlock_lock(&enic->memzone_list_lock);
 370         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 371         rte_spinlock_unlock(&enic->memzone_list_lock);
 372
 373         return vaddr;
 374 }
 375
 376 static void
 377 enic_free_consistent(void *priv,
 378                      __rte_unused size_t size,
 379                      void *vaddr,
 380                      dma_addr_t dma_handle)
 381 {
 382         struct enic_memzone_entry *mze;
 383         struct enic *enic = (struct enic *)priv;
 384
 385         rte_spinlock_lock(&enic->memzone_list_lock);
 386         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 387                 if (mze->rz->addr == vaddr &&
 388                     mze->rz->iova == dma_handle)
 389                         break;
 390         }
 391         if (mze == NULL) {
 392                 rte_spinlock_unlock(&enic->memzone_list_lock);
 393                 dev_warning(enic,
 394                             "Tried to free memory, but couldn't find it in the memzone list\n");
 395                 return;
 396         }
 397         LIST_REMOVE(mze, entries);
 398         rte_spinlock_unlock(&enic->memzone_list_lock);
 399         rte_memzone_free(mze->rz);
 400         rte_free(mze);
 401 }
 402
 403 int enic_link_update(struct enic *enic)
 404 {
 405         struct rte_eth_dev *eth_dev = enic->rte_dev;
 406         struct rte_eth_link link;
 407
 408         memset(&link, 0, sizeof(link));
 409         link.link_status = enic_get_link_status(enic);
 410         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 411         link.link_speed = vnic_dev_port_speed(enic->vdev);
 412
 413         return rte_eth_linkstatus_set(eth_dev, &link);
 414 }
 415
 416 static void
 417 enic_intr_handler(void *arg)
 418 {
 419         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 420         struct enic *enic = pmd_priv(dev);
 421
 422         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 423
 424         enic_link_update(enic);
 425         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 426         enic_log_q_error(enic);
 427 }
 428
 429 static int enic_rxq_intr_init(struct enic *enic)
 430 {
 431         struct rte_intr_handle *intr_handle;
 432         uint32_t rxq_intr_count, i;
 433         int err;
 434
 435         intr_handle = enic->rte_dev->intr_handle;
 436         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 437                 return 0;
 438         /*
 439          * Rx queue interrupts only work when we have MSI-X interrupts,
 440          * one per queue. Sharing one interrupt is technically
 441          * possible with VIC, but it is not worth the complications it brings.
 442          */
 443         if (!rte_intr_cap_multiple(intr_handle)) {
 444                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 445                         " (vfio-pci driver)\n");
 446                 return -ENOTSUP;
 447         }
 448         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 449         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 450         if (err) {
 451                 dev_err(enic, "Failed to enable event fds for Rx queue"
 452                         " interrupts\n");
 453                 return err;
 454         }
 455         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 456                                             rxq_intr_count * sizeof(int), 0);
 457         if (intr_handle->intr_vec == NULL) {
 458                 dev_err(enic, "Failed to allocate intr_vec\n");
 459                 return -ENOMEM;
 460         }
 461         for (i = 0; i < rxq_intr_count; i++)
 462                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 463         return 0;
 464 }
 465
 466 static void enic_rxq_intr_deinit(struct enic *enic)
 467 {
 468         struct rte_intr_handle *intr_handle;
 469
 470         intr_handle = enic->rte_dev->intr_handle;
 471         rte_intr_efd_disable(intr_handle);
 472         if (intr_handle->intr_vec != NULL) {
 473                 rte_free(intr_handle->intr_vec);
 474                 intr_handle->intr_vec = NULL;
 475         }
 476 }
 477
 478 int enic_enable(struct enic *enic)
 479 {
 480         unsigned int index;
 481         int err;
 482         struct rte_eth_dev *eth_dev = enic->rte_dev;
 483
 484         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 485         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 486
 487         /* vnic notification of link status has already been turned on in
 488          * enic_dev_init() which is called during probe time.  Here we are
 489          * just turning on interrupt vector 0 if needed.
 490          */
 491         if (eth_dev->data->dev_conf.intr_conf.lsc)
 492                 vnic_dev_notify_set(enic->vdev, 0);
 493
 494         err = enic_rxq_intr_init(enic);
 495         if (err)
 496                 return err;
 497         if (enic_clsf_init(enic))
 498                 dev_warning(enic, "Init of hash table for clsf failed."\
 499                         "Flow director feature will not work\n");
 500
 501         for (index = 0; index < enic->rq_count; index++) {
 502                 err = enic_alloc_rx_queue_mbufs(enic,
 503                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 504                 if (err) {
 505                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 506                         return err;
 507                 }
 508                 err = enic_alloc_rx_queue_mbufs(enic,
 509                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 510                 if (err) {
 511                         /* release the allocated mbufs for the sop rq*/
 512                         enic_rxmbuf_queue_release(enic,
 513                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 514
 515                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 516                         return err;
 517                 }
 518         }
 519
 520         for (index = 0; index < enic->wq_count; index++)
 521                 enic_start_wq(enic, index);
 522         for (index = 0; index < enic->rq_count; index++)
 523                 enic_start_rq(enic, index);
 524
 525         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 526
 527         vnic_dev_enable_wait(enic->vdev);
 528
 529         /* Register and enable error interrupt */
 530         rte_intr_callback_register(&(enic->pdev->intr_handle),
 531                 enic_intr_handler, (void *)enic->rte_dev);
 532
 533         rte_intr_enable(&(enic->pdev->intr_handle));
 534         /* Unmask LSC interrupt */
 535         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 536
 537         return 0;
 538 }
 539
 540 int enic_alloc_intr_resources(struct enic *enic)
 541 {
 542         int err;
 543         unsigned int i;
 544
 545         dev_info(enic, "vNIC resources used:  "\
 546                 "wq %d rq %d cq %d intr %d\n",
 547                 enic->wq_count, enic_vnic_rq_count(enic),
 548                 enic->cq_count, enic->intr_count);
 549
 550         for (i = 0; i < enic->intr_count; i++) {
 551                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 552                 if (err) {
 553                         enic_free_vnic_resources(enic);
 554                         return err;
 555                 }
 556         }
 557         return 0;
 558 }
 559
 560 void enic_free_rq(void *rxq)
 561 {
 562         struct vnic_rq *rq_sop, *rq_data;
 563         struct enic *enic;
 564
 565         if (rxq == NULL)
 566                 return;
 567
 568         rq_sop = (struct vnic_rq *)rxq;
 569         enic = vnic_dev_priv(rq_sop->vdev);
 570         rq_data = &enic->rq[rq_sop->data_queue_idx];
 571
 572         enic_rxmbuf_queue_release(enic, rq_sop);
 573         if (rq_data->in_use)
 574                 enic_rxmbuf_queue_release(enic, rq_data);
 575
 576         rte_free(rq_sop->mbuf_ring);
 577         if (rq_data->in_use)
 578                 rte_free(rq_data->mbuf_ring);
 579
 580         rq_sop->mbuf_ring = NULL;
 581         rq_data->mbuf_ring = NULL;
 582
 583         vnic_rq_free(rq_sop);
 584         if (rq_data->in_use)
 585                 vnic_rq_free(rq_data);
 586
 587         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 588
 589         rq_sop->in_use = 0;
 590         rq_data->in_use = 0;
 591 }
 592
 593 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 594 {
 595         struct rte_eth_dev *eth_dev = enic->rte_dev;
 596         vnic_wq_enable(&enic->wq[queue_idx]);
 597         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 598 }
 599
 600 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 601 {
 602         struct rte_eth_dev *eth_dev = enic->rte_dev;
 603         int ret;
 604
 605         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 606         if (ret)
 607                 return ret;
 608
 609         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 610         return 0;
 611 }
 612
 613 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 614 {
 615         struct vnic_rq *rq_sop;
 616         struct vnic_rq *rq_data;
 617         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 618         rq_data = &enic->rq[rq_sop->data_queue_idx];
 619         struct rte_eth_dev *eth_dev = enic->rte_dev;
 620
 621         if (rq_data->in_use)
 622                 vnic_rq_enable(rq_data);
 623         rte_mb();
 624         vnic_rq_enable(rq_sop);
 625         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 626 }
 627
 628 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 629 {
 630         int ret1 = 0, ret2 = 0;
 631         struct rte_eth_dev *eth_dev = enic->rte_dev;
 632         struct vnic_rq *rq_sop;
 633         struct vnic_rq *rq_data;
 634         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 635         rq_data = &enic->rq[rq_sop->data_queue_idx];
 636
 637         ret2 = vnic_rq_disable(rq_sop);
 638         rte_mb();
 639         if (rq_data->in_use)
 640                 ret1 = vnic_rq_disable(rq_data);
 641
 642         if (ret2)
 643                 return ret2;
 644         else if (ret1)
 645                 return ret1;
 646
 647         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 648         return 0;
 649 }
 650
 651 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 652         unsigned int socket_id, struct rte_mempool *mp,
 653         uint16_t nb_desc, uint16_t free_thresh)
 654 {
 655         int rc;
 656         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 657         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 658         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 659         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 660         unsigned int mbuf_size, mbufs_per_pkt;
 661         unsigned int nb_sop_desc, nb_data_desc;
 662         uint16_t min_sop, max_sop, min_data, max_data;
 663         uint32_t max_rx_pkt_len;
 664
 665         rq_sop->is_sop = 1;
 666         rq_sop->data_queue_idx = data_queue_idx;
 667         rq_data->is_sop = 0;
 668         rq_data->data_queue_idx = 0;
 669         rq_sop->socket_id = socket_id;
 670         rq_sop->mp = mp;
 671         rq_data->socket_id = socket_id;
 672         rq_data->mp = mp;
 673         rq_sop->in_use = 1;
 674         rq_sop->rx_free_thresh = free_thresh;
 675         rq_data->rx_free_thresh = free_thresh;
 676         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 677                   free_thresh);
 678
 679         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 680                                RTE_PKTMBUF_HEADROOM);
 681         /* max_rx_pkt_len includes the ethernet header and CRC. */
 682         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 683
 684         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 685             DEV_RX_OFFLOAD_SCATTER) {
 686                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 687                 /* ceil((max pkt len)/mbuf_size) */
 688                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 689         } else {
 690                 dev_info(enic, "Scatter rx mode disabled\n");
 691                 mbufs_per_pkt = 1;
 692                 if (max_rx_pkt_len > mbuf_size) {
 693                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 694                                     " larger than the mbuf size (%u), and"
 695                                     " scatter is disabled. Larger packets will"
 696                                     " be truncated.\n",
 697                                     max_rx_pkt_len, mbuf_size);
 698                 }
 699         }
 700
 701         if (mbufs_per_pkt > 1) {
 702                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 703                 rq_sop->data_queue_enable = 1;
 704                 rq_data->in_use = 1;
 705                 /*
 706                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 707                  * receives packet sizes up to the "max" MTU.
 708                  * If not using scatter, we can achieve the effect of dropping
 709                  * larger packets by reducing the size of posted buffers.
 710                  * See enic_alloc_rx_queue_mbufs().
 711                  */
 712                 if (max_rx_pkt_len <
 713                     enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu)) {
 714                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 715                                     " when scatter rx mode is in use.\n");
 716                 }
 717         } else {
 718                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 719                          queue_idx);
 720                 rq_sop->data_queue_enable = 0;
 721                 rq_data->in_use = 0;
 722         }
 723
 724         /* number of descriptors have to be a multiple of 32 */
 725         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
 726         nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
 727
 728         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 729         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 730
 731         if (mbufs_per_pkt > 1) {
 732                 min_sop = 64;
 733                 max_sop = ((enic->config.rq_desc_count /
 734                             (mbufs_per_pkt - 1)) & ~0x1F);
 735                 min_data = min_sop * (mbufs_per_pkt - 1);
 736                 max_data = enic->config.rq_desc_count;
 737         } else {
 738                 min_sop = 64;
 739                 max_sop = enic->config.rq_desc_count;
 740                 min_data = 0;
 741                 max_data = 0;
 742         }
 743
 744         if (nb_desc < (min_sop + min_data)) {
 745                 dev_warning(enic,
 746                             "Number of rx descs too low, adjusting to minimum\n");
 747                 nb_sop_desc = min_sop;
 748                 nb_data_desc = min_data;
 749         } else if (nb_desc > (max_sop + max_data)) {
 750                 dev_warning(enic,
 751                             "Number of rx_descs too high, adjusting to maximum\n");
 752                 nb_sop_desc = max_sop;
 753                 nb_data_desc = max_data;
 754         }
 755         if (mbufs_per_pkt > 1) {
 756                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 757                          " rx descriptor range is %u to %u\n",
 758                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 759                          max_sop + max_data);
 760         }
 761         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 762                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 763
 764         /* Allocate sop queue resources */
 765         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 766                 nb_sop_desc, sizeof(struct rq_enet_desc));
 767         if (rc) {
 768                 dev_err(enic, "error in allocation of sop rq\n");
 769                 goto err_exit;
 770         }
 771         nb_sop_desc = rq_sop->ring.desc_count;
 772
 773         if (rq_data->in_use) {
 774                 /* Allocate data queue resources */
 775                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 776                                    nb_data_desc,
 777                                    sizeof(struct rq_enet_desc));
 778                 if (rc) {
 779                         dev_err(enic, "error in allocation of data rq\n");
 780                         goto err_free_rq_sop;
 781                 }
 782                 nb_data_desc = rq_data->ring.desc_count;
 783         }
 784         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 785                            socket_id, nb_sop_desc + nb_data_desc,
 786                            sizeof(struct cq_enet_rq_desc));
 787         if (rc) {
 788                 dev_err(enic, "error in allocation of cq for rq\n");
 789                 goto err_free_rq_data;
 790         }
 791
 792         /* Allocate the mbuf rings */
 793         rq_sop->mbuf_ring = (struct rte_mbuf **)
 794                 rte_zmalloc_socket("rq->mbuf_ring",
 795                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 796                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 797         if (rq_sop->mbuf_ring == NULL)
 798                 goto err_free_cq;
 799
 800         if (rq_data->in_use) {
 801                 rq_data->mbuf_ring = (struct rte_mbuf **)
 802                         rte_zmalloc_socket("rq->mbuf_ring",
 803                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 804                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 805                 if (rq_data->mbuf_ring == NULL)
 806                         goto err_free_sop_mbuf;
 807         }
 808
 809         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 810
 811         return 0;
 812
 813 err_free_sop_mbuf:
 814         rte_free(rq_sop->mbuf_ring);
 815 err_free_cq:
 816         /* cleanup on error */
 817         vnic_cq_free(&enic->cq[queue_idx]);
 818 err_free_rq_data:
 819         if (rq_data->in_use)
 820                 vnic_rq_free(rq_data);
 821 err_free_rq_sop:
 822         vnic_rq_free(rq_sop);
 823 err_exit:
 824         return -ENOMEM;
 825 }
 826
 827 void enic_free_wq(void *txq)
 828 {
 829         struct vnic_wq *wq;
 830         struct enic *enic;
 831
 832         if (txq == NULL)
 833                 return;
 834
 835         wq = (struct vnic_wq *)txq;
 836         enic = vnic_dev_priv(wq->vdev);
 837         rte_memzone_free(wq->cqmsg_rz);
 838         vnic_wq_free(wq);
 839         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 840 }
 841
 842 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 843         unsigned int socket_id, uint16_t nb_desc)
 844 {
 845         int err;
 846         struct vnic_wq *wq = &enic->wq[queue_idx];
 847         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 848         char name[NAME_MAX];
 849         static int instance;
 850
 851         wq->socket_id = socket_id;
 852         if (nb_desc) {
 853                 if (nb_desc > enic->config.wq_desc_count) {
 854                         dev_warning(enic,
 855                                 "WQ %d - number of tx desc in cmd line (%d)"\
 856                                 "is greater than that in the UCSM/CIMC adapter"\
 857                                 "policy.  Applying the value in the adapter "\
 858                                 "policy (%d)\n",
 859                                 queue_idx, nb_desc, enic->config.wq_desc_count);
 860                 } else if (nb_desc != enic->config.wq_desc_count) {
 861                         enic->config.wq_desc_count = nb_desc;
 862                         dev_info(enic,
 863                                 "TX Queues - effective number of descs:%d\n",
 864                                 nb_desc);
 865                 }
 866         }
 867
 868         /* Allocate queue resources */
 869         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
 870                 enic->config.wq_desc_count,
 871                 sizeof(struct wq_enet_desc));
 872         if (err) {
 873                 dev_err(enic, "error in allocation of wq\n");
 874                 return err;
 875         }
 876
 877         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
 878                 socket_id, enic->config.wq_desc_count,
 879                 sizeof(struct cq_enet_wq_desc));
 880         if (err) {
 881                 vnic_wq_free(wq);
 882                 dev_err(enic, "error in allocation of cq for wq\n");
 883         }
 884
 885         /* setup up CQ message */
 886         snprintf((char *)name, sizeof(name),
 887                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
 888                 instance++);
 889
 890         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
 891                         sizeof(uint32_t), SOCKET_ID_ANY,
 892                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 893         if (!wq->cqmsg_rz)
 894                 return -ENOMEM;
 895
 896         return err;
 897 }
 898
 899 int enic_disable(struct enic *enic)
 900 {
 901         unsigned int i;
 902         int err;
 903
 904         for (i = 0; i < enic->intr_count; i++) {
 905                 vnic_intr_mask(&enic->intr[i]);
 906                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
 907         }
 908         enic_rxq_intr_deinit(enic);
 909         rte_intr_disable(&enic->pdev->intr_handle);
 910         rte_intr_callback_unregister(&enic->pdev->intr_handle,
 911                                      enic_intr_handler,
 912                                      (void *)enic->rte_dev);
 913
 914         vnic_dev_disable(enic->vdev);
 915
 916         enic_clsf_destroy(enic);
 917
 918         if (!enic_is_sriov_vf(enic))
 919                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
 920
 921         for (i = 0; i < enic->wq_count; i++) {
 922                 err = vnic_wq_disable(&enic->wq[i]);
 923                 if (err)
 924                         return err;
 925         }
 926         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
 927                 if (enic->rq[i].in_use) {
 928                         err = vnic_rq_disable(&enic->rq[i]);
 929                         if (err)
 930                                 return err;
 931                 }
 932         }
 933
 934         /* If we were using interrupts, set the interrupt vector to -1
 935          * to disable interrupts.  We are not disabling link notifcations,
 936          * though, as we want the polling of link status to continue working.
 937          */
 938         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
 939                 vnic_dev_notify_set(enic->vdev, -1);
 940
 941         vnic_dev_set_reset_flag(enic->vdev, 1);
 942
 943         for (i = 0; i < enic->wq_count; i++)
 944                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
 945
 946         for (i = 0; i < enic_vnic_rq_count(enic); i++)
 947                 if (enic->rq[i].in_use)
 948                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
 949         for (i = 0; i < enic->cq_count; i++)
 950                 vnic_cq_clean(&enic->cq[i]);
 951         for (i = 0; i < enic->intr_count; i++)
 952                 vnic_intr_clean(&enic->intr[i]);
 953
 954         return 0;
 955 }
 956
 957 static int enic_dev_wait(struct vnic_dev *vdev,
 958         int (*start)(struct vnic_dev *, int),
 959         int (*finished)(struct vnic_dev *, int *),
 960         int arg)
 961 {
 962         int done;
 963         int err;
 964         int i;
 965
 966         err = start(vdev, arg);
 967         if (err)
 968                 return err;
 969
 970         /* Wait for func to complete...2 seconds max */
 971         for (i = 0; i < 2000; i++) {
 972                 err = finished(vdev, &done);
 973                 if (err)
 974                         return err;
 975                 if (done)
 976                         return 0;
 977                 usleep(1000);
 978         }
 979         return -ETIMEDOUT;
 980 }
 981
 982 static int enic_dev_open(struct enic *enic)
 983 {
 984         int err;
 985
 986         err = enic_dev_wait(enic->vdev, vnic_dev_open,
 987                 vnic_dev_open_done, 0);
 988         if (err)
 989                 dev_err(enic_get_dev(enic),
 990                         "vNIC device open failed, err %d\n", err);
 991
 992         return err;
 993 }
 994
 995 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
 996 {
 997         dma_addr_t rss_key_buf_pa;
 998         union vnic_rss_key *rss_key_buf_va = NULL;
 999         int err, i;
1000         u8 name[NAME_MAX];
1001
1002         RTE_ASSERT(user_key != NULL);
1003         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1004         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1005                 &rss_key_buf_pa, name);
1006         if (!rss_key_buf_va)
1007                 return -ENOMEM;
1008
1009         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1010                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1011
1012         err = enic_set_rss_key(enic,
1013                 rss_key_buf_pa,
1014                 sizeof(union vnic_rss_key));
1015
1016         /* Save for later queries */
1017         if (!err) {
1018                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1019                            sizeof(union vnic_rss_key));
1020         }
1021         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1022                 rss_key_buf_va, rss_key_buf_pa);
1023
1024         return err;
1025 }
1026
1027 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1028 {
1029         dma_addr_t rss_cpu_buf_pa;
1030         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1031         int err;
1032         u8 name[NAME_MAX];
1033
1034         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1035         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1036                 &rss_cpu_buf_pa, name);
1037         if (!rss_cpu_buf_va)
1038                 return -ENOMEM;
1039
1040         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1041
1042         err = enic_set_rss_cpu(enic,
1043                 rss_cpu_buf_pa,
1044                 sizeof(union vnic_rss_cpu));
1045
1046         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1047                 rss_cpu_buf_va, rss_cpu_buf_pa);
1048
1049         /* Save for later queries */
1050         if (!err)
1051                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1052         return err;
1053 }
1054
1055 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1056         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1057 {
1058         const u8 tso_ipid_split_en = 0;
1059         int err;
1060
1061         err = enic_set_nic_cfg(enic,
1062                 rss_default_cpu, rss_hash_type,
1063                 rss_hash_bits, rss_base_cpu,
1064                 rss_enable, tso_ipid_split_en,
1065                 enic->ig_vlan_strip_en);
1066
1067         return err;
1068 }
1069
1070 /* Initialize RSS with defaults, called from dev_configure */
1071 int enic_init_rss_nic_cfg(struct enic *enic)
1072 {
1073         static uint8_t default_rss_key[] = {
1074                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1075                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1076                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1077                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1078         };
1079         struct rte_eth_rss_conf rss_conf;
1080         union vnic_rss_cpu rss_cpu;
1081         int ret, i;
1082
1083         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1084         /*
1085          * If setting key for the first time, and the user gives us none, then
1086          * push the default key to NIC.
1087          */
1088         if (rss_conf.rss_key == NULL) {
1089                 rss_conf.rss_key = default_rss_key;
1090                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1091         }
1092         ret = enic_set_rss_conf(enic, &rss_conf);
1093         if (ret) {
1094                 dev_err(enic, "Failed to configure RSS\n");
1095                 return ret;
1096         }
1097         if (enic->rss_enable) {
1098                 /* If enabling RSS, use the default reta */
1099                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1100                         rss_cpu.cpu[i / 4].b[i % 4] =
1101                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1102                 }
1103                 ret = enic_set_rss_reta(enic, &rss_cpu);
1104                 if (ret)
1105                         dev_err(enic, "Failed to set RSS indirection table\n");
1106         }
1107         return ret;
1108 }
1109
1110 int enic_setup_finish(struct enic *enic)
1111 {
1112         enic_init_soft_stats(enic);
1113
1114         /* Default conf */
1115         vnic_dev_packet_filter(enic->vdev,
1116                 1 /* directed  */,
1117                 1 /* multicast */,
1118                 1 /* broadcast */,
1119                 0 /* promisc   */,
1120                 1 /* allmulti  */);
1121
1122         enic->promisc = 0;
1123         enic->allmulti = 1;
1124
1125         return 0;
1126 }
1127
1128 static int enic_rss_conf_valid(struct enic *enic,
1129                                struct rte_eth_rss_conf *rss_conf)
1130 {
1131         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1132         if (enic->flow_type_rss_offloads == 0)
1133                 return 0;
1134         if (rss_conf->rss_key != NULL &&
1135             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1136                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1137                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1138                 return -EINVAL;
1139         }
1140         if (rss_conf->rss_hf != 0 &&
1141             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1142                 dev_err(enic, "Given rss_hf contains none of the supported"
1143                         " types\n");
1144                 return -EINVAL;
1145         }
1146         return 0;
1147 }
1148
1149 /* Set hash type and key according to rss_conf */
1150 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1151 {
1152         struct rte_eth_dev *eth_dev;
1153         uint64_t rss_hf;
1154         u8 rss_hash_type;
1155         u8 rss_enable;
1156         int ret;
1157
1158         RTE_ASSERT(rss_conf != NULL);
1159         ret = enic_rss_conf_valid(enic, rss_conf);
1160         if (ret) {
1161                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1162                 return ret;
1163         }
1164
1165         eth_dev = enic->rte_dev;
1166         rss_hash_type = 0;
1167         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1168         if (enic->rq_count > 1 &&
1169             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1170             rss_hf != 0) {
1171                 rss_enable = 1;
1172                 if (rss_hf & ETH_RSS_IPV4)
1173                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1174                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1175                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1176                 if (rss_hf & ETH_RSS_IPV6)
1177                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1178                 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1179                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1180                 if (rss_hf & ETH_RSS_IPV6_EX)
1181                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6_EX;
1182                 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
1183                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX;
1184         } else {
1185                 rss_enable = 0;
1186                 rss_hf = 0;
1187         }
1188
1189         /* Set the hash key if provided */
1190         if (rss_enable && rss_conf->rss_key) {
1191                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1192                 if (ret) {
1193                         dev_err(enic, "Failed to set RSS key\n");
1194                         return ret;
1195                 }
1196         }
1197
1198         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1199                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1200                               rss_enable);
1201         if (!ret) {
1202                 enic->rss_hf = rss_hf;
1203                 enic->rss_hash_type = rss_hash_type;
1204                 enic->rss_enable = rss_enable;
1205         }
1206         return 0;
1207 }
1208
1209 int enic_set_vlan_strip(struct enic *enic)
1210 {
1211         /*
1212          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1213          * together. So, re-do niccfg, preserving the current RSS settings.
1214          */
1215         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1216                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1217                                enic->rss_enable);
1218 }
1219
1220 void enic_add_packet_filter(struct enic *enic)
1221 {
1222         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1223         vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1224                 enic->promisc, enic->allmulti);
1225 }
1226
1227 int enic_get_link_status(struct enic *enic)
1228 {
1229         return vnic_dev_link_status(enic->vdev);
1230 }
1231
1232 static void enic_dev_deinit(struct enic *enic)
1233 {
1234         struct rte_eth_dev *eth_dev = enic->rte_dev;
1235
1236         /* stop link status checking */
1237         vnic_dev_notify_unset(enic->vdev);
1238
1239         rte_free(eth_dev->data->mac_addrs);
1240         rte_free(enic->cq);
1241         rte_free(enic->intr);
1242         rte_free(enic->rq);
1243         rte_free(enic->wq);
1244 }
1245
1246
1247 int enic_set_vnic_res(struct enic *enic)
1248 {
1249         struct rte_eth_dev *eth_dev = enic->rte_dev;
1250         int rc = 0;
1251         unsigned int required_rq, required_wq, required_cq, required_intr;
1252
1253         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1254         required_rq = eth_dev->data->nb_rx_queues * 2;
1255         required_wq = eth_dev->data->nb_tx_queues;
1256         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1257         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1258         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1259                 required_intr += eth_dev->data->nb_rx_queues;
1260         }
1261
1262         if (enic->conf_rq_count < required_rq) {
1263                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1264                         eth_dev->data->nb_rx_queues,
1265                         required_rq, enic->conf_rq_count);
1266                 rc = -EINVAL;
1267         }
1268         if (enic->conf_wq_count < required_wq) {
1269                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1270                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1271                 rc = -EINVAL;
1272         }
1273
1274         if (enic->conf_cq_count < required_cq) {
1275                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1276                         required_cq, enic->conf_cq_count);
1277                 rc = -EINVAL;
1278         }
1279         if (enic->conf_intr_count < required_intr) {
1280                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1281                         " interrupts. Required:%u, Configured:%u\n",
1282                         required_intr, enic->conf_intr_count);
1283                 rc = -EINVAL;
1284         }
1285
1286         if (rc == 0) {
1287                 enic->rq_count = eth_dev->data->nb_rx_queues;
1288                 enic->wq_count = eth_dev->data->nb_tx_queues;
1289                 enic->cq_count = enic->rq_count + enic->wq_count;
1290                 enic->intr_count = required_intr;
1291         }
1292
1293         return rc;
1294 }
1295
1296 /* Initialize the completion queue for an RQ */
1297 static int
1298 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1299 {
1300         struct vnic_rq *sop_rq, *data_rq;
1301         unsigned int cq_idx;
1302         int rc = 0;
1303
1304         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1305         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1306         cq_idx = rq_idx;
1307
1308         vnic_cq_clean(&enic->cq[cq_idx]);
1309         vnic_cq_init(&enic->cq[cq_idx],
1310                      0 /* flow_control_enable */,
1311                      1 /* color_enable */,
1312                      0 /* cq_head */,
1313                      0 /* cq_tail */,
1314                      1 /* cq_tail_color */,
1315                      0 /* interrupt_enable */,
1316                      1 /* cq_entry_enable */,
1317                      0 /* cq_message_enable */,
1318                      0 /* interrupt offset */,
1319                      0 /* cq_message_addr */);
1320
1321
1322         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1323                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1324                            sop_rq->ring.desc_count - 1, 1, 0);
1325         if (data_rq->in_use) {
1326                 vnic_rq_init_start(data_rq,
1327                                    enic_cq_rq(enic,
1328                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1329                                    data_rq->ring.desc_count - 1, 1, 0);
1330         }
1331
1332         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1333         if (rc)
1334                 return rc;
1335
1336         if (data_rq->in_use) {
1337                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1338                 if (rc) {
1339                         enic_rxmbuf_queue_release(enic, sop_rq);
1340                         return rc;
1341                 }
1342         }
1343
1344         return 0;
1345 }
1346
1347 /* The Cisco NIC can send and receive packets up to a max packet size
1348  * determined by the NIC type and firmware. There is also an MTU
1349  * configured into the NIC via the CIMC/UCSM management interface
1350  * which can be overridden by this function (up to the max packet size).
1351  * Depending on the network setup, doing so may cause packet drops
1352  * and unexpected behavior.
1353  */
1354 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1355 {
1356         unsigned int rq_idx;
1357         struct vnic_rq *rq;
1358         int rc = 0;
1359         uint16_t old_mtu;       /* previous setting */
1360         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1361         struct rte_eth_dev *eth_dev = enic->rte_dev;
1362
1363         old_mtu = eth_dev->data->mtu;
1364         config_mtu = enic->config.mtu;
1365
1366         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1367                 return -E_RTE_SECONDARY;
1368
1369         if (new_mtu > enic->max_mtu) {
1370                 dev_err(enic,
1371                         "MTU not updated: requested (%u) greater than max (%u)\n",
1372                         new_mtu, enic->max_mtu);
1373                 return -EINVAL;
1374         }
1375         if (new_mtu < ENIC_MIN_MTU) {
1376                 dev_info(enic,
1377                         "MTU not updated: requested (%u) less than min (%u)\n",
1378                         new_mtu, ENIC_MIN_MTU);
1379                 return -EINVAL;
1380         }
1381         if (new_mtu > config_mtu)
1382                 dev_warning(enic,
1383                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1384                         new_mtu, config_mtu);
1385
1386         /* The easy case is when scatter is disabled. However if the MTU
1387          * becomes greater than the mbuf data size, packet drops will ensue.
1388          */
1389         if (!(enic->rte_dev->data->dev_conf.rxmode.offloads &
1390               DEV_RX_OFFLOAD_SCATTER)) {
1391                 eth_dev->data->mtu = new_mtu;
1392                 goto set_mtu_done;
1393         }
1394
1395         /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to
1396          * change Rx scatter mode if necessary for better performance. I.e. if
1397          * MTU was greater than the mbuf size and now it's less, scatter Rx
1398          * doesn't have to be used and vice versa.
1399           */
1400         rte_spinlock_lock(&enic->mtu_lock);
1401
1402         /* Stop traffic on all RQs */
1403         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1404                 rq = &enic->rq[rq_idx];
1405                 if (rq->is_sop && rq->in_use) {
1406                         rc = enic_stop_rq(enic,
1407                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1408                         if (rc) {
1409                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1410                                 goto set_mtu_done;
1411                         }
1412                 }
1413         }
1414
1415         /* replace Rx function with a no-op to avoid getting stale pkts */
1416         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1417         rte_mb();
1418
1419         /* Allow time for threads to exit the real Rx function. */
1420         usleep(100000);
1421
1422         /* now it is safe to reconfigure the RQs */
1423
1424         /* update the mtu */
1425         eth_dev->data->mtu = new_mtu;
1426
1427         /* free and reallocate RQs with the new MTU */
1428         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1429                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1430
1431                 enic_free_rq(rq);
1432                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1433                                    rq->tot_nb_desc, rq->rx_free_thresh);
1434                 if (rc) {
1435                         dev_err(enic,
1436                                 "Fatal MTU alloc error- No traffic will pass\n");
1437                         goto set_mtu_done;
1438                 }
1439
1440                 rc = enic_reinit_rq(enic, rq_idx);
1441                 if (rc) {
1442                         dev_err(enic,
1443                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1444                         goto set_mtu_done;
1445                 }
1446         }
1447
1448         /* put back the real receive function */
1449         rte_mb();
1450         eth_dev->rx_pkt_burst = enic_recv_pkts;
1451         rte_mb();
1452
1453         /* restart Rx traffic */
1454         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1455                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1456                 if (rq->is_sop && rq->in_use)
1457                         enic_start_rq(enic, rq_idx);
1458         }
1459
1460 set_mtu_done:
1461         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1462         rte_spinlock_unlock(&enic->mtu_lock);
1463         return rc;
1464 }
1465
1466 static int enic_dev_init(struct enic *enic)
1467 {
1468         int err;
1469         struct rte_eth_dev *eth_dev = enic->rte_dev;
1470
1471         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1472
1473         /* Get vNIC configuration
1474         */
1475         err = enic_get_vnic_config(enic);
1476         if (err) {
1477                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1478                 return err;
1479         }
1480
1481         /* Get available resource counts */
1482         enic_get_res_counts(enic);
1483         if (enic->conf_rq_count == 1) {
1484                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1485                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1486                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1487                 return -EINVAL;
1488         }
1489         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1490         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1491                                enic->conf_cq_count, 8);
1492         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1493                                  enic->conf_intr_count, 8);
1494         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1495                                enic->conf_rq_count, 8);
1496         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1497                                enic->conf_wq_count, 8);
1498         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1499                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1500                 return -1;
1501         }
1502         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1503                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1504                 return -1;
1505         }
1506         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1507                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1508                 return -1;
1509         }
1510         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1511                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1512                 return -1;
1513         }
1514
1515         /* Get the supported filters */
1516         enic_fdir_info(enic);
1517
1518         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1519                                                 * ENIC_MAX_MAC_ADDR, 0);
1520         if (!eth_dev->data->mac_addrs) {
1521                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1522                 return -1;
1523         }
1524         ether_addr_copy((struct ether_addr *) enic->mac_addr,
1525                         eth_dev->data->mac_addrs);
1526
1527         vnic_dev_set_reset_flag(enic->vdev, 0);
1528
1529         LIST_INIT(&enic->flows);
1530         rte_spinlock_init(&enic->flows_lock);
1531
1532         /* set up link status checking */
1533         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1534
1535         return 0;
1536
1537 }
1538
1539 int enic_probe(struct enic *enic)
1540 {
1541         struct rte_pci_device *pdev = enic->pdev;
1542         int err = -1;
1543
1544         dev_debug(enic, " Initializing ENIC PMD\n");
1545
1546         /* if this is a secondary process the hardware is already initialized */
1547         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1548                 return 0;
1549
1550         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1551         enic->bar0.len = pdev->mem_resource[0].len;
1552
1553         /* Register vNIC device */
1554         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1555         if (!enic->vdev) {
1556                 dev_err(enic, "vNIC registration failed, aborting\n");
1557                 goto err_out;
1558         }
1559
1560         LIST_INIT(&enic->memzone_list);
1561         rte_spinlock_init(&enic->memzone_list_lock);
1562
1563         vnic_register_cbacks(enic->vdev,
1564                 enic_alloc_consistent,
1565                 enic_free_consistent);
1566
1567         /*
1568          * Allocate the consistent memory for stats upfront so both primary and
1569          * secondary processes can dump stats.
1570          */
1571         err = vnic_dev_alloc_stats_mem(enic->vdev);
1572         if (err) {
1573                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1574                 goto err_out_unregister;
1575         }
1576         /* Issue device open to get device in known state */
1577         err = enic_dev_open(enic);
1578         if (err) {
1579                 dev_err(enic, "vNIC dev open failed, aborting\n");
1580                 goto err_out_unregister;
1581         }
1582
1583         /* Set ingress vlan rewrite mode before vnic initialization */
1584         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1585                 IG_VLAN_REWRITE_MODE_PASS_THRU);
1586         if (err) {
1587                 dev_err(enic,
1588                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1589                 goto err_out_dev_close;
1590         }
1591
1592         /* Issue device init to initialize the vnic-to-switch link.
1593          * We'll start with carrier off and wait for link UP
1594          * notification later to turn on carrier.  We don't need
1595          * to wait here for the vnic-to-switch link initialization
1596          * to complete; link UP notification is the indication that
1597          * the process is complete.
1598          */
1599
1600         err = vnic_dev_init(enic->vdev, 0);
1601         if (err) {
1602                 dev_err(enic, "vNIC dev init failed, aborting\n");
1603                 goto err_out_dev_close;
1604         }
1605
1606         err = enic_dev_init(enic);
1607         if (err) {
1608                 dev_err(enic, "Device initialization failed, aborting\n");
1609                 goto err_out_dev_close;
1610         }
1611
1612         return 0;
1613
1614 err_out_dev_close:
1615         vnic_dev_close(enic->vdev);
1616 err_out_unregister:
1617         vnic_dev_unregister(enic->vdev);
1618 err_out:
1619         return err;
1620 }
1621
1622 void enic_remove(struct enic *enic)
1623 {
1624         enic_dev_deinit(enic);
1625         vnic_dev_close(enic->vdev);
1626         vnic_dev_unregister(enic->vdev);
1627 }