drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11 #include <libgen.h>
  12
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_ethdev_driver.h>
  20
  21 #include "enic_compat.h"
  22 #include "enic.h"
  23 #include "wq_enet_desc.h"
  24 #include "rq_enet_desc.h"
  25 #include "cq_enet_desc.h"
  26 #include "vnic_enet.h"
  27 #include "vnic_dev.h"
  28 #include "vnic_wq.h"
  29 #include "vnic_rq.h"
  30 #include "vnic_cq.h"
  31 #include "vnic_intr.h"
  32 #include "vnic_nic.h"
  33
  34 static inline int enic_is_sriov_vf(struct enic *enic)
  35 {
  36         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  37 }
  38
  39 static int is_zero_addr(uint8_t *addr)
  40 {
  41         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  42 }
  43
  44 static int is_mcast_addr(uint8_t *addr)
  45 {
  46         return addr[0] & 1;
  47 }
  48
  49 static int is_eth_addr_valid(uint8_t *addr)
  50 {
  51         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  52 }
  53
  54 static void
  55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  56 {
  57         uint16_t i;
  58
  59         if (!rq || !rq->mbuf_ring) {
  60                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  61                 return;
  62         }
  63
  64         for (i = 0; i < rq->ring.desc_count; i++) {
  65                 if (rq->mbuf_ring[i]) {
  66                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  67                         rq->mbuf_ring[i] = NULL;
  68                 }
  69         }
  70 }
  71
  72 static void enic_free_wq_buf(struct vnic_wq_buf *buf)
  73 {
  74         struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
  75
  76         rte_pktmbuf_free_seg(mbuf);
  77         buf->mb = NULL;
  78 }
  79
  80 static void enic_log_q_error(struct enic *enic)
  81 {
  82         unsigned int i;
  83         u32 error_status;
  84
  85         for (i = 0; i < enic->wq_count; i++) {
  86                 error_status = vnic_wq_error_status(&enic->wq[i]);
  87                 if (error_status)
  88                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  89                                 error_status);
  90         }
  91
  92         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  93                 if (!enic->rq[i].in_use)
  94                         continue;
  95                 error_status = vnic_rq_error_status(&enic->rq[i]);
  96                 if (error_status)
  97                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  98                                 error_status);
  99         }
 100 }
 101
 102 static void enic_clear_soft_stats(struct enic *enic)
 103 {
 104         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 105         rte_atomic64_clear(&soft_stats->rx_nombuf);
 106         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 107         rte_atomic64_clear(&soft_stats->tx_oversized);
 108 }
 109
 110 static void enic_init_soft_stats(struct enic *enic)
 111 {
 112         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 113         rte_atomic64_init(&soft_stats->rx_nombuf);
 114         rte_atomic64_init(&soft_stats->rx_packet_errors);
 115         rte_atomic64_init(&soft_stats->tx_oversized);
 116         enic_clear_soft_stats(enic);
 117 }
 118
 119 void enic_dev_stats_clear(struct enic *enic)
 120 {
 121         if (vnic_dev_stats_clear(enic->vdev))
 122                 dev_err(enic, "Error in clearing stats\n");
 123         enic_clear_soft_stats(enic);
 124 }
 125
 126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 127 {
 128         struct vnic_stats *stats;
 129         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 130         int64_t rx_truncated;
 131         uint64_t rx_packet_errors;
 132         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 133
 134         if (ret) {
 135                 dev_err(enic, "Error in getting stats\n");
 136                 return ret;
 137         }
 138
 139         /* The number of truncated packets can only be calculated by
 140          * subtracting a hardware counter from error packets received by
 141          * the driver. Note: this causes transient inaccuracies in the
 142          * ipackets count. Also, the length of truncated packets are
 143          * counted in ibytes even though truncated packets are dropped
 144          * which can make ibytes be slightly higher than it should be.
 145          */
 146         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 147         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 148
 149         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 150         r_stats->opackets = stats->tx.tx_frames_ok;
 151
 152         r_stats->ibytes = stats->rx.rx_bytes_ok;
 153         r_stats->obytes = stats->tx.tx_bytes_ok;
 154
 155         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 156         r_stats->oerrors = stats->tx.tx_errors
 157                            + rte_atomic64_read(&soft_stats->tx_oversized);
 158
 159         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 160
 161         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 162         return 0;
 163 }
 164
 165 void enic_del_mac_address(struct enic *enic, int mac_index)
 166 {
 167         struct rte_eth_dev *eth_dev = enic->rte_dev;
 168         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 169
 170         if (vnic_dev_del_addr(enic->vdev, mac_addr))
 171                 dev_err(enic, "del mac addr failed\n");
 172 }
 173
 174 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 175 {
 176         int err;
 177
 178         if (!is_eth_addr_valid(mac_addr)) {
 179                 dev_err(enic, "invalid mac address\n");
 180                 return -EINVAL;
 181         }
 182
 183         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 184         if (err)
 185                 dev_err(enic, "add mac addr failed\n");
 186         return err;
 187 }
 188
 189 static void
 190 enic_free_rq_buf(struct rte_mbuf **mbuf)
 191 {
 192         if (*mbuf == NULL)
 193                 return;
 194
 195         rte_pktmbuf_free(*mbuf);
 196         *mbuf = NULL;
 197 }
 198
 199 void enic_init_vnic_resources(struct enic *enic)
 200 {
 201         unsigned int error_interrupt_enable = 1;
 202         unsigned int error_interrupt_offset = 0;
 203         unsigned int rxq_interrupt_enable = 0;
 204         unsigned int rxq_interrupt_offset;
 205         unsigned int index = 0;
 206         unsigned int cq_idx;
 207         struct vnic_rq *data_rq;
 208
 209         if (enic->rte_dev->data->dev_conf.intr_conf.rxq) {
 210                 rxq_interrupt_enable = 1;
 211                 rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 212         }
 213         for (index = 0; index < enic->rq_count; index++) {
 214                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 215
 216                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 217                         cq_idx,
 218                         error_interrupt_enable,
 219                         error_interrupt_offset);
 220
 221                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 222                 if (data_rq->in_use)
 223                         vnic_rq_init(data_rq,
 224                                      cq_idx,
 225                                      error_interrupt_enable,
 226                                      error_interrupt_offset);
 227
 228                 vnic_cq_init(&enic->cq[cq_idx],
 229                         0 /* flow_control_enable */,
 230                         1 /* color_enable */,
 231                         0 /* cq_head */,
 232                         0 /* cq_tail */,
 233                         1 /* cq_tail_color */,
 234                         rxq_interrupt_enable,
 235                         1 /* cq_entry_enable */,
 236                         0 /* cq_message_enable */,
 237                         rxq_interrupt_offset,
 238                         0 /* cq_message_addr */);
 239                 if (rxq_interrupt_enable)
 240                         rxq_interrupt_offset++;
 241         }
 242
 243         for (index = 0; index < enic->wq_count; index++) {
 244                 vnic_wq_init(&enic->wq[index],
 245                         enic_cq_wq(enic, index),
 246                         error_interrupt_enable,
 247                         error_interrupt_offset);
 248
 249                 cq_idx = enic_cq_wq(enic, index);
 250                 vnic_cq_init(&enic->cq[cq_idx],
 251                         0 /* flow_control_enable */,
 252                         1 /* color_enable */,
 253                         0 /* cq_head */,
 254                         0 /* cq_tail */,
 255                         1 /* cq_tail_color */,
 256                         0 /* interrupt_enable */,
 257                         0 /* cq_entry_enable */,
 258                         1 /* cq_message_enable */,
 259                         0 /* interrupt offset */,
 260                         (u64)enic->wq[index].cqmsg_rz->iova);
 261         }
 262
 263         for (index = 0; index < enic->intr_count; index++) {
 264                 vnic_intr_init(&enic->intr[index],
 265                                enic->config.intr_timer_usec,
 266                                enic->config.intr_timer_type,
 267                                /*mask_on_assertion*/1);
 268         }
 269 }
 270
 271
 272 static int
 273 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 274 {
 275         struct rte_mbuf *mb;
 276         struct rq_enet_desc *rqd = rq->ring.descs;
 277         unsigned i;
 278         dma_addr_t dma_addr;
 279         uint32_t max_rx_pkt_len;
 280         uint16_t rq_buf_len;
 281
 282         if (!rq->in_use)
 283                 return 0;
 284
 285         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 286                   rq->ring.desc_count);
 287
 288         /*
 289          * If *not* using scatter and the mbuf size is smaller than the
 290          * requested max packet size (max_rx_pkt_len), then reduce the
 291          * posted buffer size to max_rx_pkt_len. HW still receives packets
 292          * larger than max_rx_pkt_len, but they will be truncated, which we
 293          * drop in the rx handler. Not ideal, but better than returning
 294          * large packets when the user is not expecting them.
 295          */
 296         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 297         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 298         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 299                 rq_buf_len = max_rx_pkt_len;
 300         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 301                 mb = rte_mbuf_raw_alloc(rq->mp);
 302                 if (mb == NULL) {
 303                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 304                         (unsigned)rq->index);
 305                         return -ENOMEM;
 306                 }
 307
 308                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 309                 dma_addr = (dma_addr_t)(mb->buf_iova
 310                            + RTE_PKTMBUF_HEADROOM);
 311                 rq_enet_desc_enc(rqd, dma_addr,
 312                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 313                                 : RQ_ENET_TYPE_NOT_SOP),
 314                                 rq_buf_len);
 315                 rq->mbuf_ring[i] = mb;
 316         }
 317
 318         /* make sure all prior writes are complete before doing the PIO write */
 319         rte_rmb();
 320
 321         /* Post all but the last buffer to VIC. */
 322         rq->posted_index = rq->ring.desc_count - 1;
 323
 324         rq->rx_nb_hold = 0;
 325
 326         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 327                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 328         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 329         iowrite32(0, &rq->ctrl->fetch_index);
 330         rte_rmb();
 331
 332         return 0;
 333
 334 }
 335
 336 static void *
 337 enic_alloc_consistent(void *priv, size_t size,
 338         dma_addr_t *dma_handle, u8 *name)
 339 {
 340         void *vaddr;
 341         const struct rte_memzone *rz;
 342         *dma_handle = 0;
 343         struct enic *enic = (struct enic *)priv;
 344         struct enic_memzone_entry *mze;
 345
 346         rz = rte_memzone_reserve_aligned((const char *)name, size,
 347                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 348         if (!rz) {
 349                 pr_err("%s : Failed to allocate memory requested for %s\n",
 350                         __func__, name);
 351                 return NULL;
 352         }
 353
 354         vaddr = rz->addr;
 355         *dma_handle = (dma_addr_t)rz->iova;
 356
 357         mze = rte_malloc("enic memzone entry",
 358                          sizeof(struct enic_memzone_entry), 0);
 359
 360         if (!mze) {
 361                 pr_err("%s : Failed to allocate memory for memzone list\n",
 362                        __func__);
 363                 rte_memzone_free(rz);
 364                 return NULL;
 365         }
 366
 367         mze->rz = rz;
 368
 369         rte_spinlock_lock(&enic->memzone_list_lock);
 370         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 371         rte_spinlock_unlock(&enic->memzone_list_lock);
 372
 373         return vaddr;
 374 }
 375
 376 static void
 377 enic_free_consistent(void *priv,
 378                      __rte_unused size_t size,
 379                      void *vaddr,
 380                      dma_addr_t dma_handle)
 381 {
 382         struct enic_memzone_entry *mze;
 383         struct enic *enic = (struct enic *)priv;
 384
 385         rte_spinlock_lock(&enic->memzone_list_lock);
 386         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 387                 if (mze->rz->addr == vaddr &&
 388                     mze->rz->iova == dma_handle)
 389                         break;
 390         }
 391         if (mze == NULL) {
 392                 rte_spinlock_unlock(&enic->memzone_list_lock);
 393                 dev_warning(enic,
 394                             "Tried to free memory, but couldn't find it in the memzone list\n");
 395                 return;
 396         }
 397         LIST_REMOVE(mze, entries);
 398         rte_spinlock_unlock(&enic->memzone_list_lock);
 399         rte_memzone_free(mze->rz);
 400         rte_free(mze);
 401 }
 402
 403 int enic_link_update(struct enic *enic)
 404 {
 405         struct rte_eth_dev *eth_dev = enic->rte_dev;
 406         struct rte_eth_link link;
 407
 408         memset(&link, 0, sizeof(link));
 409         link.link_status = enic_get_link_status(enic);
 410         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 411         link.link_speed = vnic_dev_port_speed(enic->vdev);
 412
 413         return rte_eth_linkstatus_set(eth_dev, &link);
 414 }
 415
 416 static void
 417 enic_intr_handler(void *arg)
 418 {
 419         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 420         struct enic *enic = pmd_priv(dev);
 421
 422         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 423
 424         enic_link_update(enic);
 425         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 426         enic_log_q_error(enic);
 427 }
 428
 429 static int enic_rxq_intr_init(struct enic *enic)
 430 {
 431         struct rte_intr_handle *intr_handle;
 432         uint32_t rxq_intr_count, i;
 433         int err;
 434
 435         intr_handle = enic->rte_dev->intr_handle;
 436         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 437                 return 0;
 438         /*
 439          * Rx queue interrupts only work when we have MSI-X interrupts,
 440          * one per queue. Sharing one interrupt is technically
 441          * possible with VIC, but it is not worth the complications it brings.
 442          */
 443         if (!rte_intr_cap_multiple(intr_handle)) {
 444                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 445                         " (vfio-pci driver)\n");
 446                 return -ENOTSUP;
 447         }
 448         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 449         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 450         if (err) {
 451                 dev_err(enic, "Failed to enable event fds for Rx queue"
 452                         " interrupts\n");
 453                 return err;
 454         }
 455         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 456                                             rxq_intr_count * sizeof(int), 0);
 457         if (intr_handle->intr_vec == NULL) {
 458                 dev_err(enic, "Failed to allocate intr_vec\n");
 459                 return -ENOMEM;
 460         }
 461         for (i = 0; i < rxq_intr_count; i++)
 462                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 463         return 0;
 464 }
 465
 466 static void enic_rxq_intr_deinit(struct enic *enic)
 467 {
 468         struct rte_intr_handle *intr_handle;
 469
 470         intr_handle = enic->rte_dev->intr_handle;
 471         rte_intr_efd_disable(intr_handle);
 472         if (intr_handle->intr_vec != NULL) {
 473                 rte_free(intr_handle->intr_vec);
 474                 intr_handle->intr_vec = NULL;
 475         }
 476 }
 477
 478 int enic_enable(struct enic *enic)
 479 {
 480         unsigned int index;
 481         int err;
 482         struct rte_eth_dev *eth_dev = enic->rte_dev;
 483
 484         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 485         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 486
 487         /* vnic notification of link status has already been turned on in
 488          * enic_dev_init() which is called during probe time.  Here we are
 489          * just turning on interrupt vector 0 if needed.
 490          */
 491         if (eth_dev->data->dev_conf.intr_conf.lsc)
 492                 vnic_dev_notify_set(enic->vdev, 0);
 493
 494         err = enic_rxq_intr_init(enic);
 495         if (err)
 496                 return err;
 497         if (enic_clsf_init(enic))
 498                 dev_warning(enic, "Init of hash table for clsf failed."\
 499                         "Flow director feature will not work\n");
 500
 501         for (index = 0; index < enic->rq_count; index++) {
 502                 err = enic_alloc_rx_queue_mbufs(enic,
 503                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 504                 if (err) {
 505                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 506                         return err;
 507                 }
 508                 err = enic_alloc_rx_queue_mbufs(enic,
 509                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 510                 if (err) {
 511                         /* release the allocated mbufs for the sop rq*/
 512                         enic_rxmbuf_queue_release(enic,
 513                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 514
 515                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 516                         return err;
 517                 }
 518         }
 519
 520         for (index = 0; index < enic->wq_count; index++)
 521                 enic_start_wq(enic, index);
 522         for (index = 0; index < enic->rq_count; index++)
 523                 enic_start_rq(enic, index);
 524
 525         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 526
 527         vnic_dev_enable_wait(enic->vdev);
 528
 529         /* Register and enable error interrupt */
 530         rte_intr_callback_register(&(enic->pdev->intr_handle),
 531                 enic_intr_handler, (void *)enic->rte_dev);
 532
 533         rte_intr_enable(&(enic->pdev->intr_handle));
 534         /* Unmask LSC interrupt */
 535         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 536
 537         return 0;
 538 }
 539
 540 int enic_alloc_intr_resources(struct enic *enic)
 541 {
 542         int err;
 543         unsigned int i;
 544
 545         dev_info(enic, "vNIC resources used:  "\
 546                 "wq %d rq %d cq %d intr %d\n",
 547                 enic->wq_count, enic_vnic_rq_count(enic),
 548                 enic->cq_count, enic->intr_count);
 549
 550         for (i = 0; i < enic->intr_count; i++) {
 551                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 552                 if (err) {
 553                         enic_free_vnic_resources(enic);
 554                         return err;
 555                 }
 556         }
 557         return 0;
 558 }
 559
 560 void enic_free_rq(void *rxq)
 561 {
 562         struct vnic_rq *rq_sop, *rq_data;
 563         struct enic *enic;
 564
 565         if (rxq == NULL)
 566                 return;
 567
 568         rq_sop = (struct vnic_rq *)rxq;
 569         enic = vnic_dev_priv(rq_sop->vdev);
 570         rq_data = &enic->rq[rq_sop->data_queue_idx];
 571
 572         enic_rxmbuf_queue_release(enic, rq_sop);
 573         if (rq_data->in_use)
 574                 enic_rxmbuf_queue_release(enic, rq_data);
 575
 576         rte_free(rq_sop->mbuf_ring);
 577         if (rq_data->in_use)
 578                 rte_free(rq_data->mbuf_ring);
 579
 580         rq_sop->mbuf_ring = NULL;
 581         rq_data->mbuf_ring = NULL;
 582
 583         vnic_rq_free(rq_sop);
 584         if (rq_data->in_use)
 585                 vnic_rq_free(rq_data);
 586
 587         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 588
 589         rq_sop->in_use = 0;
 590         rq_data->in_use = 0;
 591 }
 592
 593 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 594 {
 595         struct rte_eth_dev *eth_dev = enic->rte_dev;
 596         vnic_wq_enable(&enic->wq[queue_idx]);
 597         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 598 }
 599
 600 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 601 {
 602         struct rte_eth_dev *eth_dev = enic->rte_dev;
 603         int ret;
 604
 605         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 606         if (ret)
 607                 return ret;
 608
 609         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 610         return 0;
 611 }
 612
 613 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 614 {
 615         struct vnic_rq *rq_sop;
 616         struct vnic_rq *rq_data;
 617         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 618         rq_data = &enic->rq[rq_sop->data_queue_idx];
 619         struct rte_eth_dev *eth_dev = enic->rte_dev;
 620
 621         if (rq_data->in_use)
 622                 vnic_rq_enable(rq_data);
 623         rte_mb();
 624         vnic_rq_enable(rq_sop);
 625         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 626 }
 627
 628 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 629 {
 630         int ret1 = 0, ret2 = 0;
 631         struct rte_eth_dev *eth_dev = enic->rte_dev;
 632         struct vnic_rq *rq_sop;
 633         struct vnic_rq *rq_data;
 634         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 635         rq_data = &enic->rq[rq_sop->data_queue_idx];
 636
 637         ret2 = vnic_rq_disable(rq_sop);
 638         rte_mb();
 639         if (rq_data->in_use)
 640                 ret1 = vnic_rq_disable(rq_data);
 641
 642         if (ret2)
 643                 return ret2;
 644         else if (ret1)
 645                 return ret1;
 646
 647         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 648         return 0;
 649 }
 650
 651 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 652         unsigned int socket_id, struct rte_mempool *mp,
 653         uint16_t nb_desc, uint16_t free_thresh)
 654 {
 655         int rc;
 656         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 657         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 658         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 659         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 660         unsigned int mbuf_size, mbufs_per_pkt;
 661         unsigned int nb_sop_desc, nb_data_desc;
 662         uint16_t min_sop, max_sop, min_data, max_data;
 663         uint32_t max_rx_pkt_len;
 664
 665         rq_sop->is_sop = 1;
 666         rq_sop->data_queue_idx = data_queue_idx;
 667         rq_data->is_sop = 0;
 668         rq_data->data_queue_idx = 0;
 669         rq_sop->socket_id = socket_id;
 670         rq_sop->mp = mp;
 671         rq_data->socket_id = socket_id;
 672         rq_data->mp = mp;
 673         rq_sop->in_use = 1;
 674         rq_sop->rx_free_thresh = free_thresh;
 675         rq_data->rx_free_thresh = free_thresh;
 676         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 677                   free_thresh);
 678
 679         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 680                                RTE_PKTMBUF_HEADROOM);
 681         /* max_rx_pkt_len includes the ethernet header and CRC. */
 682         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 683
 684         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 685             DEV_RX_OFFLOAD_SCATTER) {
 686                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 687                 /* ceil((max pkt len)/mbuf_size) */
 688                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 689         } else {
 690                 dev_info(enic, "Scatter rx mode disabled\n");
 691                 mbufs_per_pkt = 1;
 692                 if (max_rx_pkt_len > mbuf_size) {
 693                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 694                                     " larger than the mbuf size (%u), and"
 695                                     " scatter is disabled. Larger packets will"
 696                                     " be truncated.\n",
 697                                     max_rx_pkt_len, mbuf_size);
 698                 }
 699         }
 700
 701         if (mbufs_per_pkt > 1) {
 702                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 703                 rq_sop->data_queue_enable = 1;
 704                 rq_data->in_use = 1;
 705                 /*
 706                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 707                  * receives packet sizes up to the "max" MTU.
 708                  * If not using scatter, we can achieve the effect of dropping
 709                  * larger packets by reducing the size of posted buffers.
 710                  * See enic_alloc_rx_queue_mbufs().
 711                  */
 712                 if (max_rx_pkt_len <
 713                     enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu)) {
 714                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 715                                     " when scatter rx mode is in use.\n");
 716                 }
 717         } else {
 718                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 719                          queue_idx);
 720                 rq_sop->data_queue_enable = 0;
 721                 rq_data->in_use = 0;
 722         }
 723
 724         /* number of descriptors have to be a multiple of 32 */
 725         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
 726         nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
 727
 728         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 729         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 730
 731         if (mbufs_per_pkt > 1) {
 732                 min_sop = 64;
 733                 max_sop = ((enic->config.rq_desc_count /
 734                             (mbufs_per_pkt - 1)) & ~0x1F);
 735                 min_data = min_sop * (mbufs_per_pkt - 1);
 736                 max_data = enic->config.rq_desc_count;
 737         } else {
 738                 min_sop = 64;
 739                 max_sop = enic->config.rq_desc_count;
 740                 min_data = 0;
 741                 max_data = 0;
 742         }
 743
 744         if (nb_desc < (min_sop + min_data)) {
 745                 dev_warning(enic,
 746                             "Number of rx descs too low, adjusting to minimum\n");
 747                 nb_sop_desc = min_sop;
 748                 nb_data_desc = min_data;
 749         } else if (nb_desc > (max_sop + max_data)) {
 750                 dev_warning(enic,
 751                             "Number of rx_descs too high, adjusting to maximum\n");
 752                 nb_sop_desc = max_sop;
 753                 nb_data_desc = max_data;
 754         }
 755         if (mbufs_per_pkt > 1) {
 756                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 757                          " rx descriptor range is %u to %u\n",
 758                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 759                          max_sop + max_data);
 760         }
 761         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 762                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 763
 764         /* Allocate sop queue resources */
 765         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 766                 nb_sop_desc, sizeof(struct rq_enet_desc));
 767         if (rc) {
 768                 dev_err(enic, "error in allocation of sop rq\n");
 769                 goto err_exit;
 770         }
 771         nb_sop_desc = rq_sop->ring.desc_count;
 772
 773         if (rq_data->in_use) {
 774                 /* Allocate data queue resources */
 775                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 776                                    nb_data_desc,
 777                                    sizeof(struct rq_enet_desc));
 778                 if (rc) {
 779                         dev_err(enic, "error in allocation of data rq\n");
 780                         goto err_free_rq_sop;
 781                 }
 782                 nb_data_desc = rq_data->ring.desc_count;
 783         }
 784         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 785                            socket_id, nb_sop_desc + nb_data_desc,
 786                            sizeof(struct cq_enet_rq_desc));
 787         if (rc) {
 788                 dev_err(enic, "error in allocation of cq for rq\n");
 789                 goto err_free_rq_data;
 790         }
 791
 792         /* Allocate the mbuf rings */
 793         rq_sop->mbuf_ring = (struct rte_mbuf **)
 794                 rte_zmalloc_socket("rq->mbuf_ring",
 795                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 796                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 797         if (rq_sop->mbuf_ring == NULL)
 798                 goto err_free_cq;
 799
 800         if (rq_data->in_use) {
 801                 rq_data->mbuf_ring = (struct rte_mbuf **)
 802                         rte_zmalloc_socket("rq->mbuf_ring",
 803                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 804                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 805                 if (rq_data->mbuf_ring == NULL)
 806                         goto err_free_sop_mbuf;
 807         }
 808
 809         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 810
 811         return 0;
 812
 813 err_free_sop_mbuf:
 814         rte_free(rq_sop->mbuf_ring);
 815 err_free_cq:
 816         /* cleanup on error */
 817         vnic_cq_free(&enic->cq[queue_idx]);
 818 err_free_rq_data:
 819         if (rq_data->in_use)
 820                 vnic_rq_free(rq_data);
 821 err_free_rq_sop:
 822         vnic_rq_free(rq_sop);
 823 err_exit:
 824         return -ENOMEM;
 825 }
 826
 827 void enic_free_wq(void *txq)
 828 {
 829         struct vnic_wq *wq;
 830         struct enic *enic;
 831
 832         if (txq == NULL)
 833                 return;
 834
 835         wq = (struct vnic_wq *)txq;
 836         enic = vnic_dev_priv(wq->vdev);
 837         rte_memzone_free(wq->cqmsg_rz);
 838         vnic_wq_free(wq);
 839         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 840 }
 841
 842 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 843         unsigned int socket_id, uint16_t nb_desc)
 844 {
 845         int err;
 846         struct vnic_wq *wq = &enic->wq[queue_idx];
 847         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 848         char name[NAME_MAX];
 849         static int instance;
 850
 851         wq->socket_id = socket_id;
 852         if (nb_desc) {
 853                 if (nb_desc > enic->config.wq_desc_count) {
 854                         dev_warning(enic,
 855                                 "WQ %d - number of tx desc in cmd line (%d)"\
 856                                 "is greater than that in the UCSM/CIMC adapter"\
 857                                 "policy.  Applying the value in the adapter "\
 858                                 "policy (%d)\n",
 859                                 queue_idx, nb_desc, enic->config.wq_desc_count);
 860                 } else if (nb_desc != enic->config.wq_desc_count) {
 861                         enic->config.wq_desc_count = nb_desc;
 862                         dev_info(enic,
 863                                 "TX Queues - effective number of descs:%d\n",
 864                                 nb_desc);
 865                 }
 866         }
 867
 868         /* Allocate queue resources */
 869         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
 870                 enic->config.wq_desc_count,
 871                 sizeof(struct wq_enet_desc));
 872         if (err) {
 873                 dev_err(enic, "error in allocation of wq\n");
 874                 return err;
 875         }
 876
 877         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
 878                 socket_id, enic->config.wq_desc_count,
 879                 sizeof(struct cq_enet_wq_desc));
 880         if (err) {
 881                 vnic_wq_free(wq);
 882                 dev_err(enic, "error in allocation of cq for wq\n");
 883         }
 884
 885         /* setup up CQ message */
 886         snprintf((char *)name, sizeof(name),
 887                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
 888                 instance++);
 889
 890         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
 891                         sizeof(uint32_t), SOCKET_ID_ANY,
 892                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 893         if (!wq->cqmsg_rz)
 894                 return -ENOMEM;
 895
 896         return err;
 897 }
 898
 899 int enic_disable(struct enic *enic)
 900 {
 901         unsigned int i;
 902         int err;
 903
 904         for (i = 0; i < enic->intr_count; i++) {
 905                 vnic_intr_mask(&enic->intr[i]);
 906                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
 907         }
 908         enic_rxq_intr_deinit(enic);
 909         rte_intr_disable(&enic->pdev->intr_handle);
 910         rte_intr_callback_unregister(&enic->pdev->intr_handle,
 911                                      enic_intr_handler,
 912                                      (void *)enic->rte_dev);
 913
 914         vnic_dev_disable(enic->vdev);
 915
 916         enic_clsf_destroy(enic);
 917
 918         if (!enic_is_sriov_vf(enic))
 919                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
 920
 921         for (i = 0; i < enic->wq_count; i++) {
 922                 err = vnic_wq_disable(&enic->wq[i]);
 923                 if (err)
 924                         return err;
 925         }
 926         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
 927                 if (enic->rq[i].in_use) {
 928                         err = vnic_rq_disable(&enic->rq[i]);
 929                         if (err)
 930                                 return err;
 931                 }
 932         }
 933
 934         /* If we were using interrupts, set the interrupt vector to -1
 935          * to disable interrupts.  We are not disabling link notifcations,
 936          * though, as we want the polling of link status to continue working.
 937          */
 938         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
 939                 vnic_dev_notify_set(enic->vdev, -1);
 940
 941         vnic_dev_set_reset_flag(enic->vdev, 1);
 942
 943         for (i = 0; i < enic->wq_count; i++)
 944                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
 945
 946         for (i = 0; i < enic_vnic_rq_count(enic); i++)
 947                 if (enic->rq[i].in_use)
 948                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
 949         for (i = 0; i < enic->cq_count; i++)
 950                 vnic_cq_clean(&enic->cq[i]);
 951         for (i = 0; i < enic->intr_count; i++)
 952                 vnic_intr_clean(&enic->intr[i]);
 953
 954         return 0;
 955 }
 956
 957 static int enic_dev_wait(struct vnic_dev *vdev,
 958         int (*start)(struct vnic_dev *, int),
 959         int (*finished)(struct vnic_dev *, int *),
 960         int arg)
 961 {
 962         int done;
 963         int err;
 964         int i;
 965
 966         err = start(vdev, arg);
 967         if (err)
 968                 return err;
 969
 970         /* Wait for func to complete...2 seconds max */
 971         for (i = 0; i < 2000; i++) {
 972                 err = finished(vdev, &done);
 973                 if (err)
 974                         return err;
 975                 if (done)
 976                         return 0;
 977                 usleep(1000);
 978         }
 979         return -ETIMEDOUT;
 980 }
 981
 982 static int enic_dev_open(struct enic *enic)
 983 {
 984         int err;
 985         int flags = CMD_OPENF_IG_DESCCACHE;
 986
 987         err = enic_dev_wait(enic->vdev, vnic_dev_open,
 988                 vnic_dev_open_done, flags);
 989         if (err)
 990                 dev_err(enic_get_dev(enic),
 991                         "vNIC device open failed, err %d\n", err);
 992
 993         return err;
 994 }
 995
 996 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
 997 {
 998         dma_addr_t rss_key_buf_pa;
 999         union vnic_rss_key *rss_key_buf_va = NULL;
1000         int err, i;
1001         u8 name[NAME_MAX];
1002
1003         RTE_ASSERT(user_key != NULL);
1004         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1005         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1006                 &rss_key_buf_pa, name);
1007         if (!rss_key_buf_va)
1008                 return -ENOMEM;
1009
1010         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1011                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1012
1013         err = enic_set_rss_key(enic,
1014                 rss_key_buf_pa,
1015                 sizeof(union vnic_rss_key));
1016
1017         /* Save for later queries */
1018         if (!err) {
1019                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1020                            sizeof(union vnic_rss_key));
1021         }
1022         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1023                 rss_key_buf_va, rss_key_buf_pa);
1024
1025         return err;
1026 }
1027
1028 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1029 {
1030         dma_addr_t rss_cpu_buf_pa;
1031         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1032         int err;
1033         u8 name[NAME_MAX];
1034
1035         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1036         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1037                 &rss_cpu_buf_pa, name);
1038         if (!rss_cpu_buf_va)
1039                 return -ENOMEM;
1040
1041         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1042
1043         err = enic_set_rss_cpu(enic,
1044                 rss_cpu_buf_pa,
1045                 sizeof(union vnic_rss_cpu));
1046
1047         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1048                 rss_cpu_buf_va, rss_cpu_buf_pa);
1049
1050         /* Save for later queries */
1051         if (!err)
1052                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1053         return err;
1054 }
1055
1056 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1057         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1058 {
1059         const u8 tso_ipid_split_en = 0;
1060         int err;
1061
1062         err = enic_set_nic_cfg(enic,
1063                 rss_default_cpu, rss_hash_type,
1064                 rss_hash_bits, rss_base_cpu,
1065                 rss_enable, tso_ipid_split_en,
1066                 enic->ig_vlan_strip_en);
1067
1068         return err;
1069 }
1070
1071 /* Initialize RSS with defaults, called from dev_configure */
1072 int enic_init_rss_nic_cfg(struct enic *enic)
1073 {
1074         static uint8_t default_rss_key[] = {
1075                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1076                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1077                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1078                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1079         };
1080         struct rte_eth_rss_conf rss_conf;
1081         union vnic_rss_cpu rss_cpu;
1082         int ret, i;
1083
1084         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1085         /*
1086          * If setting key for the first time, and the user gives us none, then
1087          * push the default key to NIC.
1088          */
1089         if (rss_conf.rss_key == NULL) {
1090                 rss_conf.rss_key = default_rss_key;
1091                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1092         }
1093         ret = enic_set_rss_conf(enic, &rss_conf);
1094         if (ret) {
1095                 dev_err(enic, "Failed to configure RSS\n");
1096                 return ret;
1097         }
1098         if (enic->rss_enable) {
1099                 /* If enabling RSS, use the default reta */
1100                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1101                         rss_cpu.cpu[i / 4].b[i % 4] =
1102                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1103                 }
1104                 ret = enic_set_rss_reta(enic, &rss_cpu);
1105                 if (ret)
1106                         dev_err(enic, "Failed to set RSS indirection table\n");
1107         }
1108         return ret;
1109 }
1110
1111 int enic_setup_finish(struct enic *enic)
1112 {
1113         enic_init_soft_stats(enic);
1114
1115         /* Default conf */
1116         vnic_dev_packet_filter(enic->vdev,
1117                 1 /* directed  */,
1118                 1 /* multicast */,
1119                 1 /* broadcast */,
1120                 0 /* promisc   */,
1121                 1 /* allmulti  */);
1122
1123         enic->promisc = 0;
1124         enic->allmulti = 1;
1125
1126         return 0;
1127 }
1128
1129 static int enic_rss_conf_valid(struct enic *enic,
1130                                struct rte_eth_rss_conf *rss_conf)
1131 {
1132         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1133         if (enic->flow_type_rss_offloads == 0)
1134                 return 0;
1135         if (rss_conf->rss_key != NULL &&
1136             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1137                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1138                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1139                 return -EINVAL;
1140         }
1141         if (rss_conf->rss_hf != 0 &&
1142             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1143                 dev_err(enic, "Given rss_hf contains none of the supported"
1144                         " types\n");
1145                 return -EINVAL;
1146         }
1147         return 0;
1148 }
1149
1150 /* Set hash type and key according to rss_conf */
1151 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1152 {
1153         struct rte_eth_dev *eth_dev;
1154         uint64_t rss_hf;
1155         u8 rss_hash_type;
1156         u8 rss_enable;
1157         int ret;
1158
1159         RTE_ASSERT(rss_conf != NULL);
1160         ret = enic_rss_conf_valid(enic, rss_conf);
1161         if (ret) {
1162                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1163                 return ret;
1164         }
1165
1166         eth_dev = enic->rte_dev;
1167         rss_hash_type = 0;
1168         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1169         if (enic->rq_count > 1 &&
1170             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1171             rss_hf != 0) {
1172                 rss_enable = 1;
1173                 if (rss_hf & ETH_RSS_IPV4)
1174                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1175                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1176                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1177                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1178                         /*
1179                          * 'TCP' is not a typo. HW does not have a separate
1180                          * enable bit for UDP RSS. The TCP bit enables both TCP
1181                          * and UDP RSS..
1182                          */
1183                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1184                 }
1185                 if (rss_hf & ETH_RSS_IPV6)
1186                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1187                 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1188                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1189                 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) {
1190                         /* Again, 'TCP' is not a typo. */
1191                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1192                 }
1193                 if (rss_hf & ETH_RSS_IPV6_EX)
1194                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6_EX;
1195                 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
1196                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX;
1197         } else {
1198                 rss_enable = 0;
1199                 rss_hf = 0;
1200         }
1201
1202         /* Set the hash key if provided */
1203         if (rss_enable && rss_conf->rss_key) {
1204                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1205                 if (ret) {
1206                         dev_err(enic, "Failed to set RSS key\n");
1207                         return ret;
1208                 }
1209         }
1210
1211         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1212                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1213                               rss_enable);
1214         if (!ret) {
1215                 enic->rss_hf = rss_hf;
1216                 enic->rss_hash_type = rss_hash_type;
1217                 enic->rss_enable = rss_enable;
1218         }
1219         return 0;
1220 }
1221
1222 int enic_set_vlan_strip(struct enic *enic)
1223 {
1224         /*
1225          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1226          * together. So, re-do niccfg, preserving the current RSS settings.
1227          */
1228         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1229                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1230                                enic->rss_enable);
1231 }
1232
1233 void enic_add_packet_filter(struct enic *enic)
1234 {
1235         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1236         vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1237                 enic->promisc, enic->allmulti);
1238 }
1239
1240 int enic_get_link_status(struct enic *enic)
1241 {
1242         return vnic_dev_link_status(enic->vdev);
1243 }
1244
1245 static void enic_dev_deinit(struct enic *enic)
1246 {
1247         struct rte_eth_dev *eth_dev = enic->rte_dev;
1248
1249         /* stop link status checking */
1250         vnic_dev_notify_unset(enic->vdev);
1251
1252         rte_free(eth_dev->data->mac_addrs);
1253         rte_free(enic->cq);
1254         rte_free(enic->intr);
1255         rte_free(enic->rq);
1256         rte_free(enic->wq);
1257 }
1258
1259
1260 int enic_set_vnic_res(struct enic *enic)
1261 {
1262         struct rte_eth_dev *eth_dev = enic->rte_dev;
1263         int rc = 0;
1264         unsigned int required_rq, required_wq, required_cq, required_intr;
1265
1266         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1267         required_rq = eth_dev->data->nb_rx_queues * 2;
1268         required_wq = eth_dev->data->nb_tx_queues;
1269         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1270         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1271         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1272                 required_intr += eth_dev->data->nb_rx_queues;
1273         }
1274
1275         if (enic->conf_rq_count < required_rq) {
1276                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1277                         eth_dev->data->nb_rx_queues,
1278                         required_rq, enic->conf_rq_count);
1279                 rc = -EINVAL;
1280         }
1281         if (enic->conf_wq_count < required_wq) {
1282                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1283                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1284                 rc = -EINVAL;
1285         }
1286
1287         if (enic->conf_cq_count < required_cq) {
1288                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1289                         required_cq, enic->conf_cq_count);
1290                 rc = -EINVAL;
1291         }
1292         if (enic->conf_intr_count < required_intr) {
1293                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1294                         " interrupts. Required:%u, Configured:%u\n",
1295                         required_intr, enic->conf_intr_count);
1296                 rc = -EINVAL;
1297         }
1298
1299         if (rc == 0) {
1300                 enic->rq_count = eth_dev->data->nb_rx_queues;
1301                 enic->wq_count = eth_dev->data->nb_tx_queues;
1302                 enic->cq_count = enic->rq_count + enic->wq_count;
1303                 enic->intr_count = required_intr;
1304         }
1305
1306         return rc;
1307 }
1308
1309 /* Initialize the completion queue for an RQ */
1310 static int
1311 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1312 {
1313         struct vnic_rq *sop_rq, *data_rq;
1314         unsigned int cq_idx;
1315         int rc = 0;
1316
1317         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1318         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1319         cq_idx = rq_idx;
1320
1321         vnic_cq_clean(&enic->cq[cq_idx]);
1322         vnic_cq_init(&enic->cq[cq_idx],
1323                      0 /* flow_control_enable */,
1324                      1 /* color_enable */,
1325                      0 /* cq_head */,
1326                      0 /* cq_tail */,
1327                      1 /* cq_tail_color */,
1328                      0 /* interrupt_enable */,
1329                      1 /* cq_entry_enable */,
1330                      0 /* cq_message_enable */,
1331                      0 /* interrupt offset */,
1332                      0 /* cq_message_addr */);
1333
1334
1335         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1336                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1337                            sop_rq->ring.desc_count - 1, 1, 0);
1338         if (data_rq->in_use) {
1339                 vnic_rq_init_start(data_rq,
1340                                    enic_cq_rq(enic,
1341                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1342                                    data_rq->ring.desc_count - 1, 1, 0);
1343         }
1344
1345         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1346         if (rc)
1347                 return rc;
1348
1349         if (data_rq->in_use) {
1350                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1351                 if (rc) {
1352                         enic_rxmbuf_queue_release(enic, sop_rq);
1353                         return rc;
1354                 }
1355         }
1356
1357         return 0;
1358 }
1359
1360 /* The Cisco NIC can send and receive packets up to a max packet size
1361  * determined by the NIC type and firmware. There is also an MTU
1362  * configured into the NIC via the CIMC/UCSM management interface
1363  * which can be overridden by this function (up to the max packet size).
1364  * Depending on the network setup, doing so may cause packet drops
1365  * and unexpected behavior.
1366  */
1367 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1368 {
1369         unsigned int rq_idx;
1370         struct vnic_rq *rq;
1371         int rc = 0;
1372         uint16_t old_mtu;       /* previous setting */
1373         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1374         struct rte_eth_dev *eth_dev = enic->rte_dev;
1375
1376         old_mtu = eth_dev->data->mtu;
1377         config_mtu = enic->config.mtu;
1378
1379         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1380                 return -E_RTE_SECONDARY;
1381
1382         if (new_mtu > enic->max_mtu) {
1383                 dev_err(enic,
1384                         "MTU not updated: requested (%u) greater than max (%u)\n",
1385                         new_mtu, enic->max_mtu);
1386                 return -EINVAL;
1387         }
1388         if (new_mtu < ENIC_MIN_MTU) {
1389                 dev_info(enic,
1390                         "MTU not updated: requested (%u) less than min (%u)\n",
1391                         new_mtu, ENIC_MIN_MTU);
1392                 return -EINVAL;
1393         }
1394         if (new_mtu > config_mtu)
1395                 dev_warning(enic,
1396                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1397                         new_mtu, config_mtu);
1398
1399         /* The easy case is when scatter is disabled. However if the MTU
1400          * becomes greater than the mbuf data size, packet drops will ensue.
1401          */
1402         if (!(enic->rte_dev->data->dev_conf.rxmode.offloads &
1403               DEV_RX_OFFLOAD_SCATTER)) {
1404                 eth_dev->data->mtu = new_mtu;
1405                 goto set_mtu_done;
1406         }
1407
1408         /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to
1409          * change Rx scatter mode if necessary for better performance. I.e. if
1410          * MTU was greater than the mbuf size and now it's less, scatter Rx
1411          * doesn't have to be used and vice versa.
1412           */
1413         rte_spinlock_lock(&enic->mtu_lock);
1414
1415         /* Stop traffic on all RQs */
1416         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1417                 rq = &enic->rq[rq_idx];
1418                 if (rq->is_sop && rq->in_use) {
1419                         rc = enic_stop_rq(enic,
1420                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1421                         if (rc) {
1422                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1423                                 goto set_mtu_done;
1424                         }
1425                 }
1426         }
1427
1428         /* replace Rx function with a no-op to avoid getting stale pkts */
1429         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1430         rte_mb();
1431
1432         /* Allow time for threads to exit the real Rx function. */
1433         usleep(100000);
1434
1435         /* now it is safe to reconfigure the RQs */
1436
1437         /* update the mtu */
1438         eth_dev->data->mtu = new_mtu;
1439
1440         /* free and reallocate RQs with the new MTU */
1441         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1442                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1443
1444                 enic_free_rq(rq);
1445                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1446                                    rq->tot_nb_desc, rq->rx_free_thresh);
1447                 if (rc) {
1448                         dev_err(enic,
1449                                 "Fatal MTU alloc error- No traffic will pass\n");
1450                         goto set_mtu_done;
1451                 }
1452
1453                 rc = enic_reinit_rq(enic, rq_idx);
1454                 if (rc) {
1455                         dev_err(enic,
1456                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1457                         goto set_mtu_done;
1458                 }
1459         }
1460
1461         /* put back the real receive function */
1462         rte_mb();
1463         eth_dev->rx_pkt_burst = enic_recv_pkts;
1464         rte_mb();
1465
1466         /* restart Rx traffic */
1467         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1468                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1469                 if (rq->is_sop && rq->in_use)
1470                         enic_start_rq(enic, rq_idx);
1471         }
1472
1473 set_mtu_done:
1474         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1475         rte_spinlock_unlock(&enic->mtu_lock);
1476         return rc;
1477 }
1478
1479 static int enic_dev_init(struct enic *enic)
1480 {
1481         int err;
1482         struct rte_eth_dev *eth_dev = enic->rte_dev;
1483
1484         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1485
1486         /* Get vNIC configuration
1487         */
1488         err = enic_get_vnic_config(enic);
1489         if (err) {
1490                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1491                 return err;
1492         }
1493
1494         /* Get available resource counts */
1495         enic_get_res_counts(enic);
1496         if (enic->conf_rq_count == 1) {
1497                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1498                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1499                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1500                 return -EINVAL;
1501         }
1502         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1503         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1504                                enic->conf_cq_count, 8);
1505         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1506                                  enic->conf_intr_count, 8);
1507         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1508                                enic->conf_rq_count, 8);
1509         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1510                                enic->conf_wq_count, 8);
1511         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1512                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1513                 return -1;
1514         }
1515         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1516                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1517                 return -1;
1518         }
1519         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1520                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1521                 return -1;
1522         }
1523         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1524                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1525                 return -1;
1526         }
1527
1528         /* Get the supported filters */
1529         enic_fdir_info(enic);
1530
1531         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1532                                                 * ENIC_MAX_MAC_ADDR, 0);
1533         if (!eth_dev->data->mac_addrs) {
1534                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1535                 return -1;
1536         }
1537         ether_addr_copy((struct ether_addr *) enic->mac_addr,
1538                         eth_dev->data->mac_addrs);
1539
1540         vnic_dev_set_reset_flag(enic->vdev, 0);
1541
1542         LIST_INIT(&enic->flows);
1543         rte_spinlock_init(&enic->flows_lock);
1544
1545         /* set up link status checking */
1546         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1547
1548         return 0;
1549
1550 }
1551
1552 int enic_probe(struct enic *enic)
1553 {
1554         struct rte_pci_device *pdev = enic->pdev;
1555         int err = -1;
1556
1557         dev_debug(enic, " Initializing ENIC PMD\n");
1558
1559         /* if this is a secondary process the hardware is already initialized */
1560         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1561                 return 0;
1562
1563         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1564         enic->bar0.len = pdev->mem_resource[0].len;
1565
1566         /* Register vNIC device */
1567         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1568         if (!enic->vdev) {
1569                 dev_err(enic, "vNIC registration failed, aborting\n");
1570                 goto err_out;
1571         }
1572
1573         LIST_INIT(&enic->memzone_list);
1574         rte_spinlock_init(&enic->memzone_list_lock);
1575
1576         vnic_register_cbacks(enic->vdev,
1577                 enic_alloc_consistent,
1578                 enic_free_consistent);
1579
1580         /*
1581          * Allocate the consistent memory for stats upfront so both primary and
1582          * secondary processes can dump stats.
1583          */
1584         err = vnic_dev_alloc_stats_mem(enic->vdev);
1585         if (err) {
1586                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1587                 goto err_out_unregister;
1588         }
1589         /* Issue device open to get device in known state */
1590         err = enic_dev_open(enic);
1591         if (err) {
1592                 dev_err(enic, "vNIC dev open failed, aborting\n");
1593                 goto err_out_unregister;
1594         }
1595
1596         /* Set ingress vlan rewrite mode before vnic initialization */
1597         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1598                 IG_VLAN_REWRITE_MODE_PASS_THRU);
1599         if (err) {
1600                 dev_err(enic,
1601                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1602                 goto err_out_dev_close;
1603         }
1604
1605         /* Issue device init to initialize the vnic-to-switch link.
1606          * We'll start with carrier off and wait for link UP
1607          * notification later to turn on carrier.  We don't need
1608          * to wait here for the vnic-to-switch link initialization
1609          * to complete; link UP notification is the indication that
1610          * the process is complete.
1611          */
1612
1613         err = vnic_dev_init(enic->vdev, 0);
1614         if (err) {
1615                 dev_err(enic, "vNIC dev init failed, aborting\n");
1616                 goto err_out_dev_close;
1617         }
1618
1619         err = enic_dev_init(enic);
1620         if (err) {
1621                 dev_err(enic, "Device initialization failed, aborting\n");
1622                 goto err_out_dev_close;
1623         }
1624
1625         return 0;
1626
1627 err_out_dev_close:
1628         vnic_dev_close(enic->vdev);
1629 err_out_unregister:
1630         vnic_dev_unregister(enic->vdev);
1631 err_out:
1632         return err;
1633 }
1634
1635 void enic_remove(struct enic *enic)
1636 {
1637         enic_dev_deinit(enic);
1638         vnic_dev_close(enic->vdev);
1639         vnic_dev_unregister(enic->vdev);
1640 }