drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11 #include <libgen.h>
  12
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_ethdev_driver.h>
  20
  21 #include "enic_compat.h"
  22 #include "enic.h"
  23 #include "wq_enet_desc.h"
  24 #include "rq_enet_desc.h"
  25 #include "cq_enet_desc.h"
  26 #include "vnic_enet.h"
  27 #include "vnic_dev.h"
  28 #include "vnic_wq.h"
  29 #include "vnic_rq.h"
  30 #include "vnic_cq.h"
  31 #include "vnic_intr.h"
  32 #include "vnic_nic.h"
  33
  34 static inline int enic_is_sriov_vf(struct enic *enic)
  35 {
  36         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  37 }
  38
  39 static int is_zero_addr(uint8_t *addr)
  40 {
  41         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  42 }
  43
  44 static int is_mcast_addr(uint8_t *addr)
  45 {
  46         return addr[0] & 1;
  47 }
  48
  49 static int is_eth_addr_valid(uint8_t *addr)
  50 {
  51         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  52 }
  53
  54 static void
  55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  56 {
  57         uint16_t i;
  58
  59         if (!rq || !rq->mbuf_ring) {
  60                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  61                 return;
  62         }
  63
  64         for (i = 0; i < rq->ring.desc_count; i++) {
  65                 if (rq->mbuf_ring[i]) {
  66                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  67                         rq->mbuf_ring[i] = NULL;
  68                 }
  69         }
  70 }
  71
  72 static void enic_free_wq_buf(struct vnic_wq_buf *buf)
  73 {
  74         struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
  75
  76         rte_pktmbuf_free_seg(mbuf);
  77         buf->mb = NULL;
  78 }
  79
  80 static void enic_log_q_error(struct enic *enic)
  81 {
  82         unsigned int i;
  83         u32 error_status;
  84
  85         for (i = 0; i < enic->wq_count; i++) {
  86                 error_status = vnic_wq_error_status(&enic->wq[i]);
  87                 if (error_status)
  88                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  89                                 error_status);
  90         }
  91
  92         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  93                 if (!enic->rq[i].in_use)
  94                         continue;
  95                 error_status = vnic_rq_error_status(&enic->rq[i]);
  96                 if (error_status)
  97                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  98                                 error_status);
  99         }
 100 }
 101
 102 static void enic_clear_soft_stats(struct enic *enic)
 103 {
 104         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 105         rte_atomic64_clear(&soft_stats->rx_nombuf);
 106         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 107         rte_atomic64_clear(&soft_stats->tx_oversized);
 108 }
 109
 110 static void enic_init_soft_stats(struct enic *enic)
 111 {
 112         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 113         rte_atomic64_init(&soft_stats->rx_nombuf);
 114         rte_atomic64_init(&soft_stats->rx_packet_errors);
 115         rte_atomic64_init(&soft_stats->tx_oversized);
 116         enic_clear_soft_stats(enic);
 117 }
 118
 119 void enic_dev_stats_clear(struct enic *enic)
 120 {
 121         if (vnic_dev_stats_clear(enic->vdev))
 122                 dev_err(enic, "Error in clearing stats\n");
 123         enic_clear_soft_stats(enic);
 124 }
 125
 126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 127 {
 128         struct vnic_stats *stats;
 129         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 130         int64_t rx_truncated;
 131         uint64_t rx_packet_errors;
 132         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 133
 134         if (ret) {
 135                 dev_err(enic, "Error in getting stats\n");
 136                 return ret;
 137         }
 138
 139         /* The number of truncated packets can only be calculated by
 140          * subtracting a hardware counter from error packets received by
 141          * the driver. Note: this causes transient inaccuracies in the
 142          * ipackets count. Also, the length of truncated packets are
 143          * counted in ibytes even though truncated packets are dropped
 144          * which can make ibytes be slightly higher than it should be.
 145          */
 146         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 147         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 148
 149         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 150         r_stats->opackets = stats->tx.tx_frames_ok;
 151
 152         r_stats->ibytes = stats->rx.rx_bytes_ok;
 153         r_stats->obytes = stats->tx.tx_bytes_ok;
 154
 155         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 156         r_stats->oerrors = stats->tx.tx_errors
 157                            + rte_atomic64_read(&soft_stats->tx_oversized);
 158
 159         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 160
 161         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 162         return 0;
 163 }
 164
 165 void enic_del_mac_address(struct enic *enic, int mac_index)
 166 {
 167         struct rte_eth_dev *eth_dev = enic->rte_dev;
 168         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 169
 170         if (vnic_dev_del_addr(enic->vdev, mac_addr))
 171                 dev_err(enic, "del mac addr failed\n");
 172 }
 173
 174 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 175 {
 176         int err;
 177
 178         if (!is_eth_addr_valid(mac_addr)) {
 179                 dev_err(enic, "invalid mac address\n");
 180                 return -EINVAL;
 181         }
 182
 183         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 184         if (err)
 185                 dev_err(enic, "add mac addr failed\n");
 186         return err;
 187 }
 188
 189 static void
 190 enic_free_rq_buf(struct rte_mbuf **mbuf)
 191 {
 192         if (*mbuf == NULL)
 193                 return;
 194
 195         rte_pktmbuf_free(*mbuf);
 196         *mbuf = NULL;
 197 }
 198
 199 void enic_init_vnic_resources(struct enic *enic)
 200 {
 201         unsigned int error_interrupt_enable = 1;
 202         unsigned int error_interrupt_offset = 0;
 203         unsigned int rxq_interrupt_enable = 0;
 204         unsigned int rxq_interrupt_offset;
 205         unsigned int index = 0;
 206         unsigned int cq_idx;
 207         struct vnic_rq *data_rq;
 208
 209         if (enic->rte_dev->data->dev_conf.intr_conf.rxq) {
 210                 rxq_interrupt_enable = 1;
 211                 rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 212         }
 213         for (index = 0; index < enic->rq_count; index++) {
 214                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 215
 216                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 217                         cq_idx,
 218                         error_interrupt_enable,
 219                         error_interrupt_offset);
 220
 221                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 222                 if (data_rq->in_use)
 223                         vnic_rq_init(data_rq,
 224                                      cq_idx,
 225                                      error_interrupt_enable,
 226                                      error_interrupt_offset);
 227
 228                 vnic_cq_init(&enic->cq[cq_idx],
 229                         0 /* flow_control_enable */,
 230                         1 /* color_enable */,
 231                         0 /* cq_head */,
 232                         0 /* cq_tail */,
 233                         1 /* cq_tail_color */,
 234                         rxq_interrupt_enable,
 235                         1 /* cq_entry_enable */,
 236                         0 /* cq_message_enable */,
 237                         rxq_interrupt_offset,
 238                         0 /* cq_message_addr */);
 239                 if (rxq_interrupt_enable)
 240                         rxq_interrupt_offset++;
 241         }
 242
 243         for (index = 0; index < enic->wq_count; index++) {
 244                 vnic_wq_init(&enic->wq[index],
 245                         enic_cq_wq(enic, index),
 246                         error_interrupt_enable,
 247                         error_interrupt_offset);
 248
 249                 cq_idx = enic_cq_wq(enic, index);
 250                 vnic_cq_init(&enic->cq[cq_idx],
 251                         0 /* flow_control_enable */,
 252                         1 /* color_enable */,
 253                         0 /* cq_head */,
 254                         0 /* cq_tail */,
 255                         1 /* cq_tail_color */,
 256                         0 /* interrupt_enable */,
 257                         0 /* cq_entry_enable */,
 258                         1 /* cq_message_enable */,
 259                         0 /* interrupt offset */,
 260                         (u64)enic->wq[index].cqmsg_rz->iova);
 261         }
 262
 263         for (index = 0; index < enic->intr_count; index++) {
 264                 vnic_intr_init(&enic->intr[index],
 265                                enic->config.intr_timer_usec,
 266                                enic->config.intr_timer_type,
 267                                /*mask_on_assertion*/1);
 268         }
 269 }
 270
 271
 272 static int
 273 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 274 {
 275         struct rte_mbuf *mb;
 276         struct rq_enet_desc *rqd = rq->ring.descs;
 277         unsigned i;
 278         dma_addr_t dma_addr;
 279         uint32_t max_rx_pkt_len;
 280         uint16_t rq_buf_len;
 281
 282         if (!rq->in_use)
 283                 return 0;
 284
 285         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 286                   rq->ring.desc_count);
 287
 288         /*
 289          * If *not* using scatter and the mbuf size is smaller than the
 290          * requested max packet size (max_rx_pkt_len), then reduce the
 291          * posted buffer size to max_rx_pkt_len. HW still receives packets
 292          * larger than max_rx_pkt_len, but they will be truncated, which we
 293          * drop in the rx handler. Not ideal, but better than returning
 294          * large packets when the user is not expecting them.
 295          */
 296         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 297         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 298         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 299                 rq_buf_len = max_rx_pkt_len;
 300         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 301                 mb = rte_mbuf_raw_alloc(rq->mp);
 302                 if (mb == NULL) {
 303                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 304                         (unsigned)rq->index);
 305                         return -ENOMEM;
 306                 }
 307
 308                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 309                 dma_addr = (dma_addr_t)(mb->buf_iova
 310                            + RTE_PKTMBUF_HEADROOM);
 311                 rq_enet_desc_enc(rqd, dma_addr,
 312                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 313                                 : RQ_ENET_TYPE_NOT_SOP),
 314                                 rq_buf_len);
 315                 rq->mbuf_ring[i] = mb;
 316         }
 317
 318         /* make sure all prior writes are complete before doing the PIO write */
 319         rte_rmb();
 320
 321         /* Post all but the last buffer to VIC. */
 322         rq->posted_index = rq->ring.desc_count - 1;
 323
 324         rq->rx_nb_hold = 0;
 325
 326         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 327                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 328         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 329         iowrite32(0, &rq->ctrl->fetch_index);
 330         rte_rmb();
 331
 332         return 0;
 333
 334 }
 335
 336 static void *
 337 enic_alloc_consistent(void *priv, size_t size,
 338         dma_addr_t *dma_handle, u8 *name)
 339 {
 340         void *vaddr;
 341         const struct rte_memzone *rz;
 342         *dma_handle = 0;
 343         struct enic *enic = (struct enic *)priv;
 344         struct enic_memzone_entry *mze;
 345
 346         rz = rte_memzone_reserve_aligned((const char *)name,
 347                                          size, SOCKET_ID_ANY, 0, ENIC_ALIGN);
 348         if (!rz) {
 349                 pr_err("%s : Failed to allocate memory requested for %s\n",
 350                         __func__, name);
 351                 return NULL;
 352         }
 353
 354         vaddr = rz->addr;
 355         *dma_handle = (dma_addr_t)rz->iova;
 356
 357         mze = rte_malloc("enic memzone entry",
 358                          sizeof(struct enic_memzone_entry), 0);
 359
 360         if (!mze) {
 361                 pr_err("%s : Failed to allocate memory for memzone list\n",
 362                        __func__);
 363                 rte_memzone_free(rz);
 364                 return NULL;
 365         }
 366
 367         mze->rz = rz;
 368
 369         rte_spinlock_lock(&enic->memzone_list_lock);
 370         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 371         rte_spinlock_unlock(&enic->memzone_list_lock);
 372
 373         return vaddr;
 374 }
 375
 376 static void
 377 enic_free_consistent(void *priv,
 378                      __rte_unused size_t size,
 379                      void *vaddr,
 380                      dma_addr_t dma_handle)
 381 {
 382         struct enic_memzone_entry *mze;
 383         struct enic *enic = (struct enic *)priv;
 384
 385         rte_spinlock_lock(&enic->memzone_list_lock);
 386         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 387                 if (mze->rz->addr == vaddr &&
 388                     mze->rz->iova == dma_handle)
 389                         break;
 390         }
 391         if (mze == NULL) {
 392                 rte_spinlock_unlock(&enic->memzone_list_lock);
 393                 dev_warning(enic,
 394                             "Tried to free memory, but couldn't find it in the memzone list\n");
 395                 return;
 396         }
 397         LIST_REMOVE(mze, entries);
 398         rte_spinlock_unlock(&enic->memzone_list_lock);
 399         rte_memzone_free(mze->rz);
 400         rte_free(mze);
 401 }
 402
 403 int enic_link_update(struct enic *enic)
 404 {
 405         struct rte_eth_dev *eth_dev = enic->rte_dev;
 406         struct rte_eth_link link;
 407
 408         memset(&link, 0, sizeof(link));
 409         link.link_status = enic_get_link_status(enic);
 410         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 411         link.link_speed = vnic_dev_port_speed(enic->vdev);
 412
 413         return rte_eth_linkstatus_set(eth_dev, &link);
 414 }
 415
 416 static void
 417 enic_intr_handler(void *arg)
 418 {
 419         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 420         struct enic *enic = pmd_priv(dev);
 421
 422         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 423
 424         enic_link_update(enic);
 425         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 426         enic_log_q_error(enic);
 427 }
 428
 429 static int enic_rxq_intr_init(struct enic *enic)
 430 {
 431         struct rte_intr_handle *intr_handle;
 432         uint32_t rxq_intr_count, i;
 433         int err;
 434
 435         intr_handle = enic->rte_dev->intr_handle;
 436         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 437                 return 0;
 438         /*
 439          * Rx queue interrupts only work when we have MSI-X interrupts,
 440          * one per queue. Sharing one interrupt is technically
 441          * possible with VIC, but it is not worth the complications it brings.
 442          */
 443         if (!rte_intr_cap_multiple(intr_handle)) {
 444                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 445                         " (vfio-pci driver)\n");
 446                 return -ENOTSUP;
 447         }
 448         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 449         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 450         if (err) {
 451                 dev_err(enic, "Failed to enable event fds for Rx queue"
 452                         " interrupts\n");
 453                 return err;
 454         }
 455         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 456                                             rxq_intr_count * sizeof(int), 0);
 457         if (intr_handle->intr_vec == NULL) {
 458                 dev_err(enic, "Failed to allocate intr_vec\n");
 459                 return -ENOMEM;
 460         }
 461         for (i = 0; i < rxq_intr_count; i++)
 462                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 463         return 0;
 464 }
 465
 466 static void enic_rxq_intr_deinit(struct enic *enic)
 467 {
 468         struct rte_intr_handle *intr_handle;
 469
 470         intr_handle = enic->rte_dev->intr_handle;
 471         rte_intr_efd_disable(intr_handle);
 472         if (intr_handle->intr_vec != NULL) {
 473                 rte_free(intr_handle->intr_vec);
 474                 intr_handle->intr_vec = NULL;
 475         }
 476 }
 477
 478 int enic_enable(struct enic *enic)
 479 {
 480         unsigned int index;
 481         int err;
 482         struct rte_eth_dev *eth_dev = enic->rte_dev;
 483
 484         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 485         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 486
 487         /* vnic notification of link status has already been turned on in
 488          * enic_dev_init() which is called during probe time.  Here we are
 489          * just turning on interrupt vector 0 if needed.
 490          */
 491         if (eth_dev->data->dev_conf.intr_conf.lsc)
 492                 vnic_dev_notify_set(enic->vdev, 0);
 493
 494         err = enic_rxq_intr_init(enic);
 495         if (err)
 496                 return err;
 497         if (enic_clsf_init(enic))
 498                 dev_warning(enic, "Init of hash table for clsf failed."\
 499                         "Flow director feature will not work\n");
 500
 501         for (index = 0; index < enic->rq_count; index++) {
 502                 err = enic_alloc_rx_queue_mbufs(enic,
 503                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 504                 if (err) {
 505                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 506                         return err;
 507                 }
 508                 err = enic_alloc_rx_queue_mbufs(enic,
 509                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 510                 if (err) {
 511                         /* release the allocated mbufs for the sop rq*/
 512                         enic_rxmbuf_queue_release(enic,
 513                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 514
 515                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 516                         return err;
 517                 }
 518         }
 519
 520         for (index = 0; index < enic->wq_count; index++)
 521                 enic_start_wq(enic, index);
 522         for (index = 0; index < enic->rq_count; index++)
 523                 enic_start_rq(enic, index);
 524
 525         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 526
 527         vnic_dev_enable_wait(enic->vdev);
 528
 529         /* Register and enable error interrupt */
 530         rte_intr_callback_register(&(enic->pdev->intr_handle),
 531                 enic_intr_handler, (void *)enic->rte_dev);
 532
 533         rte_intr_enable(&(enic->pdev->intr_handle));
 534         /* Unmask LSC interrupt */
 535         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 536
 537         return 0;
 538 }
 539
 540 int enic_alloc_intr_resources(struct enic *enic)
 541 {
 542         int err;
 543         unsigned int i;
 544
 545         dev_info(enic, "vNIC resources used:  "\
 546                 "wq %d rq %d cq %d intr %d\n",
 547                 enic->wq_count, enic_vnic_rq_count(enic),
 548                 enic->cq_count, enic->intr_count);
 549
 550         for (i = 0; i < enic->intr_count; i++) {
 551                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 552                 if (err) {
 553                         enic_free_vnic_resources(enic);
 554                         return err;
 555                 }
 556         }
 557         return 0;
 558 }
 559
 560 void enic_free_rq(void *rxq)
 561 {
 562         struct vnic_rq *rq_sop, *rq_data;
 563         struct enic *enic;
 564
 565         if (rxq == NULL)
 566                 return;
 567
 568         rq_sop = (struct vnic_rq *)rxq;
 569         enic = vnic_dev_priv(rq_sop->vdev);
 570         rq_data = &enic->rq[rq_sop->data_queue_idx];
 571
 572         enic_rxmbuf_queue_release(enic, rq_sop);
 573         if (rq_data->in_use)
 574                 enic_rxmbuf_queue_release(enic, rq_data);
 575
 576         rte_free(rq_sop->mbuf_ring);
 577         if (rq_data->in_use)
 578                 rte_free(rq_data->mbuf_ring);
 579
 580         rq_sop->mbuf_ring = NULL;
 581         rq_data->mbuf_ring = NULL;
 582
 583         vnic_rq_free(rq_sop);
 584         if (rq_data->in_use)
 585                 vnic_rq_free(rq_data);
 586
 587         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 588
 589         rq_sop->in_use = 0;
 590         rq_data->in_use = 0;
 591 }
 592
 593 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 594 {
 595         struct rte_eth_dev *eth_dev = enic->rte_dev;
 596         vnic_wq_enable(&enic->wq[queue_idx]);
 597         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 598 }
 599
 600 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 601 {
 602         struct rte_eth_dev *eth_dev = enic->rte_dev;
 603         int ret;
 604
 605         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 606         if (ret)
 607                 return ret;
 608
 609         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 610         return 0;
 611 }
 612
 613 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 614 {
 615         struct vnic_rq *rq_sop;
 616         struct vnic_rq *rq_data;
 617         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 618         rq_data = &enic->rq[rq_sop->data_queue_idx];
 619         struct rte_eth_dev *eth_dev = enic->rte_dev;
 620
 621         if (rq_data->in_use)
 622                 vnic_rq_enable(rq_data);
 623         rte_mb();
 624         vnic_rq_enable(rq_sop);
 625         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 626 }
 627
 628 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 629 {
 630         int ret1 = 0, ret2 = 0;
 631         struct rte_eth_dev *eth_dev = enic->rte_dev;
 632         struct vnic_rq *rq_sop;
 633         struct vnic_rq *rq_data;
 634         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 635         rq_data = &enic->rq[rq_sop->data_queue_idx];
 636
 637         ret2 = vnic_rq_disable(rq_sop);
 638         rte_mb();
 639         if (rq_data->in_use)
 640                 ret1 = vnic_rq_disable(rq_data);
 641
 642         if (ret2)
 643                 return ret2;
 644         else if (ret1)
 645                 return ret1;
 646
 647         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 648         return 0;
 649 }
 650
 651 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 652         unsigned int socket_id, struct rte_mempool *mp,
 653         uint16_t nb_desc, uint16_t free_thresh)
 654 {
 655         int rc;
 656         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 657         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 658         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 659         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 660         unsigned int mbuf_size, mbufs_per_pkt;
 661         unsigned int nb_sop_desc, nb_data_desc;
 662         uint16_t min_sop, max_sop, min_data, max_data;
 663         uint32_t max_rx_pkt_len;
 664
 665         rq_sop->is_sop = 1;
 666         rq_sop->data_queue_idx = data_queue_idx;
 667         rq_data->is_sop = 0;
 668         rq_data->data_queue_idx = 0;
 669         rq_sop->socket_id = socket_id;
 670         rq_sop->mp = mp;
 671         rq_data->socket_id = socket_id;
 672         rq_data->mp = mp;
 673         rq_sop->in_use = 1;
 674         rq_sop->rx_free_thresh = free_thresh;
 675         rq_data->rx_free_thresh = free_thresh;
 676         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 677                   free_thresh);
 678
 679         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 680                                RTE_PKTMBUF_HEADROOM);
 681         /* max_rx_pkt_len includes the ethernet header and CRC. */
 682         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 683
 684         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 685             DEV_RX_OFFLOAD_SCATTER) {
 686                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 687                 /* ceil((max pkt len)/mbuf_size) */
 688                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 689         } else {
 690                 dev_info(enic, "Scatter rx mode disabled\n");
 691                 mbufs_per_pkt = 1;
 692                 if (max_rx_pkt_len > mbuf_size) {
 693                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 694                                     " larger than the mbuf size (%u), and"
 695                                     " scatter is disabled. Larger packets will"
 696                                     " be truncated.\n",
 697                                     max_rx_pkt_len, mbuf_size);
 698                 }
 699         }
 700
 701         if (mbufs_per_pkt > 1) {
 702                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 703                 rq_sop->data_queue_enable = 1;
 704                 rq_data->in_use = 1;
 705                 /*
 706                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 707                  * receives packet sizes up to the "max" MTU.
 708                  * If not using scatter, we can achieve the effect of dropping
 709                  * larger packets by reducing the size of posted buffers.
 710                  * See enic_alloc_rx_queue_mbufs().
 711                  */
 712                 if (max_rx_pkt_len <
 713                     enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu)) {
 714                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 715                                     " when scatter rx mode is in use.\n");
 716                 }
 717         } else {
 718                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 719                          queue_idx);
 720                 rq_sop->data_queue_enable = 0;
 721                 rq_data->in_use = 0;
 722         }
 723
 724         /* number of descriptors have to be a multiple of 32 */
 725         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
 726         nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
 727
 728         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 729         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 730
 731         if (mbufs_per_pkt > 1) {
 732                 min_sop = 64;
 733                 max_sop = ((enic->config.rq_desc_count /
 734                             (mbufs_per_pkt - 1)) & ~0x1F);
 735                 min_data = min_sop * (mbufs_per_pkt - 1);
 736                 max_data = enic->config.rq_desc_count;
 737         } else {
 738                 min_sop = 64;
 739                 max_sop = enic->config.rq_desc_count;
 740                 min_data = 0;
 741                 max_data = 0;
 742         }
 743
 744         if (nb_desc < (min_sop + min_data)) {
 745                 dev_warning(enic,
 746                             "Number of rx descs too low, adjusting to minimum\n");
 747                 nb_sop_desc = min_sop;
 748                 nb_data_desc = min_data;
 749         } else if (nb_desc > (max_sop + max_data)) {
 750                 dev_warning(enic,
 751                             "Number of rx_descs too high, adjusting to maximum\n");
 752                 nb_sop_desc = max_sop;
 753                 nb_data_desc = max_data;
 754         }
 755         if (mbufs_per_pkt > 1) {
 756                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 757                          " rx descriptor range is %u to %u\n",
 758                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 759                          max_sop + max_data);
 760         }
 761         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 762                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 763
 764         /* Allocate sop queue resources */
 765         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 766                 nb_sop_desc, sizeof(struct rq_enet_desc));
 767         if (rc) {
 768                 dev_err(enic, "error in allocation of sop rq\n");
 769                 goto err_exit;
 770         }
 771         nb_sop_desc = rq_sop->ring.desc_count;
 772
 773         if (rq_data->in_use) {
 774                 /* Allocate data queue resources */
 775                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 776                                    nb_data_desc,
 777                                    sizeof(struct rq_enet_desc));
 778                 if (rc) {
 779                         dev_err(enic, "error in allocation of data rq\n");
 780                         goto err_free_rq_sop;
 781                 }
 782                 nb_data_desc = rq_data->ring.desc_count;
 783         }
 784         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 785                            socket_id, nb_sop_desc + nb_data_desc,
 786                            sizeof(struct cq_enet_rq_desc));
 787         if (rc) {
 788                 dev_err(enic, "error in allocation of cq for rq\n");
 789                 goto err_free_rq_data;
 790         }
 791
 792         /* Allocate the mbuf rings */
 793         rq_sop->mbuf_ring = (struct rte_mbuf **)
 794                 rte_zmalloc_socket("rq->mbuf_ring",
 795                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 796                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 797         if (rq_sop->mbuf_ring == NULL)
 798                 goto err_free_cq;
 799
 800         if (rq_data->in_use) {
 801                 rq_data->mbuf_ring = (struct rte_mbuf **)
 802                         rte_zmalloc_socket("rq->mbuf_ring",
 803                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 804                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 805                 if (rq_data->mbuf_ring == NULL)
 806                         goto err_free_sop_mbuf;
 807         }
 808
 809         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 810
 811         return 0;
 812
 813 err_free_sop_mbuf:
 814         rte_free(rq_sop->mbuf_ring);
 815 err_free_cq:
 816         /* cleanup on error */
 817         vnic_cq_free(&enic->cq[queue_idx]);
 818 err_free_rq_data:
 819         if (rq_data->in_use)
 820                 vnic_rq_free(rq_data);
 821 err_free_rq_sop:
 822         vnic_rq_free(rq_sop);
 823 err_exit:
 824         return -ENOMEM;
 825 }
 826
 827 void enic_free_wq(void *txq)
 828 {
 829         struct vnic_wq *wq;
 830         struct enic *enic;
 831
 832         if (txq == NULL)
 833                 return;
 834
 835         wq = (struct vnic_wq *)txq;
 836         enic = vnic_dev_priv(wq->vdev);
 837         rte_memzone_free(wq->cqmsg_rz);
 838         vnic_wq_free(wq);
 839         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 840 }
 841
 842 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 843         unsigned int socket_id, uint16_t nb_desc)
 844 {
 845         int err;
 846         struct vnic_wq *wq = &enic->wq[queue_idx];
 847         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 848         char name[NAME_MAX];
 849         static int instance;
 850
 851         wq->socket_id = socket_id;
 852         if (nb_desc) {
 853                 if (nb_desc > enic->config.wq_desc_count) {
 854                         dev_warning(enic,
 855                                 "WQ %d - number of tx desc in cmd line (%d)"\
 856                                 "is greater than that in the UCSM/CIMC adapter"\
 857                                 "policy.  Applying the value in the adapter "\
 858                                 "policy (%d)\n",
 859                                 queue_idx, nb_desc, enic->config.wq_desc_count);
 860                 } else if (nb_desc != enic->config.wq_desc_count) {
 861                         enic->config.wq_desc_count = nb_desc;
 862                         dev_info(enic,
 863                                 "TX Queues - effective number of descs:%d\n",
 864                                 nb_desc);
 865                 }
 866         }
 867
 868         /* Allocate queue resources */
 869         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
 870                 enic->config.wq_desc_count,
 871                 sizeof(struct wq_enet_desc));
 872         if (err) {
 873                 dev_err(enic, "error in allocation of wq\n");
 874                 return err;
 875         }
 876
 877         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
 878                 socket_id, enic->config.wq_desc_count,
 879                 sizeof(struct cq_enet_wq_desc));
 880         if (err) {
 881                 vnic_wq_free(wq);
 882                 dev_err(enic, "error in allocation of cq for wq\n");
 883         }
 884
 885         /* setup up CQ message */
 886         snprintf((char *)name, sizeof(name),
 887                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
 888                 instance++);
 889
 890         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
 891                                                    sizeof(uint32_t),
 892                                                    SOCKET_ID_ANY, 0,
 893                                                    ENIC_ALIGN);
 894         if (!wq->cqmsg_rz)
 895                 return -ENOMEM;
 896
 897         return err;
 898 }
 899
 900 int enic_disable(struct enic *enic)
 901 {
 902         unsigned int i;
 903         int err;
 904
 905         for (i = 0; i < enic->intr_count; i++) {
 906                 vnic_intr_mask(&enic->intr[i]);
 907                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
 908         }
 909         enic_rxq_intr_deinit(enic);
 910         rte_intr_disable(&enic->pdev->intr_handle);
 911         rte_intr_callback_unregister(&enic->pdev->intr_handle,
 912                                      enic_intr_handler,
 913                                      (void *)enic->rte_dev);
 914
 915         vnic_dev_disable(enic->vdev);
 916
 917         enic_clsf_destroy(enic);
 918
 919         if (!enic_is_sriov_vf(enic))
 920                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
 921
 922         for (i = 0; i < enic->wq_count; i++) {
 923                 err = vnic_wq_disable(&enic->wq[i]);
 924                 if (err)
 925                         return err;
 926         }
 927         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
 928                 if (enic->rq[i].in_use) {
 929                         err = vnic_rq_disable(&enic->rq[i]);
 930                         if (err)
 931                                 return err;
 932                 }
 933         }
 934
 935         /* If we were using interrupts, set the interrupt vector to -1
 936          * to disable interrupts.  We are not disabling link notifcations,
 937          * though, as we want the polling of link status to continue working.
 938          */
 939         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
 940                 vnic_dev_notify_set(enic->vdev, -1);
 941
 942         vnic_dev_set_reset_flag(enic->vdev, 1);
 943
 944         for (i = 0; i < enic->wq_count; i++)
 945                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
 946
 947         for (i = 0; i < enic_vnic_rq_count(enic); i++)
 948                 if (enic->rq[i].in_use)
 949                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
 950         for (i = 0; i < enic->cq_count; i++)
 951                 vnic_cq_clean(&enic->cq[i]);
 952         for (i = 0; i < enic->intr_count; i++)
 953                 vnic_intr_clean(&enic->intr[i]);
 954
 955         return 0;
 956 }
 957
 958 static int enic_dev_wait(struct vnic_dev *vdev,
 959         int (*start)(struct vnic_dev *, int),
 960         int (*finished)(struct vnic_dev *, int *),
 961         int arg)
 962 {
 963         int done;
 964         int err;
 965         int i;
 966
 967         err = start(vdev, arg);
 968         if (err)
 969                 return err;
 970
 971         /* Wait for func to complete...2 seconds max */
 972         for (i = 0; i < 2000; i++) {
 973                 err = finished(vdev, &done);
 974                 if (err)
 975                         return err;
 976                 if (done)
 977                         return 0;
 978                 usleep(1000);
 979         }
 980         return -ETIMEDOUT;
 981 }
 982
 983 static int enic_dev_open(struct enic *enic)
 984 {
 985         int err;
 986
 987         err = enic_dev_wait(enic->vdev, vnic_dev_open,
 988                 vnic_dev_open_done, 0);
 989         if (err)
 990                 dev_err(enic_get_dev(enic),
 991                         "vNIC device open failed, err %d\n", err);
 992
 993         return err;
 994 }
 995
 996 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
 997 {
 998         dma_addr_t rss_key_buf_pa;
 999         union vnic_rss_key *rss_key_buf_va = NULL;
1000         int err, i;
1001         u8 name[NAME_MAX];
1002
1003         RTE_ASSERT(user_key != NULL);
1004         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1005         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1006                 &rss_key_buf_pa, name);
1007         if (!rss_key_buf_va)
1008                 return -ENOMEM;
1009
1010         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1011                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1012
1013         err = enic_set_rss_key(enic,
1014                 rss_key_buf_pa,
1015                 sizeof(union vnic_rss_key));
1016
1017         /* Save for later queries */
1018         if (!err) {
1019                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1020                            sizeof(union vnic_rss_key));
1021         }
1022         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1023                 rss_key_buf_va, rss_key_buf_pa);
1024
1025         return err;
1026 }
1027
1028 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1029 {
1030         dma_addr_t rss_cpu_buf_pa;
1031         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1032         int err;
1033         u8 name[NAME_MAX];
1034
1035         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1036         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1037                 &rss_cpu_buf_pa, name);
1038         if (!rss_cpu_buf_va)
1039                 return -ENOMEM;
1040
1041         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1042
1043         err = enic_set_rss_cpu(enic,
1044                 rss_cpu_buf_pa,
1045                 sizeof(union vnic_rss_cpu));
1046
1047         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1048                 rss_cpu_buf_va, rss_cpu_buf_pa);
1049
1050         /* Save for later queries */
1051         if (!err)
1052                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1053         return err;
1054 }
1055
1056 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1057         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1058 {
1059         const u8 tso_ipid_split_en = 0;
1060         int err;
1061
1062         err = enic_set_nic_cfg(enic,
1063                 rss_default_cpu, rss_hash_type,
1064                 rss_hash_bits, rss_base_cpu,
1065                 rss_enable, tso_ipid_split_en,
1066                 enic->ig_vlan_strip_en);
1067
1068         return err;
1069 }
1070
1071 /* Initialize RSS with defaults, called from dev_configure */
1072 int enic_init_rss_nic_cfg(struct enic *enic)
1073 {
1074         static uint8_t default_rss_key[] = {
1075                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1076                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1077                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1078                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1079         };
1080         struct rte_eth_rss_conf rss_conf;
1081         union vnic_rss_cpu rss_cpu;
1082         int ret, i;
1083
1084         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1085         /*
1086          * If setting key for the first time, and the user gives us none, then
1087          * push the default key to NIC.
1088          */
1089         if (rss_conf.rss_key == NULL) {
1090                 rss_conf.rss_key = default_rss_key;
1091                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1092         }
1093         ret = enic_set_rss_conf(enic, &rss_conf);
1094         if (ret) {
1095                 dev_err(enic, "Failed to configure RSS\n");
1096                 return ret;
1097         }
1098         if (enic->rss_enable) {
1099                 /* If enabling RSS, use the default reta */
1100                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1101                         rss_cpu.cpu[i / 4].b[i % 4] =
1102                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1103                 }
1104                 ret = enic_set_rss_reta(enic, &rss_cpu);
1105                 if (ret)
1106                         dev_err(enic, "Failed to set RSS indirection table\n");
1107         }
1108         return ret;
1109 }
1110
1111 int enic_setup_finish(struct enic *enic)
1112 {
1113         enic_init_soft_stats(enic);
1114
1115         /* Default conf */
1116         vnic_dev_packet_filter(enic->vdev,
1117                 1 /* directed  */,
1118                 1 /* multicast */,
1119                 1 /* broadcast */,
1120                 0 /* promisc   */,
1121                 1 /* allmulti  */);
1122
1123         enic->promisc = 0;
1124         enic->allmulti = 1;
1125
1126         return 0;
1127 }
1128
1129 static int enic_rss_conf_valid(struct enic *enic,
1130                                struct rte_eth_rss_conf *rss_conf)
1131 {
1132         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1133         if (enic->flow_type_rss_offloads == 0)
1134                 return 0;
1135         if (rss_conf->rss_key != NULL &&
1136             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1137                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1138                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1139                 return -EINVAL;
1140         }
1141         if (rss_conf->rss_hf != 0 &&
1142             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1143                 dev_err(enic, "Given rss_hf contains none of the supported"
1144                         " types\n");
1145                 return -EINVAL;
1146         }
1147         return 0;
1148 }
1149
1150 /* Set hash type and key according to rss_conf */
1151 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1152 {
1153         struct rte_eth_dev *eth_dev;
1154         uint64_t rss_hf;
1155         u8 rss_hash_type;
1156         u8 rss_enable;
1157         int ret;
1158
1159         RTE_ASSERT(rss_conf != NULL);
1160         ret = enic_rss_conf_valid(enic, rss_conf);
1161         if (ret) {
1162                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1163                 return ret;
1164         }
1165
1166         eth_dev = enic->rte_dev;
1167         rss_hash_type = 0;
1168         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1169         if (enic->rq_count > 1 &&
1170             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1171             rss_hf != 0) {
1172                 rss_enable = 1;
1173                 if (rss_hf & ETH_RSS_IPV4)
1174                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1175                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1176                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1177                 if (rss_hf & ETH_RSS_IPV6)
1178                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1179                 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1180                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1181                 if (rss_hf & ETH_RSS_IPV6_EX)
1182                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6_EX;
1183                 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
1184                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX;
1185         } else {
1186                 rss_enable = 0;
1187                 rss_hf = 0;
1188         }
1189
1190         /* Set the hash key if provided */
1191         if (rss_enable && rss_conf->rss_key) {
1192                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1193                 if (ret) {
1194                         dev_err(enic, "Failed to set RSS key\n");
1195                         return ret;
1196                 }
1197         }
1198
1199         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1200                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1201                               rss_enable);
1202         if (!ret) {
1203                 enic->rss_hf = rss_hf;
1204                 enic->rss_hash_type = rss_hash_type;
1205                 enic->rss_enable = rss_enable;
1206         }
1207         return 0;
1208 }
1209
1210 int enic_set_vlan_strip(struct enic *enic)
1211 {
1212         /*
1213          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1214          * together. So, re-do niccfg, preserving the current RSS settings.
1215          */
1216         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1217                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1218                                enic->rss_enable);
1219 }
1220
1221 void enic_add_packet_filter(struct enic *enic)
1222 {
1223         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1224         vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1225                 enic->promisc, enic->allmulti);
1226 }
1227
1228 int enic_get_link_status(struct enic *enic)
1229 {
1230         return vnic_dev_link_status(enic->vdev);
1231 }
1232
1233 static void enic_dev_deinit(struct enic *enic)
1234 {
1235         struct rte_eth_dev *eth_dev = enic->rte_dev;
1236
1237         /* stop link status checking */
1238         vnic_dev_notify_unset(enic->vdev);
1239
1240         rte_free(eth_dev->data->mac_addrs);
1241         rte_free(enic->cq);
1242         rte_free(enic->intr);
1243         rte_free(enic->rq);
1244         rte_free(enic->wq);
1245 }
1246
1247
1248 int enic_set_vnic_res(struct enic *enic)
1249 {
1250         struct rte_eth_dev *eth_dev = enic->rte_dev;
1251         int rc = 0;
1252         unsigned int required_rq, required_wq, required_cq, required_intr;
1253
1254         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1255         required_rq = eth_dev->data->nb_rx_queues * 2;
1256         required_wq = eth_dev->data->nb_tx_queues;
1257         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1258         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1259         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1260                 required_intr += eth_dev->data->nb_rx_queues;
1261         }
1262
1263         if (enic->conf_rq_count < required_rq) {
1264                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1265                         eth_dev->data->nb_rx_queues,
1266                         required_rq, enic->conf_rq_count);
1267                 rc = -EINVAL;
1268         }
1269         if (enic->conf_wq_count < required_wq) {
1270                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1271                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1272                 rc = -EINVAL;
1273         }
1274
1275         if (enic->conf_cq_count < required_cq) {
1276                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1277                         required_cq, enic->conf_cq_count);
1278                 rc = -EINVAL;
1279         }
1280         if (enic->conf_intr_count < required_intr) {
1281                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1282                         " interrupts. Required:%u, Configured:%u\n",
1283                         required_intr, enic->conf_intr_count);
1284                 rc = -EINVAL;
1285         }
1286
1287         if (rc == 0) {
1288                 enic->rq_count = eth_dev->data->nb_rx_queues;
1289                 enic->wq_count = eth_dev->data->nb_tx_queues;
1290                 enic->cq_count = enic->rq_count + enic->wq_count;
1291                 enic->intr_count = required_intr;
1292         }
1293
1294         return rc;
1295 }
1296
1297 /* Initialize the completion queue for an RQ */
1298 static int
1299 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1300 {
1301         struct vnic_rq *sop_rq, *data_rq;
1302         unsigned int cq_idx;
1303         int rc = 0;
1304
1305         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1306         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1307         cq_idx = rq_idx;
1308
1309         vnic_cq_clean(&enic->cq[cq_idx]);
1310         vnic_cq_init(&enic->cq[cq_idx],
1311                      0 /* flow_control_enable */,
1312                      1 /* color_enable */,
1313                      0 /* cq_head */,
1314                      0 /* cq_tail */,
1315                      1 /* cq_tail_color */,
1316                      0 /* interrupt_enable */,
1317                      1 /* cq_entry_enable */,
1318                      0 /* cq_message_enable */,
1319                      0 /* interrupt offset */,
1320                      0 /* cq_message_addr */);
1321
1322
1323         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1324                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1325                            sop_rq->ring.desc_count - 1, 1, 0);
1326         if (data_rq->in_use) {
1327                 vnic_rq_init_start(data_rq,
1328                                    enic_cq_rq(enic,
1329                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1330                                    data_rq->ring.desc_count - 1, 1, 0);
1331         }
1332
1333         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1334         if (rc)
1335                 return rc;
1336
1337         if (data_rq->in_use) {
1338                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1339                 if (rc) {
1340                         enic_rxmbuf_queue_release(enic, sop_rq);
1341                         return rc;
1342                 }
1343         }
1344
1345         return 0;
1346 }
1347
1348 /* The Cisco NIC can send and receive packets up to a max packet size
1349  * determined by the NIC type and firmware. There is also an MTU
1350  * configured into the NIC via the CIMC/UCSM management interface
1351  * which can be overridden by this function (up to the max packet size).
1352  * Depending on the network setup, doing so may cause packet drops
1353  * and unexpected behavior.
1354  */
1355 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1356 {
1357         unsigned int rq_idx;
1358         struct vnic_rq *rq;
1359         int rc = 0;
1360         uint16_t old_mtu;       /* previous setting */
1361         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1362         struct rte_eth_dev *eth_dev = enic->rte_dev;
1363
1364         old_mtu = eth_dev->data->mtu;
1365         config_mtu = enic->config.mtu;
1366
1367         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1368                 return -E_RTE_SECONDARY;
1369
1370         if (new_mtu > enic->max_mtu) {
1371                 dev_err(enic,
1372                         "MTU not updated: requested (%u) greater than max (%u)\n",
1373                         new_mtu, enic->max_mtu);
1374                 return -EINVAL;
1375         }
1376         if (new_mtu < ENIC_MIN_MTU) {
1377                 dev_info(enic,
1378                         "MTU not updated: requested (%u) less than min (%u)\n",
1379                         new_mtu, ENIC_MIN_MTU);
1380                 return -EINVAL;
1381         }
1382         if (new_mtu > config_mtu)
1383                 dev_warning(enic,
1384                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1385                         new_mtu, config_mtu);
1386
1387         /* The easy case is when scatter is disabled. However if the MTU
1388          * becomes greater than the mbuf data size, packet drops will ensue.
1389          */
1390         if (!(enic->rte_dev->data->dev_conf.rxmode.offloads &
1391               DEV_RX_OFFLOAD_SCATTER)) {
1392                 eth_dev->data->mtu = new_mtu;
1393                 goto set_mtu_done;
1394         }
1395
1396         /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to
1397          * change Rx scatter mode if necessary for better performance. I.e. if
1398          * MTU was greater than the mbuf size and now it's less, scatter Rx
1399          * doesn't have to be used and vice versa.
1400           */
1401         rte_spinlock_lock(&enic->mtu_lock);
1402
1403         /* Stop traffic on all RQs */
1404         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1405                 rq = &enic->rq[rq_idx];
1406                 if (rq->is_sop && rq->in_use) {
1407                         rc = enic_stop_rq(enic,
1408                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1409                         if (rc) {
1410                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1411                                 goto set_mtu_done;
1412                         }
1413                 }
1414         }
1415
1416         /* replace Rx function with a no-op to avoid getting stale pkts */
1417         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1418         rte_mb();
1419
1420         /* Allow time for threads to exit the real Rx function. */
1421         usleep(100000);
1422
1423         /* now it is safe to reconfigure the RQs */
1424
1425         /* update the mtu */
1426         eth_dev->data->mtu = new_mtu;
1427
1428         /* free and reallocate RQs with the new MTU */
1429         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1430                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1431
1432                 enic_free_rq(rq);
1433                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1434                                    rq->tot_nb_desc, rq->rx_free_thresh);
1435                 if (rc) {
1436                         dev_err(enic,
1437                                 "Fatal MTU alloc error- No traffic will pass\n");
1438                         goto set_mtu_done;
1439                 }
1440
1441                 rc = enic_reinit_rq(enic, rq_idx);
1442                 if (rc) {
1443                         dev_err(enic,
1444                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1445                         goto set_mtu_done;
1446                 }
1447         }
1448
1449         /* put back the real receive function */
1450         rte_mb();
1451         eth_dev->rx_pkt_burst = enic_recv_pkts;
1452         rte_mb();
1453
1454         /* restart Rx traffic */
1455         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1456                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1457                 if (rq->is_sop && rq->in_use)
1458                         enic_start_rq(enic, rq_idx);
1459         }
1460
1461 set_mtu_done:
1462         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1463         rte_spinlock_unlock(&enic->mtu_lock);
1464         return rc;
1465 }
1466
1467 static int enic_dev_init(struct enic *enic)
1468 {
1469         int err;
1470         struct rte_eth_dev *eth_dev = enic->rte_dev;
1471
1472         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1473
1474         /* Get vNIC configuration
1475         */
1476         err = enic_get_vnic_config(enic);
1477         if (err) {
1478                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1479                 return err;
1480         }
1481
1482         /* Get available resource counts */
1483         enic_get_res_counts(enic);
1484         if (enic->conf_rq_count == 1) {
1485                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1486                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1487                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1488                 return -EINVAL;
1489         }
1490         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1491         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1492                                enic->conf_cq_count, 8);
1493         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1494                                  enic->conf_intr_count, 8);
1495         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1496                                enic->conf_rq_count, 8);
1497         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1498                                enic->conf_wq_count, 8);
1499         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1500                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1501                 return -1;
1502         }
1503         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1504                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1505                 return -1;
1506         }
1507         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1508                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1509                 return -1;
1510         }
1511         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1512                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1513                 return -1;
1514         }
1515
1516         /* Get the supported filters */
1517         enic_fdir_info(enic);
1518
1519         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1520                                                 * ENIC_MAX_MAC_ADDR, 0);
1521         if (!eth_dev->data->mac_addrs) {
1522                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1523                 return -1;
1524         }
1525         ether_addr_copy((struct ether_addr *) enic->mac_addr,
1526                         eth_dev->data->mac_addrs);
1527
1528         vnic_dev_set_reset_flag(enic->vdev, 0);
1529
1530         LIST_INIT(&enic->flows);
1531         rte_spinlock_init(&enic->flows_lock);
1532
1533         /* set up link status checking */
1534         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1535
1536         return 0;
1537
1538 }
1539
1540 int enic_probe(struct enic *enic)
1541 {
1542         struct rte_pci_device *pdev = enic->pdev;
1543         int err = -1;
1544
1545         dev_debug(enic, " Initializing ENIC PMD\n");
1546
1547         /* if this is a secondary process the hardware is already initialized */
1548         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1549                 return 0;
1550
1551         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1552         enic->bar0.len = pdev->mem_resource[0].len;
1553
1554         /* Register vNIC device */
1555         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1556         if (!enic->vdev) {
1557                 dev_err(enic, "vNIC registration failed, aborting\n");
1558                 goto err_out;
1559         }
1560
1561         LIST_INIT(&enic->memzone_list);
1562         rte_spinlock_init(&enic->memzone_list_lock);
1563
1564         vnic_register_cbacks(enic->vdev,
1565                 enic_alloc_consistent,
1566                 enic_free_consistent);
1567
1568         /*
1569          * Allocate the consistent memory for stats upfront so both primary and
1570          * secondary processes can dump stats.
1571          */
1572         err = vnic_dev_alloc_stats_mem(enic->vdev);
1573         if (err) {
1574                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1575                 goto err_out_unregister;
1576         }
1577         /* Issue device open to get device in known state */
1578         err = enic_dev_open(enic);
1579         if (err) {
1580                 dev_err(enic, "vNIC dev open failed, aborting\n");
1581                 goto err_out_unregister;
1582         }
1583
1584         /* Set ingress vlan rewrite mode before vnic initialization */
1585         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1586                 IG_VLAN_REWRITE_MODE_PASS_THRU);
1587         if (err) {
1588                 dev_err(enic,
1589                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1590                 goto err_out_dev_close;
1591         }
1592
1593         /* Issue device init to initialize the vnic-to-switch link.
1594          * We'll start with carrier off and wait for link UP
1595          * notification later to turn on carrier.  We don't need
1596          * to wait here for the vnic-to-switch link initialization
1597          * to complete; link UP notification is the indication that
1598          * the process is complete.
1599          */
1600
1601         err = vnic_dev_init(enic->vdev, 0);
1602         if (err) {
1603                 dev_err(enic, "vNIC dev init failed, aborting\n");
1604                 goto err_out_dev_close;
1605         }
1606
1607         err = enic_dev_init(enic);
1608         if (err) {
1609                 dev_err(enic, "Device initialization failed, aborting\n");
1610                 goto err_out_dev_close;
1611         }
1612
1613         return 0;
1614
1615 err_out_dev_close:
1616         vnic_dev_close(enic->vdev);
1617 err_out_unregister:
1618         vnic_dev_unregister(enic->vdev);
1619 err_out:
1620         return err;
1621 }
1622
1623 void enic_remove(struct enic *enic)
1624 {
1625         enic_dev_deinit(enic);
1626         vnic_dev_close(enic->vdev);
1627         vnic_dev_unregister(enic->vdev);
1628 }