drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11 #include <libgen.h>
  12
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_ethdev_driver.h>
  20
  21 #include "enic_compat.h"
  22 #include "enic.h"
  23 #include "wq_enet_desc.h"
  24 #include "rq_enet_desc.h"
  25 #include "cq_enet_desc.h"
  26 #include "vnic_enet.h"
  27 #include "vnic_dev.h"
  28 #include "vnic_wq.h"
  29 #include "vnic_rq.h"
  30 #include "vnic_cq.h"
  31 #include "vnic_intr.h"
  32 #include "vnic_nic.h"
  33
  34 static inline int enic_is_sriov_vf(struct enic *enic)
  35 {
  36         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  37 }
  38
  39 static int is_zero_addr(uint8_t *addr)
  40 {
  41         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  42 }
  43
  44 static int is_mcast_addr(uint8_t *addr)
  45 {
  46         return addr[0] & 1;
  47 }
  48
  49 static int is_eth_addr_valid(uint8_t *addr)
  50 {
  51         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  52 }
  53
  54 static void
  55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  56 {
  57         uint16_t i;
  58
  59         if (!rq || !rq->mbuf_ring) {
  60                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  61                 return;
  62         }
  63
  64         for (i = 0; i < rq->ring.desc_count; i++) {
  65                 if (rq->mbuf_ring[i]) {
  66                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  67                         rq->mbuf_ring[i] = NULL;
  68                 }
  69         }
  70 }
  71
  72 static void enic_free_wq_buf(struct vnic_wq_buf *buf)
  73 {
  74         struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
  75
  76         rte_pktmbuf_free_seg(mbuf);
  77         buf->mb = NULL;
  78 }
  79
  80 static void enic_log_q_error(struct enic *enic)
  81 {
  82         unsigned int i;
  83         u32 error_status;
  84
  85         for (i = 0; i < enic->wq_count; i++) {
  86                 error_status = vnic_wq_error_status(&enic->wq[i]);
  87                 if (error_status)
  88                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  89                                 error_status);
  90         }
  91
  92         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  93                 if (!enic->rq[i].in_use)
  94                         continue;
  95                 error_status = vnic_rq_error_status(&enic->rq[i]);
  96                 if (error_status)
  97                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  98                                 error_status);
  99         }
 100 }
 101
 102 static void enic_clear_soft_stats(struct enic *enic)
 103 {
 104         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 105         rte_atomic64_clear(&soft_stats->rx_nombuf);
 106         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 107         rte_atomic64_clear(&soft_stats->tx_oversized);
 108 }
 109
 110 static void enic_init_soft_stats(struct enic *enic)
 111 {
 112         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 113         rte_atomic64_init(&soft_stats->rx_nombuf);
 114         rte_atomic64_init(&soft_stats->rx_packet_errors);
 115         rte_atomic64_init(&soft_stats->tx_oversized);
 116         enic_clear_soft_stats(enic);
 117 }
 118
 119 void enic_dev_stats_clear(struct enic *enic)
 120 {
 121         if (vnic_dev_stats_clear(enic->vdev))
 122                 dev_err(enic, "Error in clearing stats\n");
 123         enic_clear_soft_stats(enic);
 124 }
 125
 126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 127 {
 128         struct vnic_stats *stats;
 129         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 130         int64_t rx_truncated;
 131         uint64_t rx_packet_errors;
 132         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 133
 134         if (ret) {
 135                 dev_err(enic, "Error in getting stats\n");
 136                 return ret;
 137         }
 138
 139         /* The number of truncated packets can only be calculated by
 140          * subtracting a hardware counter from error packets received by
 141          * the driver. Note: this causes transient inaccuracies in the
 142          * ipackets count. Also, the length of truncated packets are
 143          * counted in ibytes even though truncated packets are dropped
 144          * which can make ibytes be slightly higher than it should be.
 145          */
 146         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 147         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 148
 149         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 150         r_stats->opackets = stats->tx.tx_frames_ok;
 151
 152         r_stats->ibytes = stats->rx.rx_bytes_ok;
 153         r_stats->obytes = stats->tx.tx_bytes_ok;
 154
 155         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 156         r_stats->oerrors = stats->tx.tx_errors
 157                            + rte_atomic64_read(&soft_stats->tx_oversized);
 158
 159         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 160
 161         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 162         return 0;
 163 }
 164
 165 int enic_del_mac_address(struct enic *enic, int mac_index)
 166 {
 167         struct rte_eth_dev *eth_dev = enic->rte_dev;
 168         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 169
 170         return vnic_dev_del_addr(enic->vdev, mac_addr);
 171 }
 172
 173 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 174 {
 175         int err;
 176
 177         if (!is_eth_addr_valid(mac_addr)) {
 178                 dev_err(enic, "invalid mac address\n");
 179                 return -EINVAL;
 180         }
 181
 182         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 183         if (err)
 184                 dev_err(enic, "add mac addr failed\n");
 185         return err;
 186 }
 187
 188 static void
 189 enic_free_rq_buf(struct rte_mbuf **mbuf)
 190 {
 191         if (*mbuf == NULL)
 192                 return;
 193
 194         rte_pktmbuf_free(*mbuf);
 195         *mbuf = NULL;
 196 }
 197
 198 void enic_init_vnic_resources(struct enic *enic)
 199 {
 200         unsigned int error_interrupt_enable = 1;
 201         unsigned int error_interrupt_offset = 0;
 202         unsigned int rxq_interrupt_enable = 0;
 203         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 204         unsigned int index = 0;
 205         unsigned int cq_idx;
 206         struct vnic_rq *data_rq;
 207
 208         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 209                 rxq_interrupt_enable = 1;
 210
 211         for (index = 0; index < enic->rq_count; index++) {
 212                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 213
 214                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 215                         cq_idx,
 216                         error_interrupt_enable,
 217                         error_interrupt_offset);
 218
 219                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 220                 if (data_rq->in_use)
 221                         vnic_rq_init(data_rq,
 222                                      cq_idx,
 223                                      error_interrupt_enable,
 224                                      error_interrupt_offset);
 225
 226                 vnic_cq_init(&enic->cq[cq_idx],
 227                         0 /* flow_control_enable */,
 228                         1 /* color_enable */,
 229                         0 /* cq_head */,
 230                         0 /* cq_tail */,
 231                         1 /* cq_tail_color */,
 232                         rxq_interrupt_enable,
 233                         1 /* cq_entry_enable */,
 234                         0 /* cq_message_enable */,
 235                         rxq_interrupt_offset,
 236                         0 /* cq_message_addr */);
 237                 if (rxq_interrupt_enable)
 238                         rxq_interrupt_offset++;
 239         }
 240
 241         for (index = 0; index < enic->wq_count; index++) {
 242                 vnic_wq_init(&enic->wq[index],
 243                         enic_cq_wq(enic, index),
 244                         error_interrupt_enable,
 245                         error_interrupt_offset);
 246                 /* Compute unsupported ol flags for enic_prep_pkts() */
 247                 enic->wq[index].tx_offload_notsup_mask =
 248                         PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 249
 250                 cq_idx = enic_cq_wq(enic, index);
 251                 vnic_cq_init(&enic->cq[cq_idx],
 252                         0 /* flow_control_enable */,
 253                         1 /* color_enable */,
 254                         0 /* cq_head */,
 255                         0 /* cq_tail */,
 256                         1 /* cq_tail_color */,
 257                         0 /* interrupt_enable */,
 258                         0 /* cq_entry_enable */,
 259                         1 /* cq_message_enable */,
 260                         0 /* interrupt offset */,
 261                         (u64)enic->wq[index].cqmsg_rz->iova);
 262         }
 263
 264         for (index = 0; index < enic->intr_count; index++) {
 265                 vnic_intr_init(&enic->intr[index],
 266                                enic->config.intr_timer_usec,
 267                                enic->config.intr_timer_type,
 268                                /*mask_on_assertion*/1);
 269         }
 270 }
 271
 272
 273 static int
 274 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 275 {
 276         struct rte_mbuf *mb;
 277         struct rq_enet_desc *rqd = rq->ring.descs;
 278         unsigned i;
 279         dma_addr_t dma_addr;
 280         uint32_t max_rx_pkt_len;
 281         uint16_t rq_buf_len;
 282
 283         if (!rq->in_use)
 284                 return 0;
 285
 286         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 287                   rq->ring.desc_count);
 288
 289         /*
 290          * If *not* using scatter and the mbuf size is smaller than the
 291          * requested max packet size (max_rx_pkt_len), then reduce the
 292          * posted buffer size to max_rx_pkt_len. HW still receives packets
 293          * larger than max_rx_pkt_len, but they will be truncated, which we
 294          * drop in the rx handler. Not ideal, but better than returning
 295          * large packets when the user is not expecting them.
 296          */
 297         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 298         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 299         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 300                 rq_buf_len = max_rx_pkt_len;
 301         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 302                 mb = rte_mbuf_raw_alloc(rq->mp);
 303                 if (mb == NULL) {
 304                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 305                         (unsigned)rq->index);
 306                         return -ENOMEM;
 307                 }
 308
 309                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 310                 dma_addr = (dma_addr_t)(mb->buf_iova
 311                            + RTE_PKTMBUF_HEADROOM);
 312                 rq_enet_desc_enc(rqd, dma_addr,
 313                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 314                                 : RQ_ENET_TYPE_NOT_SOP),
 315                                 rq_buf_len);
 316                 rq->mbuf_ring[i] = mb;
 317         }
 318
 319         /* make sure all prior writes are complete before doing the PIO write */
 320         rte_rmb();
 321
 322         /* Post all but the last buffer to VIC. */
 323         rq->posted_index = rq->ring.desc_count - 1;
 324
 325         rq->rx_nb_hold = 0;
 326
 327         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 328                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 329         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 330         iowrite32(0, &rq->ctrl->fetch_index);
 331         rte_rmb();
 332
 333         return 0;
 334
 335 }
 336
 337 static void *
 338 enic_alloc_consistent(void *priv, size_t size,
 339         dma_addr_t *dma_handle, u8 *name)
 340 {
 341         void *vaddr;
 342         const struct rte_memzone *rz;
 343         *dma_handle = 0;
 344         struct enic *enic = (struct enic *)priv;
 345         struct enic_memzone_entry *mze;
 346
 347         rz = rte_memzone_reserve_aligned((const char *)name, size,
 348                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 349         if (!rz) {
 350                 pr_err("%s : Failed to allocate memory requested for %s\n",
 351                         __func__, name);
 352                 return NULL;
 353         }
 354
 355         vaddr = rz->addr;
 356         *dma_handle = (dma_addr_t)rz->iova;
 357
 358         mze = rte_malloc("enic memzone entry",
 359                          sizeof(struct enic_memzone_entry), 0);
 360
 361         if (!mze) {
 362                 pr_err("%s : Failed to allocate memory for memzone list\n",
 363                        __func__);
 364                 rte_memzone_free(rz);
 365                 return NULL;
 366         }
 367
 368         mze->rz = rz;
 369
 370         rte_spinlock_lock(&enic->memzone_list_lock);
 371         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 372         rte_spinlock_unlock(&enic->memzone_list_lock);
 373
 374         return vaddr;
 375 }
 376
 377 static void
 378 enic_free_consistent(void *priv,
 379                      __rte_unused size_t size,
 380                      void *vaddr,
 381                      dma_addr_t dma_handle)
 382 {
 383         struct enic_memzone_entry *mze;
 384         struct enic *enic = (struct enic *)priv;
 385
 386         rte_spinlock_lock(&enic->memzone_list_lock);
 387         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 388                 if (mze->rz->addr == vaddr &&
 389                     mze->rz->iova == dma_handle)
 390                         break;
 391         }
 392         if (mze == NULL) {
 393                 rte_spinlock_unlock(&enic->memzone_list_lock);
 394                 dev_warning(enic,
 395                             "Tried to free memory, but couldn't find it in the memzone list\n");
 396                 return;
 397         }
 398         LIST_REMOVE(mze, entries);
 399         rte_spinlock_unlock(&enic->memzone_list_lock);
 400         rte_memzone_free(mze->rz);
 401         rte_free(mze);
 402 }
 403
 404 int enic_link_update(struct enic *enic)
 405 {
 406         struct rte_eth_dev *eth_dev = enic->rte_dev;
 407         struct rte_eth_link link;
 408
 409         memset(&link, 0, sizeof(link));
 410         link.link_status = enic_get_link_status(enic);
 411         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 412         link.link_speed = vnic_dev_port_speed(enic->vdev);
 413
 414         return rte_eth_linkstatus_set(eth_dev, &link);
 415 }
 416
 417 static void
 418 enic_intr_handler(void *arg)
 419 {
 420         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 421         struct enic *enic = pmd_priv(dev);
 422
 423         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 424
 425         enic_link_update(enic);
 426         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 427         enic_log_q_error(enic);
 428 }
 429
 430 static int enic_rxq_intr_init(struct enic *enic)
 431 {
 432         struct rte_intr_handle *intr_handle;
 433         uint32_t rxq_intr_count, i;
 434         int err;
 435
 436         intr_handle = enic->rte_dev->intr_handle;
 437         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 438                 return 0;
 439         /*
 440          * Rx queue interrupts only work when we have MSI-X interrupts,
 441          * one per queue. Sharing one interrupt is technically
 442          * possible with VIC, but it is not worth the complications it brings.
 443          */
 444         if (!rte_intr_cap_multiple(intr_handle)) {
 445                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 446                         " (vfio-pci driver)\n");
 447                 return -ENOTSUP;
 448         }
 449         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 450         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 451         if (err) {
 452                 dev_err(enic, "Failed to enable event fds for Rx queue"
 453                         " interrupts\n");
 454                 return err;
 455         }
 456         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 457                                             rxq_intr_count * sizeof(int), 0);
 458         if (intr_handle->intr_vec == NULL) {
 459                 dev_err(enic, "Failed to allocate intr_vec\n");
 460                 return -ENOMEM;
 461         }
 462         for (i = 0; i < rxq_intr_count; i++)
 463                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 464         return 0;
 465 }
 466
 467 static void enic_rxq_intr_deinit(struct enic *enic)
 468 {
 469         struct rte_intr_handle *intr_handle;
 470
 471         intr_handle = enic->rte_dev->intr_handle;
 472         rte_intr_efd_disable(intr_handle);
 473         if (intr_handle->intr_vec != NULL) {
 474                 rte_free(intr_handle->intr_vec);
 475                 intr_handle->intr_vec = NULL;
 476         }
 477 }
 478
 479 int enic_enable(struct enic *enic)
 480 {
 481         unsigned int index;
 482         int err;
 483         struct rte_eth_dev *eth_dev = enic->rte_dev;
 484
 485         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 486         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 487
 488         /* vnic notification of link status has already been turned on in
 489          * enic_dev_init() which is called during probe time.  Here we are
 490          * just turning on interrupt vector 0 if needed.
 491          */
 492         if (eth_dev->data->dev_conf.intr_conf.lsc)
 493                 vnic_dev_notify_set(enic->vdev, 0);
 494
 495         err = enic_rxq_intr_init(enic);
 496         if (err)
 497                 return err;
 498         if (enic_clsf_init(enic))
 499                 dev_warning(enic, "Init of hash table for clsf failed."\
 500                         "Flow director feature will not work\n");
 501
 502         for (index = 0; index < enic->rq_count; index++) {
 503                 err = enic_alloc_rx_queue_mbufs(enic,
 504                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 505                 if (err) {
 506                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 507                         return err;
 508                 }
 509                 err = enic_alloc_rx_queue_mbufs(enic,
 510                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 511                 if (err) {
 512                         /* release the allocated mbufs for the sop rq*/
 513                         enic_rxmbuf_queue_release(enic,
 514                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 515
 516                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 517                         return err;
 518                 }
 519         }
 520
 521         for (index = 0; index < enic->wq_count; index++)
 522                 enic_start_wq(enic, index);
 523         for (index = 0; index < enic->rq_count; index++)
 524                 enic_start_rq(enic, index);
 525
 526         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 527
 528         vnic_dev_enable_wait(enic->vdev);
 529
 530         /* Register and enable error interrupt */
 531         rte_intr_callback_register(&(enic->pdev->intr_handle),
 532                 enic_intr_handler, (void *)enic->rte_dev);
 533
 534         rte_intr_enable(&(enic->pdev->intr_handle));
 535         /* Unmask LSC interrupt */
 536         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 537
 538         return 0;
 539 }
 540
 541 int enic_alloc_intr_resources(struct enic *enic)
 542 {
 543         int err;
 544         unsigned int i;
 545
 546         dev_info(enic, "vNIC resources used:  "\
 547                 "wq %d rq %d cq %d intr %d\n",
 548                 enic->wq_count, enic_vnic_rq_count(enic),
 549                 enic->cq_count, enic->intr_count);
 550
 551         for (i = 0; i < enic->intr_count; i++) {
 552                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 553                 if (err) {
 554                         enic_free_vnic_resources(enic);
 555                         return err;
 556                 }
 557         }
 558         return 0;
 559 }
 560
 561 void enic_free_rq(void *rxq)
 562 {
 563         struct vnic_rq *rq_sop, *rq_data;
 564         struct enic *enic;
 565
 566         if (rxq == NULL)
 567                 return;
 568
 569         rq_sop = (struct vnic_rq *)rxq;
 570         enic = vnic_dev_priv(rq_sop->vdev);
 571         rq_data = &enic->rq[rq_sop->data_queue_idx];
 572
 573         enic_rxmbuf_queue_release(enic, rq_sop);
 574         if (rq_data->in_use)
 575                 enic_rxmbuf_queue_release(enic, rq_data);
 576
 577         rte_free(rq_sop->mbuf_ring);
 578         if (rq_data->in_use)
 579                 rte_free(rq_data->mbuf_ring);
 580
 581         rq_sop->mbuf_ring = NULL;
 582         rq_data->mbuf_ring = NULL;
 583
 584         vnic_rq_free(rq_sop);
 585         if (rq_data->in_use)
 586                 vnic_rq_free(rq_data);
 587
 588         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 589
 590         rq_sop->in_use = 0;
 591         rq_data->in_use = 0;
 592 }
 593
 594 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 595 {
 596         struct rte_eth_dev *eth_dev = enic->rte_dev;
 597         vnic_wq_enable(&enic->wq[queue_idx]);
 598         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 599 }
 600
 601 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 602 {
 603         struct rte_eth_dev *eth_dev = enic->rte_dev;
 604         int ret;
 605
 606         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 607         if (ret)
 608                 return ret;
 609
 610         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 611         return 0;
 612 }
 613
 614 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 615 {
 616         struct vnic_rq *rq_sop;
 617         struct vnic_rq *rq_data;
 618         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 619         rq_data = &enic->rq[rq_sop->data_queue_idx];
 620         struct rte_eth_dev *eth_dev = enic->rte_dev;
 621
 622         if (rq_data->in_use)
 623                 vnic_rq_enable(rq_data);
 624         rte_mb();
 625         vnic_rq_enable(rq_sop);
 626         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 627 }
 628
 629 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 630 {
 631         int ret1 = 0, ret2 = 0;
 632         struct rte_eth_dev *eth_dev = enic->rte_dev;
 633         struct vnic_rq *rq_sop;
 634         struct vnic_rq *rq_data;
 635         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 636         rq_data = &enic->rq[rq_sop->data_queue_idx];
 637
 638         ret2 = vnic_rq_disable(rq_sop);
 639         rte_mb();
 640         if (rq_data->in_use)
 641                 ret1 = vnic_rq_disable(rq_data);
 642
 643         if (ret2)
 644                 return ret2;
 645         else if (ret1)
 646                 return ret1;
 647
 648         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 649         return 0;
 650 }
 651
 652 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 653         unsigned int socket_id, struct rte_mempool *mp,
 654         uint16_t nb_desc, uint16_t free_thresh)
 655 {
 656         int rc;
 657         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 658         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 659         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 660         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 661         unsigned int mbuf_size, mbufs_per_pkt;
 662         unsigned int nb_sop_desc, nb_data_desc;
 663         uint16_t min_sop, max_sop, min_data, max_data;
 664         uint32_t max_rx_pkt_len;
 665
 666         rq_sop->is_sop = 1;
 667         rq_sop->data_queue_idx = data_queue_idx;
 668         rq_data->is_sop = 0;
 669         rq_data->data_queue_idx = 0;
 670         rq_sop->socket_id = socket_id;
 671         rq_sop->mp = mp;
 672         rq_data->socket_id = socket_id;
 673         rq_data->mp = mp;
 674         rq_sop->in_use = 1;
 675         rq_sop->rx_free_thresh = free_thresh;
 676         rq_data->rx_free_thresh = free_thresh;
 677         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 678                   free_thresh);
 679
 680         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 681                                RTE_PKTMBUF_HEADROOM);
 682         /* max_rx_pkt_len includes the ethernet header and CRC. */
 683         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 684
 685         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 686             DEV_RX_OFFLOAD_SCATTER) {
 687                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 688                 /* ceil((max pkt len)/mbuf_size) */
 689                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 690         } else {
 691                 dev_info(enic, "Scatter rx mode disabled\n");
 692                 mbufs_per_pkt = 1;
 693                 if (max_rx_pkt_len > mbuf_size) {
 694                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 695                                     " larger than the mbuf size (%u), and"
 696                                     " scatter is disabled. Larger packets will"
 697                                     " be truncated.\n",
 698                                     max_rx_pkt_len, mbuf_size);
 699                 }
 700         }
 701
 702         if (mbufs_per_pkt > 1) {
 703                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 704                 rq_sop->data_queue_enable = 1;
 705                 rq_data->in_use = 1;
 706                 /*
 707                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 708                  * receives packet sizes up to the "max" MTU.
 709                  * If not using scatter, we can achieve the effect of dropping
 710                  * larger packets by reducing the size of posted buffers.
 711                  * See enic_alloc_rx_queue_mbufs().
 712                  */
 713                 if (max_rx_pkt_len <
 714                     enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu)) {
 715                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 716                                     " when scatter rx mode is in use.\n");
 717                 }
 718         } else {
 719                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 720                          queue_idx);
 721                 rq_sop->data_queue_enable = 0;
 722                 rq_data->in_use = 0;
 723         }
 724
 725         /* number of descriptors have to be a multiple of 32 */
 726         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
 727         nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
 728
 729         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 730         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 731
 732         if (mbufs_per_pkt > 1) {
 733                 min_sop = 64;
 734                 max_sop = ((enic->config.rq_desc_count /
 735                             (mbufs_per_pkt - 1)) & ~0x1F);
 736                 min_data = min_sop * (mbufs_per_pkt - 1);
 737                 max_data = enic->config.rq_desc_count;
 738         } else {
 739                 min_sop = 64;
 740                 max_sop = enic->config.rq_desc_count;
 741                 min_data = 0;
 742                 max_data = 0;
 743         }
 744
 745         if (nb_desc < (min_sop + min_data)) {
 746                 dev_warning(enic,
 747                             "Number of rx descs too low, adjusting to minimum\n");
 748                 nb_sop_desc = min_sop;
 749                 nb_data_desc = min_data;
 750         } else if (nb_desc > (max_sop + max_data)) {
 751                 dev_warning(enic,
 752                             "Number of rx_descs too high, adjusting to maximum\n");
 753                 nb_sop_desc = max_sop;
 754                 nb_data_desc = max_data;
 755         }
 756         if (mbufs_per_pkt > 1) {
 757                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 758                          " rx descriptor range is %u to %u\n",
 759                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 760                          max_sop + max_data);
 761         }
 762         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 763                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 764
 765         /* Allocate sop queue resources */
 766         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 767                 nb_sop_desc, sizeof(struct rq_enet_desc));
 768         if (rc) {
 769                 dev_err(enic, "error in allocation of sop rq\n");
 770                 goto err_exit;
 771         }
 772         nb_sop_desc = rq_sop->ring.desc_count;
 773
 774         if (rq_data->in_use) {
 775                 /* Allocate data queue resources */
 776                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 777                                    nb_data_desc,
 778                                    sizeof(struct rq_enet_desc));
 779                 if (rc) {
 780                         dev_err(enic, "error in allocation of data rq\n");
 781                         goto err_free_rq_sop;
 782                 }
 783                 nb_data_desc = rq_data->ring.desc_count;
 784         }
 785         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 786                            socket_id, nb_sop_desc + nb_data_desc,
 787                            sizeof(struct cq_enet_rq_desc));
 788         if (rc) {
 789                 dev_err(enic, "error in allocation of cq for rq\n");
 790                 goto err_free_rq_data;
 791         }
 792
 793         /* Allocate the mbuf rings */
 794         rq_sop->mbuf_ring = (struct rte_mbuf **)
 795                 rte_zmalloc_socket("rq->mbuf_ring",
 796                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 797                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 798         if (rq_sop->mbuf_ring == NULL)
 799                 goto err_free_cq;
 800
 801         if (rq_data->in_use) {
 802                 rq_data->mbuf_ring = (struct rte_mbuf **)
 803                         rte_zmalloc_socket("rq->mbuf_ring",
 804                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 805                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 806                 if (rq_data->mbuf_ring == NULL)
 807                         goto err_free_sop_mbuf;
 808         }
 809
 810         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 811
 812         return 0;
 813
 814 err_free_sop_mbuf:
 815         rte_free(rq_sop->mbuf_ring);
 816 err_free_cq:
 817         /* cleanup on error */
 818         vnic_cq_free(&enic->cq[queue_idx]);
 819 err_free_rq_data:
 820         if (rq_data->in_use)
 821                 vnic_rq_free(rq_data);
 822 err_free_rq_sop:
 823         vnic_rq_free(rq_sop);
 824 err_exit:
 825         return -ENOMEM;
 826 }
 827
 828 void enic_free_wq(void *txq)
 829 {
 830         struct vnic_wq *wq;
 831         struct enic *enic;
 832
 833         if (txq == NULL)
 834                 return;
 835
 836         wq = (struct vnic_wq *)txq;
 837         enic = vnic_dev_priv(wq->vdev);
 838         rte_memzone_free(wq->cqmsg_rz);
 839         vnic_wq_free(wq);
 840         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 841 }
 842
 843 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 844         unsigned int socket_id, uint16_t nb_desc)
 845 {
 846         int err;
 847         struct vnic_wq *wq = &enic->wq[queue_idx];
 848         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 849         char name[NAME_MAX];
 850         static int instance;
 851
 852         wq->socket_id = socket_id;
 853         if (nb_desc) {
 854                 if (nb_desc > enic->config.wq_desc_count) {
 855                         dev_warning(enic,
 856                                 "WQ %d - number of tx desc in cmd line (%d)"\
 857                                 "is greater than that in the UCSM/CIMC adapter"\
 858                                 "policy.  Applying the value in the adapter "\
 859                                 "policy (%d)\n",
 860                                 queue_idx, nb_desc, enic->config.wq_desc_count);
 861                 } else if (nb_desc != enic->config.wq_desc_count) {
 862                         enic->config.wq_desc_count = nb_desc;
 863                         dev_info(enic,
 864                                 "TX Queues - effective number of descs:%d\n",
 865                                 nb_desc);
 866                 }
 867         }
 868
 869         /* Allocate queue resources */
 870         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
 871                 enic->config.wq_desc_count,
 872                 sizeof(struct wq_enet_desc));
 873         if (err) {
 874                 dev_err(enic, "error in allocation of wq\n");
 875                 return err;
 876         }
 877
 878         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
 879                 socket_id, enic->config.wq_desc_count,
 880                 sizeof(struct cq_enet_wq_desc));
 881         if (err) {
 882                 vnic_wq_free(wq);
 883                 dev_err(enic, "error in allocation of cq for wq\n");
 884         }
 885
 886         /* setup up CQ message */
 887         snprintf((char *)name, sizeof(name),
 888                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
 889                 instance++);
 890
 891         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
 892                         sizeof(uint32_t), SOCKET_ID_ANY,
 893                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 894         if (!wq->cqmsg_rz)
 895                 return -ENOMEM;
 896
 897         return err;
 898 }
 899
 900 int enic_disable(struct enic *enic)
 901 {
 902         unsigned int i;
 903         int err;
 904
 905         for (i = 0; i < enic->intr_count; i++) {
 906                 vnic_intr_mask(&enic->intr[i]);
 907                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
 908         }
 909         enic_rxq_intr_deinit(enic);
 910         rte_intr_disable(&enic->pdev->intr_handle);
 911         rte_intr_callback_unregister(&enic->pdev->intr_handle,
 912                                      enic_intr_handler,
 913                                      (void *)enic->rte_dev);
 914
 915         vnic_dev_disable(enic->vdev);
 916
 917         enic_clsf_destroy(enic);
 918
 919         if (!enic_is_sriov_vf(enic))
 920                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
 921
 922         for (i = 0; i < enic->wq_count; i++) {
 923                 err = vnic_wq_disable(&enic->wq[i]);
 924                 if (err)
 925                         return err;
 926         }
 927         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
 928                 if (enic->rq[i].in_use) {
 929                         err = vnic_rq_disable(&enic->rq[i]);
 930                         if (err)
 931                                 return err;
 932                 }
 933         }
 934
 935         /* If we were using interrupts, set the interrupt vector to -1
 936          * to disable interrupts.  We are not disabling link notifcations,
 937          * though, as we want the polling of link status to continue working.
 938          */
 939         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
 940                 vnic_dev_notify_set(enic->vdev, -1);
 941
 942         vnic_dev_set_reset_flag(enic->vdev, 1);
 943
 944         for (i = 0; i < enic->wq_count; i++)
 945                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
 946
 947         for (i = 0; i < enic_vnic_rq_count(enic); i++)
 948                 if (enic->rq[i].in_use)
 949                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
 950         for (i = 0; i < enic->cq_count; i++)
 951                 vnic_cq_clean(&enic->cq[i]);
 952         for (i = 0; i < enic->intr_count; i++)
 953                 vnic_intr_clean(&enic->intr[i]);
 954
 955         return 0;
 956 }
 957
 958 static int enic_dev_wait(struct vnic_dev *vdev,
 959         int (*start)(struct vnic_dev *, int),
 960         int (*finished)(struct vnic_dev *, int *),
 961         int arg)
 962 {
 963         int done;
 964         int err;
 965         int i;
 966
 967         err = start(vdev, arg);
 968         if (err)
 969                 return err;
 970
 971         /* Wait for func to complete...2 seconds max */
 972         for (i = 0; i < 2000; i++) {
 973                 err = finished(vdev, &done);
 974                 if (err)
 975                         return err;
 976                 if (done)
 977                         return 0;
 978                 usleep(1000);
 979         }
 980         return -ETIMEDOUT;
 981 }
 982
 983 static int enic_dev_open(struct enic *enic)
 984 {
 985         int err;
 986         int flags = CMD_OPENF_IG_DESCCACHE;
 987
 988         err = enic_dev_wait(enic->vdev, vnic_dev_open,
 989                 vnic_dev_open_done, flags);
 990         if (err)
 991                 dev_err(enic_get_dev(enic),
 992                         "vNIC device open failed, err %d\n", err);
 993
 994         return err;
 995 }
 996
 997 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
 998 {
 999         dma_addr_t rss_key_buf_pa;
1000         union vnic_rss_key *rss_key_buf_va = NULL;
1001         int err, i;
1002         u8 name[NAME_MAX];
1003
1004         RTE_ASSERT(user_key != NULL);
1005         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1006         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1007                 &rss_key_buf_pa, name);
1008         if (!rss_key_buf_va)
1009                 return -ENOMEM;
1010
1011         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1012                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1013
1014         err = enic_set_rss_key(enic,
1015                 rss_key_buf_pa,
1016                 sizeof(union vnic_rss_key));
1017
1018         /* Save for later queries */
1019         if (!err) {
1020                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1021                            sizeof(union vnic_rss_key));
1022         }
1023         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1024                 rss_key_buf_va, rss_key_buf_pa);
1025
1026         return err;
1027 }
1028
1029 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1030 {
1031         dma_addr_t rss_cpu_buf_pa;
1032         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1033         int err;
1034         u8 name[NAME_MAX];
1035
1036         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1037         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1038                 &rss_cpu_buf_pa, name);
1039         if (!rss_cpu_buf_va)
1040                 return -ENOMEM;
1041
1042         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1043
1044         err = enic_set_rss_cpu(enic,
1045                 rss_cpu_buf_pa,
1046                 sizeof(union vnic_rss_cpu));
1047
1048         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1049                 rss_cpu_buf_va, rss_cpu_buf_pa);
1050
1051         /* Save for later queries */
1052         if (!err)
1053                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1054         return err;
1055 }
1056
1057 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1058         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1059 {
1060         const u8 tso_ipid_split_en = 0;
1061         int err;
1062
1063         err = enic_set_nic_cfg(enic,
1064                 rss_default_cpu, rss_hash_type,
1065                 rss_hash_bits, rss_base_cpu,
1066                 rss_enable, tso_ipid_split_en,
1067                 enic->ig_vlan_strip_en);
1068
1069         return err;
1070 }
1071
1072 /* Initialize RSS with defaults, called from dev_configure */
1073 int enic_init_rss_nic_cfg(struct enic *enic)
1074 {
1075         static uint8_t default_rss_key[] = {
1076                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1077                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1078                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1079                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1080         };
1081         struct rte_eth_rss_conf rss_conf;
1082         union vnic_rss_cpu rss_cpu;
1083         int ret, i;
1084
1085         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1086         /*
1087          * If setting key for the first time, and the user gives us none, then
1088          * push the default key to NIC.
1089          */
1090         if (rss_conf.rss_key == NULL) {
1091                 rss_conf.rss_key = default_rss_key;
1092                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1093         }
1094         ret = enic_set_rss_conf(enic, &rss_conf);
1095         if (ret) {
1096                 dev_err(enic, "Failed to configure RSS\n");
1097                 return ret;
1098         }
1099         if (enic->rss_enable) {
1100                 /* If enabling RSS, use the default reta */
1101                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1102                         rss_cpu.cpu[i / 4].b[i % 4] =
1103                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1104                 }
1105                 ret = enic_set_rss_reta(enic, &rss_cpu);
1106                 if (ret)
1107                         dev_err(enic, "Failed to set RSS indirection table\n");
1108         }
1109         return ret;
1110 }
1111
1112 int enic_setup_finish(struct enic *enic)
1113 {
1114         enic_init_soft_stats(enic);
1115
1116         /* Default conf */
1117         vnic_dev_packet_filter(enic->vdev,
1118                 1 /* directed  */,
1119                 1 /* multicast */,
1120                 1 /* broadcast */,
1121                 0 /* promisc   */,
1122                 1 /* allmulti  */);
1123
1124         enic->promisc = 0;
1125         enic->allmulti = 1;
1126
1127         return 0;
1128 }
1129
1130 static int enic_rss_conf_valid(struct enic *enic,
1131                                struct rte_eth_rss_conf *rss_conf)
1132 {
1133         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1134         if (enic->flow_type_rss_offloads == 0)
1135                 return 0;
1136         if (rss_conf->rss_key != NULL &&
1137             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1138                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1139                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1140                 return -EINVAL;
1141         }
1142         if (rss_conf->rss_hf != 0 &&
1143             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1144                 dev_err(enic, "Given rss_hf contains none of the supported"
1145                         " types\n");
1146                 return -EINVAL;
1147         }
1148         return 0;
1149 }
1150
1151 /* Set hash type and key according to rss_conf */
1152 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1153 {
1154         struct rte_eth_dev *eth_dev;
1155         uint64_t rss_hf;
1156         u8 rss_hash_type;
1157         u8 rss_enable;
1158         int ret;
1159
1160         RTE_ASSERT(rss_conf != NULL);
1161         ret = enic_rss_conf_valid(enic, rss_conf);
1162         if (ret) {
1163                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1164                 return ret;
1165         }
1166
1167         eth_dev = enic->rte_dev;
1168         rss_hash_type = 0;
1169         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1170         if (enic->rq_count > 1 &&
1171             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1172             rss_hf != 0) {
1173                 rss_enable = 1;
1174                 if (rss_hf & ETH_RSS_IPV4)
1175                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1176                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1177                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1178                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1179                         /*
1180                          * 'TCP' is not a typo. HW does not have a separate
1181                          * enable bit for UDP RSS. The TCP bit enables both TCP
1182                          * and UDP RSS..
1183                          */
1184                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1185                 }
1186                 if (rss_hf & ETH_RSS_IPV6)
1187                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1188                 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1189                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1190                 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) {
1191                         /* Again, 'TCP' is not a typo. */
1192                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1193                 }
1194                 if (rss_hf & ETH_RSS_IPV6_EX)
1195                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6_EX;
1196                 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
1197                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX;
1198         } else {
1199                 rss_enable = 0;
1200                 rss_hf = 0;
1201         }
1202
1203         /* Set the hash key if provided */
1204         if (rss_enable && rss_conf->rss_key) {
1205                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1206                 if (ret) {
1207                         dev_err(enic, "Failed to set RSS key\n");
1208                         return ret;
1209                 }
1210         }
1211
1212         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1213                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1214                               rss_enable);
1215         if (!ret) {
1216                 enic->rss_hf = rss_hf;
1217                 enic->rss_hash_type = rss_hash_type;
1218                 enic->rss_enable = rss_enable;
1219         }
1220         return 0;
1221 }
1222
1223 int enic_set_vlan_strip(struct enic *enic)
1224 {
1225         /*
1226          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1227          * together. So, re-do niccfg, preserving the current RSS settings.
1228          */
1229         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1230                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1231                                enic->rss_enable);
1232 }
1233
1234 void enic_add_packet_filter(struct enic *enic)
1235 {
1236         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1237         vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1238                 enic->promisc, enic->allmulti);
1239 }
1240
1241 int enic_get_link_status(struct enic *enic)
1242 {
1243         return vnic_dev_link_status(enic->vdev);
1244 }
1245
1246 static void enic_dev_deinit(struct enic *enic)
1247 {
1248         struct rte_eth_dev *eth_dev = enic->rte_dev;
1249
1250         /* stop link status checking */
1251         vnic_dev_notify_unset(enic->vdev);
1252
1253         rte_free(eth_dev->data->mac_addrs);
1254         rte_free(enic->cq);
1255         rte_free(enic->intr);
1256         rte_free(enic->rq);
1257         rte_free(enic->wq);
1258 }
1259
1260
1261 int enic_set_vnic_res(struct enic *enic)
1262 {
1263         struct rte_eth_dev *eth_dev = enic->rte_dev;
1264         int rc = 0;
1265         unsigned int required_rq, required_wq, required_cq, required_intr;
1266
1267         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1268         required_rq = eth_dev->data->nb_rx_queues * 2;
1269         required_wq = eth_dev->data->nb_tx_queues;
1270         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1271         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1272         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1273                 required_intr += eth_dev->data->nb_rx_queues;
1274         }
1275
1276         if (enic->conf_rq_count < required_rq) {
1277                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1278                         eth_dev->data->nb_rx_queues,
1279                         required_rq, enic->conf_rq_count);
1280                 rc = -EINVAL;
1281         }
1282         if (enic->conf_wq_count < required_wq) {
1283                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1284                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1285                 rc = -EINVAL;
1286         }
1287
1288         if (enic->conf_cq_count < required_cq) {
1289                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1290                         required_cq, enic->conf_cq_count);
1291                 rc = -EINVAL;
1292         }
1293         if (enic->conf_intr_count < required_intr) {
1294                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1295                         " interrupts. Required:%u, Configured:%u\n",
1296                         required_intr, enic->conf_intr_count);
1297                 rc = -EINVAL;
1298         }
1299
1300         if (rc == 0) {
1301                 enic->rq_count = eth_dev->data->nb_rx_queues;
1302                 enic->wq_count = eth_dev->data->nb_tx_queues;
1303                 enic->cq_count = enic->rq_count + enic->wq_count;
1304                 enic->intr_count = required_intr;
1305         }
1306
1307         return rc;
1308 }
1309
1310 /* Initialize the completion queue for an RQ */
1311 static int
1312 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1313 {
1314         struct vnic_rq *sop_rq, *data_rq;
1315         unsigned int cq_idx;
1316         int rc = 0;
1317
1318         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1319         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1320         cq_idx = rq_idx;
1321
1322         vnic_cq_clean(&enic->cq[cq_idx]);
1323         vnic_cq_init(&enic->cq[cq_idx],
1324                      0 /* flow_control_enable */,
1325                      1 /* color_enable */,
1326                      0 /* cq_head */,
1327                      0 /* cq_tail */,
1328                      1 /* cq_tail_color */,
1329                      0 /* interrupt_enable */,
1330                      1 /* cq_entry_enable */,
1331                      0 /* cq_message_enable */,
1332                      0 /* interrupt offset */,
1333                      0 /* cq_message_addr */);
1334
1335
1336         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1337                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1338                            sop_rq->ring.desc_count - 1, 1, 0);
1339         if (data_rq->in_use) {
1340                 vnic_rq_init_start(data_rq,
1341                                    enic_cq_rq(enic,
1342                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1343                                    data_rq->ring.desc_count - 1, 1, 0);
1344         }
1345
1346         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1347         if (rc)
1348                 return rc;
1349
1350         if (data_rq->in_use) {
1351                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1352                 if (rc) {
1353                         enic_rxmbuf_queue_release(enic, sop_rq);
1354                         return rc;
1355                 }
1356         }
1357
1358         return 0;
1359 }
1360
1361 /* The Cisco NIC can send and receive packets up to a max packet size
1362  * determined by the NIC type and firmware. There is also an MTU
1363  * configured into the NIC via the CIMC/UCSM management interface
1364  * which can be overridden by this function (up to the max packet size).
1365  * Depending on the network setup, doing so may cause packet drops
1366  * and unexpected behavior.
1367  */
1368 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1369 {
1370         unsigned int rq_idx;
1371         struct vnic_rq *rq;
1372         int rc = 0;
1373         uint16_t old_mtu;       /* previous setting */
1374         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1375         struct rte_eth_dev *eth_dev = enic->rte_dev;
1376
1377         old_mtu = eth_dev->data->mtu;
1378         config_mtu = enic->config.mtu;
1379
1380         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1381                 return -E_RTE_SECONDARY;
1382
1383         if (new_mtu > enic->max_mtu) {
1384                 dev_err(enic,
1385                         "MTU not updated: requested (%u) greater than max (%u)\n",
1386                         new_mtu, enic->max_mtu);
1387                 return -EINVAL;
1388         }
1389         if (new_mtu < ENIC_MIN_MTU) {
1390                 dev_info(enic,
1391                         "MTU not updated: requested (%u) less than min (%u)\n",
1392                         new_mtu, ENIC_MIN_MTU);
1393                 return -EINVAL;
1394         }
1395         if (new_mtu > config_mtu)
1396                 dev_warning(enic,
1397                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1398                         new_mtu, config_mtu);
1399
1400         /* The easy case is when scatter is disabled. However if the MTU
1401          * becomes greater than the mbuf data size, packet drops will ensue.
1402          */
1403         if (!(enic->rte_dev->data->dev_conf.rxmode.offloads &
1404               DEV_RX_OFFLOAD_SCATTER)) {
1405                 eth_dev->data->mtu = new_mtu;
1406                 goto set_mtu_done;
1407         }
1408
1409         /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to
1410          * change Rx scatter mode if necessary for better performance. I.e. if
1411          * MTU was greater than the mbuf size and now it's less, scatter Rx
1412          * doesn't have to be used and vice versa.
1413           */
1414         rte_spinlock_lock(&enic->mtu_lock);
1415
1416         /* Stop traffic on all RQs */
1417         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1418                 rq = &enic->rq[rq_idx];
1419                 if (rq->is_sop && rq->in_use) {
1420                         rc = enic_stop_rq(enic,
1421                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1422                         if (rc) {
1423                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1424                                 goto set_mtu_done;
1425                         }
1426                 }
1427         }
1428
1429         /* replace Rx function with a no-op to avoid getting stale pkts */
1430         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1431         rte_mb();
1432
1433         /* Allow time for threads to exit the real Rx function. */
1434         usleep(100000);
1435
1436         /* now it is safe to reconfigure the RQs */
1437
1438         /* update the mtu */
1439         eth_dev->data->mtu = new_mtu;
1440
1441         /* free and reallocate RQs with the new MTU */
1442         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1443                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1444                 if (!rq->in_use)
1445                         continue;
1446
1447                 enic_free_rq(rq);
1448                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1449                                    rq->tot_nb_desc, rq->rx_free_thresh);
1450                 if (rc) {
1451                         dev_err(enic,
1452                                 "Fatal MTU alloc error- No traffic will pass\n");
1453                         goto set_mtu_done;
1454                 }
1455
1456                 rc = enic_reinit_rq(enic, rq_idx);
1457                 if (rc) {
1458                         dev_err(enic,
1459                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1460                         goto set_mtu_done;
1461                 }
1462         }
1463
1464         /* put back the real receive function */
1465         rte_mb();
1466         eth_dev->rx_pkt_burst = enic_recv_pkts;
1467         rte_mb();
1468
1469         /* restart Rx traffic */
1470         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1471                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1472                 if (rq->is_sop && rq->in_use)
1473                         enic_start_rq(enic, rq_idx);
1474         }
1475
1476 set_mtu_done:
1477         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1478         rte_spinlock_unlock(&enic->mtu_lock);
1479         return rc;
1480 }
1481
1482 static int enic_dev_init(struct enic *enic)
1483 {
1484         int err;
1485         struct rte_eth_dev *eth_dev = enic->rte_dev;
1486
1487         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1488
1489         /* Get vNIC configuration
1490         */
1491         err = enic_get_vnic_config(enic);
1492         if (err) {
1493                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1494                 return err;
1495         }
1496
1497         /* Get available resource counts */
1498         enic_get_res_counts(enic);
1499         if (enic->conf_rq_count == 1) {
1500                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1501                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1502                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1503                 return -EINVAL;
1504         }
1505         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1506         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1507                                enic->conf_cq_count, 8);
1508         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1509                                  enic->conf_intr_count, 8);
1510         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1511                                enic->conf_rq_count, 8);
1512         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1513                                enic->conf_wq_count, 8);
1514         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1515                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1516                 return -1;
1517         }
1518         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1519                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1520                 return -1;
1521         }
1522         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1523                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1524                 return -1;
1525         }
1526         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1527                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1528                 return -1;
1529         }
1530
1531         /* Get the supported filters */
1532         enic_fdir_info(enic);
1533
1534         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1535                                                 * ENIC_MAX_MAC_ADDR, 0);
1536         if (!eth_dev->data->mac_addrs) {
1537                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1538                 return -1;
1539         }
1540         ether_addr_copy((struct ether_addr *) enic->mac_addr,
1541                         eth_dev->data->mac_addrs);
1542
1543         vnic_dev_set_reset_flag(enic->vdev, 0);
1544
1545         LIST_INIT(&enic->flows);
1546         rte_spinlock_init(&enic->flows_lock);
1547
1548         /* set up link status checking */
1549         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1550
1551         enic->overlay_offload = false;
1552         if (!enic->disable_overlay && enic->vxlan &&
1553             /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1554             vnic_dev_overlay_offload_ctrl(enic->vdev,
1555                                           OVERLAY_FEATURE_VXLAN,
1556                                           OVERLAY_OFFLOAD_ENABLE) == 0) {
1557                 enic->tx_offload_capa |=
1558                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1559                         DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1560                         DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1561                 /*
1562                  * Do not add PKT_TX_OUTER_{IPV4,IPV6} as they are not
1563                  * 'offload' flags (i.e. not part of PKT_TX_OFFLOAD_MASK).
1564                  */
1565                 enic->tx_offload_mask |=
1566                         PKT_TX_OUTER_IP_CKSUM |
1567                         PKT_TX_TUNNEL_MASK;
1568                 enic->overlay_offload = true;
1569                 dev_info(enic, "Overlay offload is enabled\n");
1570         }
1571
1572         return 0;
1573
1574 }
1575
1576 int enic_probe(struct enic *enic)
1577 {
1578         struct rte_pci_device *pdev = enic->pdev;
1579         int err = -1;
1580
1581         dev_debug(enic, " Initializing ENIC PMD\n");
1582
1583         /* if this is a secondary process the hardware is already initialized */
1584         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1585                 return 0;
1586
1587         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1588         enic->bar0.len = pdev->mem_resource[0].len;
1589
1590         /* Register vNIC device */
1591         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1592         if (!enic->vdev) {
1593                 dev_err(enic, "vNIC registration failed, aborting\n");
1594                 goto err_out;
1595         }
1596
1597         LIST_INIT(&enic->memzone_list);
1598         rte_spinlock_init(&enic->memzone_list_lock);
1599
1600         vnic_register_cbacks(enic->vdev,
1601                 enic_alloc_consistent,
1602                 enic_free_consistent);
1603
1604         /*
1605          * Allocate the consistent memory for stats upfront so both primary and
1606          * secondary processes can dump stats.
1607          */
1608         err = vnic_dev_alloc_stats_mem(enic->vdev);
1609         if (err) {
1610                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1611                 goto err_out_unregister;
1612         }
1613         /* Issue device open to get device in known state */
1614         err = enic_dev_open(enic);
1615         if (err) {
1616                 dev_err(enic, "vNIC dev open failed, aborting\n");
1617                 goto err_out_unregister;
1618         }
1619
1620         /* Set ingress vlan rewrite mode before vnic initialization */
1621         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1622                 IG_VLAN_REWRITE_MODE_PASS_THRU);
1623         if (err) {
1624                 dev_err(enic,
1625                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1626                 goto err_out_dev_close;
1627         }
1628
1629         /* Issue device init to initialize the vnic-to-switch link.
1630          * We'll start with carrier off and wait for link UP
1631          * notification later to turn on carrier.  We don't need
1632          * to wait here for the vnic-to-switch link initialization
1633          * to complete; link UP notification is the indication that
1634          * the process is complete.
1635          */
1636
1637         err = vnic_dev_init(enic->vdev, 0);
1638         if (err) {
1639                 dev_err(enic, "vNIC dev init failed, aborting\n");
1640                 goto err_out_dev_close;
1641         }
1642
1643         err = enic_dev_init(enic);
1644         if (err) {
1645                 dev_err(enic, "Device initialization failed, aborting\n");
1646                 goto err_out_dev_close;
1647         }
1648
1649         return 0;
1650
1651 err_out_dev_close:
1652         vnic_dev_close(enic->vdev);
1653 err_out_unregister:
1654         vnic_dev_unregister(enic->vdev);
1655 err_out:
1656         return err;
1657 }
1658
1659 void enic_remove(struct enic *enic)
1660 {
1661         enic_dev_deinit(enic);
1662         vnic_dev_close(enic->vdev);
1663         vnic_dev_unregister(enic->vdev);
1664 }