drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11 #include <libgen.h>
  12
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_ethdev_driver.h>
  20
  21 #include "enic_compat.h"
  22 #include "enic.h"
  23 #include "wq_enet_desc.h"
  24 #include "rq_enet_desc.h"
  25 #include "cq_enet_desc.h"
  26 #include "vnic_enet.h"
  27 #include "vnic_dev.h"
  28 #include "vnic_wq.h"
  29 #include "vnic_rq.h"
  30 #include "vnic_cq.h"
  31 #include "vnic_intr.h"
  32 #include "vnic_nic.h"
  33
  34 static inline int enic_is_sriov_vf(struct enic *enic)
  35 {
  36         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  37 }
  38
  39 static int is_zero_addr(uint8_t *addr)
  40 {
  41         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  42 }
  43
  44 static int is_mcast_addr(uint8_t *addr)
  45 {
  46         return addr[0] & 1;
  47 }
  48
  49 static int is_eth_addr_valid(uint8_t *addr)
  50 {
  51         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  52 }
  53
  54 static void
  55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  56 {
  57         uint16_t i;
  58
  59         if (!rq || !rq->mbuf_ring) {
  60                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  61                 return;
  62         }
  63
  64         for (i = 0; i < rq->ring.desc_count; i++) {
  65                 if (rq->mbuf_ring[i]) {
  66                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  67                         rq->mbuf_ring[i] = NULL;
  68                 }
  69         }
  70 }
  71
  72 static void enic_free_wq_buf(struct vnic_wq_buf *buf)
  73 {
  74         struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
  75
  76         rte_pktmbuf_free_seg(mbuf);
  77         buf->mb = NULL;
  78 }
  79
  80 static void enic_log_q_error(struct enic *enic)
  81 {
  82         unsigned int i;
  83         u32 error_status;
  84
  85         for (i = 0; i < enic->wq_count; i++) {
  86                 error_status = vnic_wq_error_status(&enic->wq[i]);
  87                 if (error_status)
  88                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  89                                 error_status);
  90         }
  91
  92         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  93                 if (!enic->rq[i].in_use)
  94                         continue;
  95                 error_status = vnic_rq_error_status(&enic->rq[i]);
  96                 if (error_status)
  97                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  98                                 error_status);
  99         }
 100 }
 101
 102 static void enic_clear_soft_stats(struct enic *enic)
 103 {
 104         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 105         rte_atomic64_clear(&soft_stats->rx_nombuf);
 106         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 107         rte_atomic64_clear(&soft_stats->tx_oversized);
 108 }
 109
 110 static void enic_init_soft_stats(struct enic *enic)
 111 {
 112         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 113         rte_atomic64_init(&soft_stats->rx_nombuf);
 114         rte_atomic64_init(&soft_stats->rx_packet_errors);
 115         rte_atomic64_init(&soft_stats->tx_oversized);
 116         enic_clear_soft_stats(enic);
 117 }
 118
 119 void enic_dev_stats_clear(struct enic *enic)
 120 {
 121         if (vnic_dev_stats_clear(enic->vdev))
 122                 dev_err(enic, "Error in clearing stats\n");
 123         enic_clear_soft_stats(enic);
 124 }
 125
 126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 127 {
 128         struct vnic_stats *stats;
 129         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 130         int64_t rx_truncated;
 131         uint64_t rx_packet_errors;
 132         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 133
 134         if (ret) {
 135                 dev_err(enic, "Error in getting stats\n");
 136                 return ret;
 137         }
 138
 139         /* The number of truncated packets can only be calculated by
 140          * subtracting a hardware counter from error packets received by
 141          * the driver. Note: this causes transient inaccuracies in the
 142          * ipackets count. Also, the length of truncated packets are
 143          * counted in ibytes even though truncated packets are dropped
 144          * which can make ibytes be slightly higher than it should be.
 145          */
 146         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 147         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 148
 149         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 150         r_stats->opackets = stats->tx.tx_frames_ok;
 151
 152         r_stats->ibytes = stats->rx.rx_bytes_ok;
 153         r_stats->obytes = stats->tx.tx_bytes_ok;
 154
 155         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 156         r_stats->oerrors = stats->tx.tx_errors
 157                            + rte_atomic64_read(&soft_stats->tx_oversized);
 158
 159         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 160
 161         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 162         return 0;
 163 }
 164
 165 void enic_del_mac_address(struct enic *enic, int mac_index)
 166 {
 167         struct rte_eth_dev *eth_dev = enic->rte_dev;
 168         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 169
 170         if (vnic_dev_del_addr(enic->vdev, mac_addr))
 171                 dev_err(enic, "del mac addr failed\n");
 172 }
 173
 174 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 175 {
 176         int err;
 177
 178         if (!is_eth_addr_valid(mac_addr)) {
 179                 dev_err(enic, "invalid mac address\n");
 180                 return -EINVAL;
 181         }
 182
 183         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 184         if (err)
 185                 dev_err(enic, "add mac addr failed\n");
 186         return err;
 187 }
 188
 189 static void
 190 enic_free_rq_buf(struct rte_mbuf **mbuf)
 191 {
 192         if (*mbuf == NULL)
 193                 return;
 194
 195         rte_pktmbuf_free(*mbuf);
 196         *mbuf = NULL;
 197 }
 198
 199 void enic_init_vnic_resources(struct enic *enic)
 200 {
 201         unsigned int error_interrupt_enable = 1;
 202         unsigned int error_interrupt_offset = 0;
 203         unsigned int index = 0;
 204         unsigned int cq_idx;
 205         struct vnic_rq *data_rq;
 206
 207         for (index = 0; index < enic->rq_count; index++) {
 208                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 209
 210                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 211                         cq_idx,
 212                         error_interrupt_enable,
 213                         error_interrupt_offset);
 214
 215                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 216                 if (data_rq->in_use)
 217                         vnic_rq_init(data_rq,
 218                                      cq_idx,
 219                                      error_interrupt_enable,
 220                                      error_interrupt_offset);
 221
 222                 vnic_cq_init(&enic->cq[cq_idx],
 223                         0 /* flow_control_enable */,
 224                         1 /* color_enable */,
 225                         0 /* cq_head */,
 226                         0 /* cq_tail */,
 227                         1 /* cq_tail_color */,
 228                         0 /* interrupt_enable */,
 229                         1 /* cq_entry_enable */,
 230                         0 /* cq_message_enable */,
 231                         0 /* interrupt offset */,
 232                         0 /* cq_message_addr */);
 233         }
 234
 235         for (index = 0; index < enic->wq_count; index++) {
 236                 vnic_wq_init(&enic->wq[index],
 237                         enic_cq_wq(enic, index),
 238                         error_interrupt_enable,
 239                         error_interrupt_offset);
 240
 241                 cq_idx = enic_cq_wq(enic, index);
 242                 vnic_cq_init(&enic->cq[cq_idx],
 243                         0 /* flow_control_enable */,
 244                         1 /* color_enable */,
 245                         0 /* cq_head */,
 246                         0 /* cq_tail */,
 247                         1 /* cq_tail_color */,
 248                         0 /* interrupt_enable */,
 249                         0 /* cq_entry_enable */,
 250                         1 /* cq_message_enable */,
 251                         0 /* interrupt offset */,
 252                         (u64)enic->wq[index].cqmsg_rz->iova);
 253         }
 254
 255         vnic_intr_init(&enic->intr,
 256                 enic->config.intr_timer_usec,
 257                 enic->config.intr_timer_type,
 258                 /*mask_on_assertion*/1);
 259 }
 260
 261
 262 static int
 263 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 264 {
 265         struct rte_mbuf *mb;
 266         struct rq_enet_desc *rqd = rq->ring.descs;
 267         unsigned i;
 268         dma_addr_t dma_addr;
 269
 270         if (!rq->in_use)
 271                 return 0;
 272
 273         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 274                   rq->ring.desc_count);
 275
 276         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 277                 mb = rte_mbuf_raw_alloc(rq->mp);
 278                 if (mb == NULL) {
 279                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 280                         (unsigned)rq->index);
 281                         return -ENOMEM;
 282                 }
 283
 284                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 285                 dma_addr = (dma_addr_t)(mb->buf_iova
 286                            + RTE_PKTMBUF_HEADROOM);
 287                 rq_enet_desc_enc(rqd, dma_addr,
 288                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 289                                 : RQ_ENET_TYPE_NOT_SOP),
 290                                 mb->buf_len - RTE_PKTMBUF_HEADROOM);
 291                 rq->mbuf_ring[i] = mb;
 292         }
 293
 294         /* make sure all prior writes are complete before doing the PIO write */
 295         rte_rmb();
 296
 297         /* Post all but the last buffer to VIC. */
 298         rq->posted_index = rq->ring.desc_count - 1;
 299
 300         rq->rx_nb_hold = 0;
 301
 302         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 303                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 304         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 305         iowrite32(0, &rq->ctrl->fetch_index);
 306         rte_rmb();
 307
 308         return 0;
 309
 310 }
 311
 312 static void *
 313 enic_alloc_consistent(void *priv, size_t size,
 314         dma_addr_t *dma_handle, u8 *name)
 315 {
 316         void *vaddr;
 317         const struct rte_memzone *rz;
 318         *dma_handle = 0;
 319         struct enic *enic = (struct enic *)priv;
 320         struct enic_memzone_entry *mze;
 321
 322         rz = rte_memzone_reserve_aligned((const char *)name,
 323                                          size, SOCKET_ID_ANY, 0, ENIC_ALIGN);
 324         if (!rz) {
 325                 pr_err("%s : Failed to allocate memory requested for %s\n",
 326                         __func__, name);
 327                 return NULL;
 328         }
 329
 330         vaddr = rz->addr;
 331         *dma_handle = (dma_addr_t)rz->iova;
 332
 333         mze = rte_malloc("enic memzone entry",
 334                          sizeof(struct enic_memzone_entry), 0);
 335
 336         if (!mze) {
 337                 pr_err("%s : Failed to allocate memory for memzone list\n",
 338                        __func__);
 339                 rte_memzone_free(rz);
 340                 return NULL;
 341         }
 342
 343         mze->rz = rz;
 344
 345         rte_spinlock_lock(&enic->memzone_list_lock);
 346         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 347         rte_spinlock_unlock(&enic->memzone_list_lock);
 348
 349         return vaddr;
 350 }
 351
 352 static void
 353 enic_free_consistent(void *priv,
 354                      __rte_unused size_t size,
 355                      void *vaddr,
 356                      dma_addr_t dma_handle)
 357 {
 358         struct enic_memzone_entry *mze;
 359         struct enic *enic = (struct enic *)priv;
 360
 361         rte_spinlock_lock(&enic->memzone_list_lock);
 362         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 363                 if (mze->rz->addr == vaddr &&
 364                     mze->rz->iova == dma_handle)
 365                         break;
 366         }
 367         if (mze == NULL) {
 368                 rte_spinlock_unlock(&enic->memzone_list_lock);
 369                 dev_warning(enic,
 370                             "Tried to free memory, but couldn't find it in the memzone list\n");
 371                 return;
 372         }
 373         LIST_REMOVE(mze, entries);
 374         rte_spinlock_unlock(&enic->memzone_list_lock);
 375         rte_memzone_free(mze->rz);
 376         rte_free(mze);
 377 }
 378
 379 int enic_link_update(struct enic *enic)
 380 {
 381         struct rte_eth_dev *eth_dev = enic->rte_dev;
 382         int ret;
 383         int link_status = 0;
 384
 385         link_status = enic_get_link_status(enic);
 386         ret = (link_status == enic->link_status);
 387         enic->link_status = link_status;
 388         eth_dev->data->dev_link.link_status = link_status;
 389         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 390         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 391         return ret;
 392 }
 393
 394 static void
 395 enic_intr_handler(void *arg)
 396 {
 397         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 398         struct enic *enic = pmd_priv(dev);
 399
 400         vnic_intr_return_all_credits(&enic->intr);
 401
 402         enic_link_update(enic);
 403         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 404         enic_log_q_error(enic);
 405 }
 406
 407 int enic_enable(struct enic *enic)
 408 {
 409         unsigned int index;
 410         int err;
 411         struct rte_eth_dev *eth_dev = enic->rte_dev;
 412
 413         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 414         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 415
 416         /* vnic notification of link status has already been turned on in
 417          * enic_dev_init() which is called during probe time.  Here we are
 418          * just turning on interrupt vector 0 if needed.
 419          */
 420         if (eth_dev->data->dev_conf.intr_conf.lsc)
 421                 vnic_dev_notify_set(enic->vdev, 0);
 422
 423         if (enic_clsf_init(enic))
 424                 dev_warning(enic, "Init of hash table for clsf failed."\
 425                         "Flow director feature will not work\n");
 426
 427         for (index = 0; index < enic->rq_count; index++) {
 428                 err = enic_alloc_rx_queue_mbufs(enic,
 429                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 430                 if (err) {
 431                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 432                         return err;
 433                 }
 434                 err = enic_alloc_rx_queue_mbufs(enic,
 435                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 436                 if (err) {
 437                         /* release the allocated mbufs for the sop rq*/
 438                         enic_rxmbuf_queue_release(enic,
 439                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 440
 441                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 442                         return err;
 443                 }
 444         }
 445
 446         for (index = 0; index < enic->wq_count; index++)
 447                 enic_start_wq(enic, index);
 448         for (index = 0; index < enic->rq_count; index++)
 449                 enic_start_rq(enic, index);
 450
 451         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 452
 453         vnic_dev_enable_wait(enic->vdev);
 454
 455         /* Register and enable error interrupt */
 456         rte_intr_callback_register(&(enic->pdev->intr_handle),
 457                 enic_intr_handler, (void *)enic->rte_dev);
 458
 459         rte_intr_enable(&(enic->pdev->intr_handle));
 460         vnic_intr_unmask(&enic->intr);
 461
 462         return 0;
 463 }
 464
 465 int enic_alloc_intr_resources(struct enic *enic)
 466 {
 467         int err;
 468
 469         dev_info(enic, "vNIC resources used:  "\
 470                 "wq %d rq %d cq %d intr %d\n",
 471                 enic->wq_count, enic_vnic_rq_count(enic),
 472                 enic->cq_count, enic->intr_count);
 473
 474         err = vnic_intr_alloc(enic->vdev, &enic->intr, 0);
 475         if (err)
 476                 enic_free_vnic_resources(enic);
 477
 478         return err;
 479 }
 480
 481 void enic_free_rq(void *rxq)
 482 {
 483         struct vnic_rq *rq_sop, *rq_data;
 484         struct enic *enic;
 485
 486         if (rxq == NULL)
 487                 return;
 488
 489         rq_sop = (struct vnic_rq *)rxq;
 490         enic = vnic_dev_priv(rq_sop->vdev);
 491         rq_data = &enic->rq[rq_sop->data_queue_idx];
 492
 493         enic_rxmbuf_queue_release(enic, rq_sop);
 494         if (rq_data->in_use)
 495                 enic_rxmbuf_queue_release(enic, rq_data);
 496
 497         rte_free(rq_sop->mbuf_ring);
 498         if (rq_data->in_use)
 499                 rte_free(rq_data->mbuf_ring);
 500
 501         rq_sop->mbuf_ring = NULL;
 502         rq_data->mbuf_ring = NULL;
 503
 504         vnic_rq_free(rq_sop);
 505         if (rq_data->in_use)
 506                 vnic_rq_free(rq_data);
 507
 508         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 509
 510         rq_sop->in_use = 0;
 511         rq_data->in_use = 0;
 512 }
 513
 514 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 515 {
 516         struct rte_eth_dev *eth_dev = enic->rte_dev;
 517         vnic_wq_enable(&enic->wq[queue_idx]);
 518         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 519 }
 520
 521 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 522 {
 523         struct rte_eth_dev *eth_dev = enic->rte_dev;
 524         int ret;
 525
 526         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 527         if (ret)
 528                 return ret;
 529
 530         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 531         return 0;
 532 }
 533
 534 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 535 {
 536         struct vnic_rq *rq_sop;
 537         struct vnic_rq *rq_data;
 538         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 539         rq_data = &enic->rq[rq_sop->data_queue_idx];
 540         struct rte_eth_dev *eth_dev = enic->rte_dev;
 541
 542         if (rq_data->in_use)
 543                 vnic_rq_enable(rq_data);
 544         rte_mb();
 545         vnic_rq_enable(rq_sop);
 546         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 547 }
 548
 549 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 550 {
 551         int ret1 = 0, ret2 = 0;
 552         struct rte_eth_dev *eth_dev = enic->rte_dev;
 553         struct vnic_rq *rq_sop;
 554         struct vnic_rq *rq_data;
 555         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 556         rq_data = &enic->rq[rq_sop->data_queue_idx];
 557
 558         ret2 = vnic_rq_disable(rq_sop);
 559         rte_mb();
 560         if (rq_data->in_use)
 561                 ret1 = vnic_rq_disable(rq_data);
 562
 563         if (ret2)
 564                 return ret2;
 565         else if (ret1)
 566                 return ret1;
 567
 568         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 569         return 0;
 570 }
 571
 572 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 573         unsigned int socket_id, struct rte_mempool *mp,
 574         uint16_t nb_desc, uint16_t free_thresh)
 575 {
 576         int rc;
 577         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 578         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 579         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 580         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 581         unsigned int mbuf_size, mbufs_per_pkt;
 582         unsigned int nb_sop_desc, nb_data_desc;
 583         uint16_t min_sop, max_sop, min_data, max_data;
 584         uint16_t mtu = enic->rte_dev->data->mtu;
 585
 586         rq_sop->is_sop = 1;
 587         rq_sop->data_queue_idx = data_queue_idx;
 588         rq_data->is_sop = 0;
 589         rq_data->data_queue_idx = 0;
 590         rq_sop->socket_id = socket_id;
 591         rq_sop->mp = mp;
 592         rq_data->socket_id = socket_id;
 593         rq_data->mp = mp;
 594         rq_sop->in_use = 1;
 595         rq_sop->rx_free_thresh = free_thresh;
 596         rq_data->rx_free_thresh = free_thresh;
 597         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 598                   free_thresh);
 599
 600         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 601                                RTE_PKTMBUF_HEADROOM);
 602
 603         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 604             DEV_RX_OFFLOAD_SCATTER) {
 605                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 606                 /* ceil((mtu + ETHER_HDR_LEN + 4)/mbuf_size) */
 607                 mbufs_per_pkt = ((mtu + ETHER_HDR_LEN + 4) +
 608                                  (mbuf_size - 1)) / mbuf_size;
 609         } else {
 610                 dev_info(enic, "Scatter rx mode disabled\n");
 611                 mbufs_per_pkt = 1;
 612         }
 613
 614         if (mbufs_per_pkt > 1) {
 615                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 616                 rq_sop->data_queue_enable = 1;
 617                 rq_data->in_use = 1;
 618         } else {
 619                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 620                          queue_idx);
 621                 rq_sop->data_queue_enable = 0;
 622                 rq_data->in_use = 0;
 623         }
 624
 625         /* number of descriptors have to be a multiple of 32 */
 626         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
 627         nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
 628
 629         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 630         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 631
 632         if (mbufs_per_pkt > 1) {
 633                 min_sop = 64;
 634                 max_sop = ((enic->config.rq_desc_count /
 635                             (mbufs_per_pkt - 1)) & ~0x1F);
 636                 min_data = min_sop * (mbufs_per_pkt - 1);
 637                 max_data = enic->config.rq_desc_count;
 638         } else {
 639                 min_sop = 64;
 640                 max_sop = enic->config.rq_desc_count;
 641                 min_data = 0;
 642                 max_data = 0;
 643         }
 644
 645         if (nb_desc < (min_sop + min_data)) {
 646                 dev_warning(enic,
 647                             "Number of rx descs too low, adjusting to minimum\n");
 648                 nb_sop_desc = min_sop;
 649                 nb_data_desc = min_data;
 650         } else if (nb_desc > (max_sop + max_data)) {
 651                 dev_warning(enic,
 652                             "Number of rx_descs too high, adjusting to maximum\n");
 653                 nb_sop_desc = max_sop;
 654                 nb_data_desc = max_data;
 655         }
 656         if (mbufs_per_pkt > 1) {
 657                 dev_info(enic, "For mtu %d and mbuf size %d valid rx descriptor range is %d to %d\n",
 658                          mtu, mbuf_size, min_sop + min_data,
 659                          max_sop + max_data);
 660         }
 661         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 662                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 663
 664         /* Allocate sop queue resources */
 665         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 666                 nb_sop_desc, sizeof(struct rq_enet_desc));
 667         if (rc) {
 668                 dev_err(enic, "error in allocation of sop rq\n");
 669                 goto err_exit;
 670         }
 671         nb_sop_desc = rq_sop->ring.desc_count;
 672
 673         if (rq_data->in_use) {
 674                 /* Allocate data queue resources */
 675                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 676                                    nb_data_desc,
 677                                    sizeof(struct rq_enet_desc));
 678                 if (rc) {
 679                         dev_err(enic, "error in allocation of data rq\n");
 680                         goto err_free_rq_sop;
 681                 }
 682                 nb_data_desc = rq_data->ring.desc_count;
 683         }
 684         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 685                            socket_id, nb_sop_desc + nb_data_desc,
 686                            sizeof(struct cq_enet_rq_desc));
 687         if (rc) {
 688                 dev_err(enic, "error in allocation of cq for rq\n");
 689                 goto err_free_rq_data;
 690         }
 691
 692         /* Allocate the mbuf rings */
 693         rq_sop->mbuf_ring = (struct rte_mbuf **)
 694                 rte_zmalloc_socket("rq->mbuf_ring",
 695                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 696                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 697         if (rq_sop->mbuf_ring == NULL)
 698                 goto err_free_cq;
 699
 700         if (rq_data->in_use) {
 701                 rq_data->mbuf_ring = (struct rte_mbuf **)
 702                         rte_zmalloc_socket("rq->mbuf_ring",
 703                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 704                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 705                 if (rq_data->mbuf_ring == NULL)
 706                         goto err_free_sop_mbuf;
 707         }
 708
 709         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 710
 711         return 0;
 712
 713 err_free_sop_mbuf:
 714         rte_free(rq_sop->mbuf_ring);
 715 err_free_cq:
 716         /* cleanup on error */
 717         vnic_cq_free(&enic->cq[queue_idx]);
 718 err_free_rq_data:
 719         if (rq_data->in_use)
 720                 vnic_rq_free(rq_data);
 721 err_free_rq_sop:
 722         vnic_rq_free(rq_sop);
 723 err_exit:
 724         return -ENOMEM;
 725 }
 726
 727 void enic_free_wq(void *txq)
 728 {
 729         struct vnic_wq *wq;
 730         struct enic *enic;
 731
 732         if (txq == NULL)
 733                 return;
 734
 735         wq = (struct vnic_wq *)txq;
 736         enic = vnic_dev_priv(wq->vdev);
 737         rte_memzone_free(wq->cqmsg_rz);
 738         vnic_wq_free(wq);
 739         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 740 }
 741
 742 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 743         unsigned int socket_id, uint16_t nb_desc)
 744 {
 745         int err;
 746         struct vnic_wq *wq = &enic->wq[queue_idx];
 747         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 748         char name[NAME_MAX];
 749         static int instance;
 750
 751         wq->socket_id = socket_id;
 752         if (nb_desc) {
 753                 if (nb_desc > enic->config.wq_desc_count) {
 754                         dev_warning(enic,
 755                                 "WQ %d - number of tx desc in cmd line (%d)"\
 756                                 "is greater than that in the UCSM/CIMC adapter"\
 757                                 "policy.  Applying the value in the adapter "\
 758                                 "policy (%d)\n",
 759                                 queue_idx, nb_desc, enic->config.wq_desc_count);
 760                 } else if (nb_desc != enic->config.wq_desc_count) {
 761                         enic->config.wq_desc_count = nb_desc;
 762                         dev_info(enic,
 763                                 "TX Queues - effective number of descs:%d\n",
 764                                 nb_desc);
 765                 }
 766         }
 767
 768         /* Allocate queue resources */
 769         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
 770                 enic->config.wq_desc_count,
 771                 sizeof(struct wq_enet_desc));
 772         if (err) {
 773                 dev_err(enic, "error in allocation of wq\n");
 774                 return err;
 775         }
 776
 777         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
 778                 socket_id, enic->config.wq_desc_count,
 779                 sizeof(struct cq_enet_wq_desc));
 780         if (err) {
 781                 vnic_wq_free(wq);
 782                 dev_err(enic, "error in allocation of cq for wq\n");
 783         }
 784
 785         /* setup up CQ message */
 786         snprintf((char *)name, sizeof(name),
 787                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
 788                 instance++);
 789
 790         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
 791                                                    sizeof(uint32_t),
 792                                                    SOCKET_ID_ANY, 0,
 793                                                    ENIC_ALIGN);
 794         if (!wq->cqmsg_rz)
 795                 return -ENOMEM;
 796
 797         return err;
 798 }
 799
 800 int enic_disable(struct enic *enic)
 801 {
 802         unsigned int i;
 803         int err;
 804
 805         vnic_intr_mask(&enic->intr);
 806         (void)vnic_intr_masked(&enic->intr); /* flush write */
 807         rte_intr_disable(&enic->pdev->intr_handle);
 808         rte_intr_callback_unregister(&enic->pdev->intr_handle,
 809                                      enic_intr_handler,
 810                                      (void *)enic->rte_dev);
 811
 812         vnic_dev_disable(enic->vdev);
 813
 814         enic_clsf_destroy(enic);
 815
 816         if (!enic_is_sriov_vf(enic))
 817                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
 818
 819         for (i = 0; i < enic->wq_count; i++) {
 820                 err = vnic_wq_disable(&enic->wq[i]);
 821                 if (err)
 822                         return err;
 823         }
 824         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
 825                 if (enic->rq[i].in_use) {
 826                         err = vnic_rq_disable(&enic->rq[i]);
 827                         if (err)
 828                                 return err;
 829                 }
 830         }
 831
 832         /* If we were using interrupts, set the interrupt vector to -1
 833          * to disable interrupts.  We are not disabling link notifcations,
 834          * though, as we want the polling of link status to continue working.
 835          */
 836         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
 837                 vnic_dev_notify_set(enic->vdev, -1);
 838
 839         vnic_dev_set_reset_flag(enic->vdev, 1);
 840
 841         for (i = 0; i < enic->wq_count; i++)
 842                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
 843
 844         for (i = 0; i < enic_vnic_rq_count(enic); i++)
 845                 if (enic->rq[i].in_use)
 846                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
 847         for (i = 0; i < enic->cq_count; i++)
 848                 vnic_cq_clean(&enic->cq[i]);
 849         vnic_intr_clean(&enic->intr);
 850
 851         return 0;
 852 }
 853
 854 static int enic_dev_wait(struct vnic_dev *vdev,
 855         int (*start)(struct vnic_dev *, int),
 856         int (*finished)(struct vnic_dev *, int *),
 857         int arg)
 858 {
 859         int done;
 860         int err;
 861         int i;
 862
 863         err = start(vdev, arg);
 864         if (err)
 865                 return err;
 866
 867         /* Wait for func to complete...2 seconds max */
 868         for (i = 0; i < 2000; i++) {
 869                 err = finished(vdev, &done);
 870                 if (err)
 871                         return err;
 872                 if (done)
 873                         return 0;
 874                 usleep(1000);
 875         }
 876         return -ETIMEDOUT;
 877 }
 878
 879 static int enic_dev_open(struct enic *enic)
 880 {
 881         int err;
 882
 883         err = enic_dev_wait(enic->vdev, vnic_dev_open,
 884                 vnic_dev_open_done, 0);
 885         if (err)
 886                 dev_err(enic_get_dev(enic),
 887                         "vNIC device open failed, err %d\n", err);
 888
 889         return err;
 890 }
 891
 892 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
 893 {
 894         dma_addr_t rss_key_buf_pa;
 895         union vnic_rss_key *rss_key_buf_va = NULL;
 896         int err, i;
 897         u8 name[NAME_MAX];
 898
 899         RTE_ASSERT(user_key != NULL);
 900         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
 901         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
 902                 &rss_key_buf_pa, name);
 903         if (!rss_key_buf_va)
 904                 return -ENOMEM;
 905
 906         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
 907                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
 908
 909         err = enic_set_rss_key(enic,
 910                 rss_key_buf_pa,
 911                 sizeof(union vnic_rss_key));
 912
 913         /* Save for later queries */
 914         if (!err) {
 915                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
 916                            sizeof(union vnic_rss_key));
 917         }
 918         enic_free_consistent(enic, sizeof(union vnic_rss_key),
 919                 rss_key_buf_va, rss_key_buf_pa);
 920
 921         return err;
 922 }
 923
 924 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
 925 {
 926         dma_addr_t rss_cpu_buf_pa;
 927         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
 928         int err;
 929         u8 name[NAME_MAX];
 930
 931         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
 932         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
 933                 &rss_cpu_buf_pa, name);
 934         if (!rss_cpu_buf_va)
 935                 return -ENOMEM;
 936
 937         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
 938
 939         err = enic_set_rss_cpu(enic,
 940                 rss_cpu_buf_pa,
 941                 sizeof(union vnic_rss_cpu));
 942
 943         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
 944                 rss_cpu_buf_va, rss_cpu_buf_pa);
 945
 946         /* Save for later queries */
 947         if (!err)
 948                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
 949         return err;
 950 }
 951
 952 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
 953         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
 954 {
 955         const u8 tso_ipid_split_en = 0;
 956         int err;
 957
 958         err = enic_set_nic_cfg(enic,
 959                 rss_default_cpu, rss_hash_type,
 960                 rss_hash_bits, rss_base_cpu,
 961                 rss_enable, tso_ipid_split_en,
 962                 enic->ig_vlan_strip_en);
 963
 964         return err;
 965 }
 966
 967 /* Initialize RSS with defaults, called from dev_configure */
 968 int enic_init_rss_nic_cfg(struct enic *enic)
 969 {
 970         static uint8_t default_rss_key[] = {
 971                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
 972                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
 973                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
 974                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
 975         };
 976         struct rte_eth_rss_conf rss_conf;
 977         union vnic_rss_cpu rss_cpu;
 978         int ret, i;
 979
 980         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
 981         /*
 982          * If setting key for the first time, and the user gives us none, then
 983          * push the default key to NIC.
 984          */
 985         if (rss_conf.rss_key == NULL) {
 986                 rss_conf.rss_key = default_rss_key;
 987                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
 988         }
 989         ret = enic_set_rss_conf(enic, &rss_conf);
 990         if (ret) {
 991                 dev_err(enic, "Failed to configure RSS\n");
 992                 return ret;
 993         }
 994         if (enic->rss_enable) {
 995                 /* If enabling RSS, use the default reta */
 996                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
 997                         rss_cpu.cpu[i / 4].b[i % 4] =
 998                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
 999                 }
1000                 ret = enic_set_rss_reta(enic, &rss_cpu);
1001                 if (ret)
1002                         dev_err(enic, "Failed to set RSS indirection table\n");
1003         }
1004         return ret;
1005 }
1006
1007 int enic_setup_finish(struct enic *enic)
1008 {
1009         enic_init_soft_stats(enic);
1010
1011         /* Default conf */
1012         vnic_dev_packet_filter(enic->vdev,
1013                 1 /* directed  */,
1014                 1 /* multicast */,
1015                 1 /* broadcast */,
1016                 0 /* promisc   */,
1017                 1 /* allmulti  */);
1018
1019         enic->promisc = 0;
1020         enic->allmulti = 1;
1021
1022         return 0;
1023 }
1024
1025 static int enic_rss_conf_valid(struct enic *enic,
1026                                struct rte_eth_rss_conf *rss_conf)
1027 {
1028         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1029         if (enic->flow_type_rss_offloads == 0)
1030                 return 0;
1031         if (rss_conf->rss_key != NULL &&
1032             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1033                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1034                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1035                 return -EINVAL;
1036         }
1037         if (rss_conf->rss_hf != 0 &&
1038             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1039                 dev_err(enic, "Given rss_hf contains none of the supported"
1040                         " types\n");
1041                 return -EINVAL;
1042         }
1043         return 0;
1044 }
1045
1046 /* Set hash type and key according to rss_conf */
1047 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1048 {
1049         struct rte_eth_dev *eth_dev;
1050         uint64_t rss_hf;
1051         u8 rss_hash_type;
1052         u8 rss_enable;
1053         int ret;
1054
1055         RTE_ASSERT(rss_conf != NULL);
1056         ret = enic_rss_conf_valid(enic, rss_conf);
1057         if (ret) {
1058                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1059                 return ret;
1060         }
1061
1062         eth_dev = enic->rte_dev;
1063         rss_hash_type = 0;
1064         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1065         if (enic->rq_count > 1 &&
1066             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1067             rss_hf != 0) {
1068                 rss_enable = 1;
1069                 if (rss_hf & ETH_RSS_IPV4)
1070                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1071                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1072                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1073                 if (rss_hf & ETH_RSS_IPV6)
1074                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1075                 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1076                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1077                 if (rss_hf & ETH_RSS_IPV6_EX)
1078                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6_EX;
1079                 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
1080                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX;
1081         } else {
1082                 rss_enable = 0;
1083                 rss_hf = 0;
1084         }
1085
1086         /* Set the hash key if provided */
1087         if (rss_enable && rss_conf->rss_key) {
1088                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1089                 if (ret) {
1090                         dev_err(enic, "Failed to set RSS key\n");
1091                         return ret;
1092                 }
1093         }
1094
1095         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1096                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1097                               rss_enable);
1098         if (!ret) {
1099                 enic->rss_hf = rss_hf;
1100                 enic->rss_hash_type = rss_hash_type;
1101                 enic->rss_enable = rss_enable;
1102         }
1103         return 0;
1104 }
1105
1106 int enic_set_vlan_strip(struct enic *enic)
1107 {
1108         /*
1109          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1110          * together. So, re-do niccfg, preserving the current RSS settings.
1111          */
1112         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1113                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1114                                enic->rss_enable);
1115 }
1116
1117 void enic_add_packet_filter(struct enic *enic)
1118 {
1119         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1120         vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1121                 enic->promisc, enic->allmulti);
1122 }
1123
1124 int enic_get_link_status(struct enic *enic)
1125 {
1126         return vnic_dev_link_status(enic->vdev);
1127 }
1128
1129 static void enic_dev_deinit(struct enic *enic)
1130 {
1131         struct rte_eth_dev *eth_dev = enic->rte_dev;
1132
1133         /* stop link status checking */
1134         vnic_dev_notify_unset(enic->vdev);
1135
1136         rte_free(eth_dev->data->mac_addrs);
1137         rte_free(enic->cq);
1138         rte_free(enic->rq);
1139         rte_free(enic->wq);
1140 }
1141
1142
1143 int enic_set_vnic_res(struct enic *enic)
1144 {
1145         struct rte_eth_dev *eth_dev = enic->rte_dev;
1146         int rc = 0;
1147         unsigned int required_rq, required_wq, required_cq;
1148
1149         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1150         required_rq = eth_dev->data->nb_rx_queues * 2;
1151         required_wq = eth_dev->data->nb_tx_queues;
1152         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1153
1154         if (enic->conf_rq_count < required_rq) {
1155                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1156                         eth_dev->data->nb_rx_queues,
1157                         required_rq, enic->conf_rq_count);
1158                 rc = -EINVAL;
1159         }
1160         if (enic->conf_wq_count < required_wq) {
1161                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1162                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1163                 rc = -EINVAL;
1164         }
1165
1166         if (enic->conf_cq_count < required_cq) {
1167                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1168                         required_cq, enic->conf_cq_count);
1169                 rc = -EINVAL;
1170         }
1171
1172         if (rc == 0) {
1173                 enic->rq_count = eth_dev->data->nb_rx_queues;
1174                 enic->wq_count = eth_dev->data->nb_tx_queues;
1175                 enic->cq_count = enic->rq_count + enic->wq_count;
1176         }
1177
1178         return rc;
1179 }
1180
1181 /* Initialize the completion queue for an RQ */
1182 static int
1183 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1184 {
1185         struct vnic_rq *sop_rq, *data_rq;
1186         unsigned int cq_idx;
1187         int rc = 0;
1188
1189         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1190         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1191         cq_idx = rq_idx;
1192
1193         vnic_cq_clean(&enic->cq[cq_idx]);
1194         vnic_cq_init(&enic->cq[cq_idx],
1195                      0 /* flow_control_enable */,
1196                      1 /* color_enable */,
1197                      0 /* cq_head */,
1198                      0 /* cq_tail */,
1199                      1 /* cq_tail_color */,
1200                      0 /* interrupt_enable */,
1201                      1 /* cq_entry_enable */,
1202                      0 /* cq_message_enable */,
1203                      0 /* interrupt offset */,
1204                      0 /* cq_message_addr */);
1205
1206
1207         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1208                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1209                            sop_rq->ring.desc_count - 1, 1, 0);
1210         if (data_rq->in_use) {
1211                 vnic_rq_init_start(data_rq,
1212                                    enic_cq_rq(enic,
1213                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1214                                    data_rq->ring.desc_count - 1, 1, 0);
1215         }
1216
1217         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1218         if (rc)
1219                 return rc;
1220
1221         if (data_rq->in_use) {
1222                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1223                 if (rc) {
1224                         enic_rxmbuf_queue_release(enic, sop_rq);
1225                         return rc;
1226                 }
1227         }
1228
1229         return 0;
1230 }
1231
1232 /* The Cisco NIC can send and receive packets up to a max packet size
1233  * determined by the NIC type and firmware. There is also an MTU
1234  * configured into the NIC via the CIMC/UCSM management interface
1235  * which can be overridden by this function (up to the max packet size).
1236  * Depending on the network setup, doing so may cause packet drops
1237  * and unexpected behavior.
1238  */
1239 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1240 {
1241         unsigned int rq_idx;
1242         struct vnic_rq *rq;
1243         int rc = 0;
1244         uint16_t old_mtu;       /* previous setting */
1245         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1246         struct rte_eth_dev *eth_dev = enic->rte_dev;
1247
1248         old_mtu = eth_dev->data->mtu;
1249         config_mtu = enic->config.mtu;
1250
1251         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1252                 return -E_RTE_SECONDARY;
1253
1254         if (new_mtu > enic->max_mtu) {
1255                 dev_err(enic,
1256                         "MTU not updated: requested (%u) greater than max (%u)\n",
1257                         new_mtu, enic->max_mtu);
1258                 return -EINVAL;
1259         }
1260         if (new_mtu < ENIC_MIN_MTU) {
1261                 dev_info(enic,
1262                         "MTU not updated: requested (%u) less than min (%u)\n",
1263                         new_mtu, ENIC_MIN_MTU);
1264                 return -EINVAL;
1265         }
1266         if (new_mtu > config_mtu)
1267                 dev_warning(enic,
1268                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1269                         new_mtu, config_mtu);
1270
1271         /* The easy case is when scatter is disabled. However if the MTU
1272          * becomes greater than the mbuf data size, packet drops will ensue.
1273          */
1274         if (!(enic->rte_dev->data->dev_conf.rxmode.offloads &
1275               DEV_RX_OFFLOAD_SCATTER)) {
1276                 eth_dev->data->mtu = new_mtu;
1277                 goto set_mtu_done;
1278         }
1279
1280         /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to
1281          * change Rx scatter mode if necessary for better performance. I.e. if
1282          * MTU was greater than the mbuf size and now it's less, scatter Rx
1283          * doesn't have to be used and vice versa.
1284           */
1285         rte_spinlock_lock(&enic->mtu_lock);
1286
1287         /* Stop traffic on all RQs */
1288         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1289                 rq = &enic->rq[rq_idx];
1290                 if (rq->is_sop && rq->in_use) {
1291                         rc = enic_stop_rq(enic,
1292                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1293                         if (rc) {
1294                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1295                                 goto set_mtu_done;
1296                         }
1297                 }
1298         }
1299
1300         /* replace Rx function with a no-op to avoid getting stale pkts */
1301         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1302         rte_mb();
1303
1304         /* Allow time for threads to exit the real Rx function. */
1305         usleep(100000);
1306
1307         /* now it is safe to reconfigure the RQs */
1308
1309         /* update the mtu */
1310         eth_dev->data->mtu = new_mtu;
1311
1312         /* free and reallocate RQs with the new MTU */
1313         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1314                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1315
1316                 enic_free_rq(rq);
1317                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1318                                    rq->tot_nb_desc, rq->rx_free_thresh);
1319                 if (rc) {
1320                         dev_err(enic,
1321                                 "Fatal MTU alloc error- No traffic will pass\n");
1322                         goto set_mtu_done;
1323                 }
1324
1325                 rc = enic_reinit_rq(enic, rq_idx);
1326                 if (rc) {
1327                         dev_err(enic,
1328                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1329                         goto set_mtu_done;
1330                 }
1331         }
1332
1333         /* put back the real receive function */
1334         rte_mb();
1335         eth_dev->rx_pkt_burst = enic_recv_pkts;
1336         rte_mb();
1337
1338         /* restart Rx traffic */
1339         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1340                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1341                 if (rq->is_sop && rq->in_use)
1342                         enic_start_rq(enic, rq_idx);
1343         }
1344
1345 set_mtu_done:
1346         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1347         rte_spinlock_unlock(&enic->mtu_lock);
1348         return rc;
1349 }
1350
1351 static int enic_dev_init(struct enic *enic)
1352 {
1353         int err;
1354         struct rte_eth_dev *eth_dev = enic->rte_dev;
1355
1356         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1357
1358         /* Get vNIC configuration
1359         */
1360         err = enic_get_vnic_config(enic);
1361         if (err) {
1362                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1363                 return err;
1364         }
1365
1366         /* Get available resource counts */
1367         enic_get_res_counts(enic);
1368         if (enic->conf_rq_count == 1) {
1369                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1370                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1371                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1372                 return -EINVAL;
1373         }
1374         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1375         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1376                                enic->conf_cq_count, 8);
1377         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1378                                enic->conf_rq_count, 8);
1379         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1380                                enic->conf_wq_count, 8);
1381         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1382                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1383                 return -1;
1384         }
1385         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1386                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1387                 return -1;
1388         }
1389         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1390                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1391                 return -1;
1392         }
1393
1394         /* Get the supported filters */
1395         enic_fdir_info(enic);
1396
1397         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1398                                                 * ENIC_MAX_MAC_ADDR, 0);
1399         if (!eth_dev->data->mac_addrs) {
1400                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1401                 return -1;
1402         }
1403         ether_addr_copy((struct ether_addr *) enic->mac_addr,
1404                         eth_dev->data->mac_addrs);
1405
1406         vnic_dev_set_reset_flag(enic->vdev, 0);
1407
1408         LIST_INIT(&enic->flows);
1409         rte_spinlock_init(&enic->flows_lock);
1410
1411         /* set up link status checking */
1412         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1413
1414         return 0;
1415
1416 }
1417
1418 int enic_probe(struct enic *enic)
1419 {
1420         struct rte_pci_device *pdev = enic->pdev;
1421         int err = -1;
1422
1423         dev_debug(enic, " Initializing ENIC PMD\n");
1424
1425         /* if this is a secondary process the hardware is already initialized */
1426         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1427                 return 0;
1428
1429         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1430         enic->bar0.len = pdev->mem_resource[0].len;
1431
1432         /* Register vNIC device */
1433         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1434         if (!enic->vdev) {
1435                 dev_err(enic, "vNIC registration failed, aborting\n");
1436                 goto err_out;
1437         }
1438
1439         LIST_INIT(&enic->memzone_list);
1440         rte_spinlock_init(&enic->memzone_list_lock);
1441
1442         vnic_register_cbacks(enic->vdev,
1443                 enic_alloc_consistent,
1444                 enic_free_consistent);
1445
1446         /* Issue device open to get device in known state */
1447         err = enic_dev_open(enic);
1448         if (err) {
1449                 dev_err(enic, "vNIC dev open failed, aborting\n");
1450                 goto err_out_unregister;
1451         }
1452
1453         /* Set ingress vlan rewrite mode before vnic initialization */
1454         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1455                 IG_VLAN_REWRITE_MODE_PASS_THRU);
1456         if (err) {
1457                 dev_err(enic,
1458                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1459                 goto err_out_dev_close;
1460         }
1461
1462         /* Issue device init to initialize the vnic-to-switch link.
1463          * We'll start with carrier off and wait for link UP
1464          * notification later to turn on carrier.  We don't need
1465          * to wait here for the vnic-to-switch link initialization
1466          * to complete; link UP notification is the indication that
1467          * the process is complete.
1468          */
1469
1470         err = vnic_dev_init(enic->vdev, 0);
1471         if (err) {
1472                 dev_err(enic, "vNIC dev init failed, aborting\n");
1473                 goto err_out_dev_close;
1474         }
1475
1476         err = enic_dev_init(enic);
1477         if (err) {
1478                 dev_err(enic, "Device initialization failed, aborting\n");
1479                 goto err_out_dev_close;
1480         }
1481
1482         return 0;
1483
1484 err_out_dev_close:
1485         vnic_dev_close(enic->vdev);
1486 err_out_unregister:
1487         vnic_dev_unregister(enic->vdev);
1488 err_out:
1489         return err;
1490 }
1491
1492 void enic_remove(struct enic *enic)
1493 {
1494         enic_dev_deinit(enic);
1495         vnic_dev_close(enic->vdev);
1496         vnic_dev_unregister(enic->vdev);
1497 }