drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11 #include <libgen.h>
  12
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_ethdev_driver.h>
  20
  21 #include "enic_compat.h"
  22 #include "enic.h"
  23 #include "wq_enet_desc.h"
  24 #include "rq_enet_desc.h"
  25 #include "cq_enet_desc.h"
  26 #include "vnic_enet.h"
  27 #include "vnic_dev.h"
  28 #include "vnic_wq.h"
  29 #include "vnic_rq.h"
  30 #include "vnic_cq.h"
  31 #include "vnic_intr.h"
  32 #include "vnic_nic.h"
  33
  34 static inline int enic_is_sriov_vf(struct enic *enic)
  35 {
  36         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  37 }
  38
  39 static int is_zero_addr(uint8_t *addr)
  40 {
  41         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  42 }
  43
  44 static int is_mcast_addr(uint8_t *addr)
  45 {
  46         return addr[0] & 1;
  47 }
  48
  49 static int is_eth_addr_valid(uint8_t *addr)
  50 {
  51         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  52 }
  53
  54 static void
  55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  56 {
  57         uint16_t i;
  58
  59         if (!rq || !rq->mbuf_ring) {
  60                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  61                 return;
  62         }
  63
  64         for (i = 0; i < rq->ring.desc_count; i++) {
  65                 if (rq->mbuf_ring[i]) {
  66                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  67                         rq->mbuf_ring[i] = NULL;
  68                 }
  69         }
  70 }
  71
  72 static void enic_free_wq_buf(struct vnic_wq_buf *buf)
  73 {
  74         struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
  75
  76         rte_pktmbuf_free_seg(mbuf);
  77         buf->mb = NULL;
  78 }
  79
  80 static void enic_log_q_error(struct enic *enic)
  81 {
  82         unsigned int i;
  83         u32 error_status;
  84
  85         for (i = 0; i < enic->wq_count; i++) {
  86                 error_status = vnic_wq_error_status(&enic->wq[i]);
  87                 if (error_status)
  88                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  89                                 error_status);
  90         }
  91
  92         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  93                 if (!enic->rq[i].in_use)
  94                         continue;
  95                 error_status = vnic_rq_error_status(&enic->rq[i]);
  96                 if (error_status)
  97                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  98                                 error_status);
  99         }
 100 }
 101
 102 static void enic_clear_soft_stats(struct enic *enic)
 103 {
 104         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 105         rte_atomic64_clear(&soft_stats->rx_nombuf);
 106         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 107         rte_atomic64_clear(&soft_stats->tx_oversized);
 108 }
 109
 110 static void enic_init_soft_stats(struct enic *enic)
 111 {
 112         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 113         rte_atomic64_init(&soft_stats->rx_nombuf);
 114         rte_atomic64_init(&soft_stats->rx_packet_errors);
 115         rte_atomic64_init(&soft_stats->tx_oversized);
 116         enic_clear_soft_stats(enic);
 117 }
 118
 119 void enic_dev_stats_clear(struct enic *enic)
 120 {
 121         if (vnic_dev_stats_clear(enic->vdev))
 122                 dev_err(enic, "Error in clearing stats\n");
 123         enic_clear_soft_stats(enic);
 124 }
 125
 126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 127 {
 128         struct vnic_stats *stats;
 129         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 130         int64_t rx_truncated;
 131         uint64_t rx_packet_errors;
 132         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 133
 134         if (ret) {
 135                 dev_err(enic, "Error in getting stats\n");
 136                 return ret;
 137         }
 138
 139         /* The number of truncated packets can only be calculated by
 140          * subtracting a hardware counter from error packets received by
 141          * the driver. Note: this causes transient inaccuracies in the
 142          * ipackets count. Also, the length of truncated packets are
 143          * counted in ibytes even though truncated packets are dropped
 144          * which can make ibytes be slightly higher than it should be.
 145          */
 146         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 147         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 148
 149         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 150         r_stats->opackets = stats->tx.tx_frames_ok;
 151
 152         r_stats->ibytes = stats->rx.rx_bytes_ok;
 153         r_stats->obytes = stats->tx.tx_bytes_ok;
 154
 155         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 156         r_stats->oerrors = stats->tx.tx_errors
 157                            + rte_atomic64_read(&soft_stats->tx_oversized);
 158
 159         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 160
 161         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 162         return 0;
 163 }
 164
 165 void enic_del_mac_address(struct enic *enic, int mac_index)
 166 {
 167         struct rte_eth_dev *eth_dev = enic->rte_dev;
 168         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 169
 170         if (vnic_dev_del_addr(enic->vdev, mac_addr))
 171                 dev_err(enic, "del mac addr failed\n");
 172 }
 173
 174 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 175 {
 176         int err;
 177
 178         if (!is_eth_addr_valid(mac_addr)) {
 179                 dev_err(enic, "invalid mac address\n");
 180                 return -EINVAL;
 181         }
 182
 183         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 184         if (err)
 185                 dev_err(enic, "add mac addr failed\n");
 186         return err;
 187 }
 188
 189 static void
 190 enic_free_rq_buf(struct rte_mbuf **mbuf)
 191 {
 192         if (*mbuf == NULL)
 193                 return;
 194
 195         rte_pktmbuf_free(*mbuf);
 196         *mbuf = NULL;
 197 }
 198
 199 void enic_init_vnic_resources(struct enic *enic)
 200 {
 201         unsigned int error_interrupt_enable = 1;
 202         unsigned int error_interrupt_offset = 0;
 203         unsigned int index = 0;
 204         unsigned int cq_idx;
 205         struct vnic_rq *data_rq;
 206
 207         for (index = 0; index < enic->rq_count; index++) {
 208                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 209
 210                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 211                         cq_idx,
 212                         error_interrupt_enable,
 213                         error_interrupt_offset);
 214
 215                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 216                 if (data_rq->in_use)
 217                         vnic_rq_init(data_rq,
 218                                      cq_idx,
 219                                      error_interrupt_enable,
 220                                      error_interrupt_offset);
 221
 222                 vnic_cq_init(&enic->cq[cq_idx],
 223                         0 /* flow_control_enable */,
 224                         1 /* color_enable */,
 225                         0 /* cq_head */,
 226                         0 /* cq_tail */,
 227                         1 /* cq_tail_color */,
 228                         0 /* interrupt_enable */,
 229                         1 /* cq_entry_enable */,
 230                         0 /* cq_message_enable */,
 231                         0 /* interrupt offset */,
 232                         0 /* cq_message_addr */);
 233         }
 234
 235         for (index = 0; index < enic->wq_count; index++) {
 236                 vnic_wq_init(&enic->wq[index],
 237                         enic_cq_wq(enic, index),
 238                         error_interrupt_enable,
 239                         error_interrupt_offset);
 240
 241                 cq_idx = enic_cq_wq(enic, index);
 242                 vnic_cq_init(&enic->cq[cq_idx],
 243                         0 /* flow_control_enable */,
 244                         1 /* color_enable */,
 245                         0 /* cq_head */,
 246                         0 /* cq_tail */,
 247                         1 /* cq_tail_color */,
 248                         0 /* interrupt_enable */,
 249                         0 /* cq_entry_enable */,
 250                         1 /* cq_message_enable */,
 251                         0 /* interrupt offset */,
 252                         (u64)enic->wq[index].cqmsg_rz->iova);
 253         }
 254
 255         vnic_intr_init(&enic->intr,
 256                 enic->config.intr_timer_usec,
 257                 enic->config.intr_timer_type,
 258                 /*mask_on_assertion*/1);
 259 }
 260
 261
 262 static int
 263 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 264 {
 265         struct rte_mbuf *mb;
 266         struct rq_enet_desc *rqd = rq->ring.descs;
 267         unsigned i;
 268         dma_addr_t dma_addr;
 269         uint32_t max_rx_pkt_len;
 270         uint16_t rq_buf_len;
 271
 272         if (!rq->in_use)
 273                 return 0;
 274
 275         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 276                   rq->ring.desc_count);
 277
 278         /*
 279          * If *not* using scatter and the mbuf size is smaller than the
 280          * requested max packet size (max_rx_pkt_len), then reduce the
 281          * posted buffer size to max_rx_pkt_len. HW still receives packets
 282          * larger than max_rx_pkt_len, but they will be truncated, which we
 283          * drop in the rx handler. Not ideal, but better than returning
 284          * large packets when the user is not expecting them.
 285          */
 286         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 287         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 288         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 289                 rq_buf_len = max_rx_pkt_len;
 290         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 291                 mb = rte_mbuf_raw_alloc(rq->mp);
 292                 if (mb == NULL) {
 293                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 294                         (unsigned)rq->index);
 295                         return -ENOMEM;
 296                 }
 297
 298                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 299                 dma_addr = (dma_addr_t)(mb->buf_iova
 300                            + RTE_PKTMBUF_HEADROOM);
 301                 rq_enet_desc_enc(rqd, dma_addr,
 302                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 303                                 : RQ_ENET_TYPE_NOT_SOP),
 304                                 rq_buf_len);
 305                 rq->mbuf_ring[i] = mb;
 306         }
 307
 308         /* make sure all prior writes are complete before doing the PIO write */
 309         rte_rmb();
 310
 311         /* Post all but the last buffer to VIC. */
 312         rq->posted_index = rq->ring.desc_count - 1;
 313
 314         rq->rx_nb_hold = 0;
 315
 316         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 317                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 318         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 319         iowrite32(0, &rq->ctrl->fetch_index);
 320         rte_rmb();
 321
 322         return 0;
 323
 324 }
 325
 326 static void *
 327 enic_alloc_consistent(void *priv, size_t size,
 328         dma_addr_t *dma_handle, u8 *name)
 329 {
 330         void *vaddr;
 331         const struct rte_memzone *rz;
 332         *dma_handle = 0;
 333         struct enic *enic = (struct enic *)priv;
 334         struct enic_memzone_entry *mze;
 335
 336         rz = rte_memzone_reserve_aligned((const char *)name,
 337                                          size, SOCKET_ID_ANY, 0, ENIC_ALIGN);
 338         if (!rz) {
 339                 pr_err("%s : Failed to allocate memory requested for %s\n",
 340                         __func__, name);
 341                 return NULL;
 342         }
 343
 344         vaddr = rz->addr;
 345         *dma_handle = (dma_addr_t)rz->iova;
 346
 347         mze = rte_malloc("enic memzone entry",
 348                          sizeof(struct enic_memzone_entry), 0);
 349
 350         if (!mze) {
 351                 pr_err("%s : Failed to allocate memory for memzone list\n",
 352                        __func__);
 353                 rte_memzone_free(rz);
 354                 return NULL;
 355         }
 356
 357         mze->rz = rz;
 358
 359         rte_spinlock_lock(&enic->memzone_list_lock);
 360         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 361         rte_spinlock_unlock(&enic->memzone_list_lock);
 362
 363         return vaddr;
 364 }
 365
 366 static void
 367 enic_free_consistent(void *priv,
 368                      __rte_unused size_t size,
 369                      void *vaddr,
 370                      dma_addr_t dma_handle)
 371 {
 372         struct enic_memzone_entry *mze;
 373         struct enic *enic = (struct enic *)priv;
 374
 375         rte_spinlock_lock(&enic->memzone_list_lock);
 376         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 377                 if (mze->rz->addr == vaddr &&
 378                     mze->rz->iova == dma_handle)
 379                         break;
 380         }
 381         if (mze == NULL) {
 382                 rte_spinlock_unlock(&enic->memzone_list_lock);
 383                 dev_warning(enic,
 384                             "Tried to free memory, but couldn't find it in the memzone list\n");
 385                 return;
 386         }
 387         LIST_REMOVE(mze, entries);
 388         rte_spinlock_unlock(&enic->memzone_list_lock);
 389         rte_memzone_free(mze->rz);
 390         rte_free(mze);
 391 }
 392
 393 int enic_link_update(struct enic *enic)
 394 {
 395         struct rte_eth_dev *eth_dev = enic->rte_dev;
 396         int ret;
 397         int link_status = 0;
 398
 399         link_status = enic_get_link_status(enic);
 400         ret = (link_status == enic->link_status);
 401         enic->link_status = link_status;
 402         eth_dev->data->dev_link.link_status = link_status;
 403         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 404         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 405         return ret;
 406 }
 407
 408 static void
 409 enic_intr_handler(void *arg)
 410 {
 411         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 412         struct enic *enic = pmd_priv(dev);
 413
 414         vnic_intr_return_all_credits(&enic->intr);
 415
 416         enic_link_update(enic);
 417         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 418         enic_log_q_error(enic);
 419 }
 420
 421 int enic_enable(struct enic *enic)
 422 {
 423         unsigned int index;
 424         int err;
 425         struct rte_eth_dev *eth_dev = enic->rte_dev;
 426
 427         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 428         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 429
 430         /* vnic notification of link status has already been turned on in
 431          * enic_dev_init() which is called during probe time.  Here we are
 432          * just turning on interrupt vector 0 if needed.
 433          */
 434         if (eth_dev->data->dev_conf.intr_conf.lsc)
 435                 vnic_dev_notify_set(enic->vdev, 0);
 436
 437         if (enic_clsf_init(enic))
 438                 dev_warning(enic, "Init of hash table for clsf failed."\
 439                         "Flow director feature will not work\n");
 440
 441         for (index = 0; index < enic->rq_count; index++) {
 442                 err = enic_alloc_rx_queue_mbufs(enic,
 443                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 444                 if (err) {
 445                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 446                         return err;
 447                 }
 448                 err = enic_alloc_rx_queue_mbufs(enic,
 449                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 450                 if (err) {
 451                         /* release the allocated mbufs for the sop rq*/
 452                         enic_rxmbuf_queue_release(enic,
 453                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 454
 455                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 456                         return err;
 457                 }
 458         }
 459
 460         for (index = 0; index < enic->wq_count; index++)
 461                 enic_start_wq(enic, index);
 462         for (index = 0; index < enic->rq_count; index++)
 463                 enic_start_rq(enic, index);
 464
 465         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 466
 467         vnic_dev_enable_wait(enic->vdev);
 468
 469         /* Register and enable error interrupt */
 470         rte_intr_callback_register(&(enic->pdev->intr_handle),
 471                 enic_intr_handler, (void *)enic->rte_dev);
 472
 473         rte_intr_enable(&(enic->pdev->intr_handle));
 474         vnic_intr_unmask(&enic->intr);
 475
 476         return 0;
 477 }
 478
 479 int enic_alloc_intr_resources(struct enic *enic)
 480 {
 481         int err;
 482
 483         dev_info(enic, "vNIC resources used:  "\
 484                 "wq %d rq %d cq %d intr %d\n",
 485                 enic->wq_count, enic_vnic_rq_count(enic),
 486                 enic->cq_count, enic->intr_count);
 487
 488         err = vnic_intr_alloc(enic->vdev, &enic->intr, 0);
 489         if (err)
 490                 enic_free_vnic_resources(enic);
 491
 492         return err;
 493 }
 494
 495 void enic_free_rq(void *rxq)
 496 {
 497         struct vnic_rq *rq_sop, *rq_data;
 498         struct enic *enic;
 499
 500         if (rxq == NULL)
 501                 return;
 502
 503         rq_sop = (struct vnic_rq *)rxq;
 504         enic = vnic_dev_priv(rq_sop->vdev);
 505         rq_data = &enic->rq[rq_sop->data_queue_idx];
 506
 507         enic_rxmbuf_queue_release(enic, rq_sop);
 508         if (rq_data->in_use)
 509                 enic_rxmbuf_queue_release(enic, rq_data);
 510
 511         rte_free(rq_sop->mbuf_ring);
 512         if (rq_data->in_use)
 513                 rte_free(rq_data->mbuf_ring);
 514
 515         rq_sop->mbuf_ring = NULL;
 516         rq_data->mbuf_ring = NULL;
 517
 518         vnic_rq_free(rq_sop);
 519         if (rq_data->in_use)
 520                 vnic_rq_free(rq_data);
 521
 522         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 523
 524         rq_sop->in_use = 0;
 525         rq_data->in_use = 0;
 526 }
 527
 528 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 529 {
 530         struct rte_eth_dev *eth_dev = enic->rte_dev;
 531         vnic_wq_enable(&enic->wq[queue_idx]);
 532         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 533 }
 534
 535 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 536 {
 537         struct rte_eth_dev *eth_dev = enic->rte_dev;
 538         int ret;
 539
 540         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 541         if (ret)
 542                 return ret;
 543
 544         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 545         return 0;
 546 }
 547
 548 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 549 {
 550         struct vnic_rq *rq_sop;
 551         struct vnic_rq *rq_data;
 552         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 553         rq_data = &enic->rq[rq_sop->data_queue_idx];
 554         struct rte_eth_dev *eth_dev = enic->rte_dev;
 555
 556         if (rq_data->in_use)
 557                 vnic_rq_enable(rq_data);
 558         rte_mb();
 559         vnic_rq_enable(rq_sop);
 560         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 561 }
 562
 563 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 564 {
 565         int ret1 = 0, ret2 = 0;
 566         struct rte_eth_dev *eth_dev = enic->rte_dev;
 567         struct vnic_rq *rq_sop;
 568         struct vnic_rq *rq_data;
 569         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 570         rq_data = &enic->rq[rq_sop->data_queue_idx];
 571
 572         ret2 = vnic_rq_disable(rq_sop);
 573         rte_mb();
 574         if (rq_data->in_use)
 575                 ret1 = vnic_rq_disable(rq_data);
 576
 577         if (ret2)
 578                 return ret2;
 579         else if (ret1)
 580                 return ret1;
 581
 582         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 583         return 0;
 584 }
 585
 586 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 587         unsigned int socket_id, struct rte_mempool *mp,
 588         uint16_t nb_desc, uint16_t free_thresh)
 589 {
 590         int rc;
 591         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 592         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 593         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 594         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 595         unsigned int mbuf_size, mbufs_per_pkt;
 596         unsigned int nb_sop_desc, nb_data_desc;
 597         uint16_t min_sop, max_sop, min_data, max_data;
 598         uint32_t max_rx_pkt_len;
 599
 600         rq_sop->is_sop = 1;
 601         rq_sop->data_queue_idx = data_queue_idx;
 602         rq_data->is_sop = 0;
 603         rq_data->data_queue_idx = 0;
 604         rq_sop->socket_id = socket_id;
 605         rq_sop->mp = mp;
 606         rq_data->socket_id = socket_id;
 607         rq_data->mp = mp;
 608         rq_sop->in_use = 1;
 609         rq_sop->rx_free_thresh = free_thresh;
 610         rq_data->rx_free_thresh = free_thresh;
 611         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 612                   free_thresh);
 613
 614         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 615                                RTE_PKTMBUF_HEADROOM);
 616         /* max_rx_pkt_len includes the ethernet header and CRC. */
 617         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 618
 619         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 620             DEV_RX_OFFLOAD_SCATTER) {
 621                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 622                 /* ceil((max pkt len)/mbuf_size) */
 623                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 624         } else {
 625                 dev_info(enic, "Scatter rx mode disabled\n");
 626                 mbufs_per_pkt = 1;
 627                 if (max_rx_pkt_len > mbuf_size) {
 628                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 629                                     " larger than the mbuf size (%u), and"
 630                                     " scatter is disabled. Larger packets will"
 631                                     " be truncated.\n",
 632                                     max_rx_pkt_len, mbuf_size);
 633                 }
 634         }
 635
 636         if (mbufs_per_pkt > 1) {
 637                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 638                 rq_sop->data_queue_enable = 1;
 639                 rq_data->in_use = 1;
 640                 /*
 641                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 642                  * receives packet sizes up to the "max" MTU.
 643                  * If not using scatter, we can achieve the effect of dropping
 644                  * larger packets by reducing the size of posted buffers.
 645                  * See enic_alloc_rx_queue_mbufs().
 646                  */
 647                 if (max_rx_pkt_len <
 648                     enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu)) {
 649                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 650                                     " when scatter rx mode is in use.\n");
 651                 }
 652         } else {
 653                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 654                          queue_idx);
 655                 rq_sop->data_queue_enable = 0;
 656                 rq_data->in_use = 0;
 657         }
 658
 659         /* number of descriptors have to be a multiple of 32 */
 660         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
 661         nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
 662
 663         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 664         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 665
 666         if (mbufs_per_pkt > 1) {
 667                 min_sop = 64;
 668                 max_sop = ((enic->config.rq_desc_count /
 669                             (mbufs_per_pkt - 1)) & ~0x1F);
 670                 min_data = min_sop * (mbufs_per_pkt - 1);
 671                 max_data = enic->config.rq_desc_count;
 672         } else {
 673                 min_sop = 64;
 674                 max_sop = enic->config.rq_desc_count;
 675                 min_data = 0;
 676                 max_data = 0;
 677         }
 678
 679         if (nb_desc < (min_sop + min_data)) {
 680                 dev_warning(enic,
 681                             "Number of rx descs too low, adjusting to minimum\n");
 682                 nb_sop_desc = min_sop;
 683                 nb_data_desc = min_data;
 684         } else if (nb_desc > (max_sop + max_data)) {
 685                 dev_warning(enic,
 686                             "Number of rx_descs too high, adjusting to maximum\n");
 687                 nb_sop_desc = max_sop;
 688                 nb_data_desc = max_data;
 689         }
 690         if (mbufs_per_pkt > 1) {
 691                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 692                          " rx descriptor range is %u to %u\n",
 693                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 694                          max_sop + max_data);
 695         }
 696         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 697                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 698
 699         /* Allocate sop queue resources */
 700         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 701                 nb_sop_desc, sizeof(struct rq_enet_desc));
 702         if (rc) {
 703                 dev_err(enic, "error in allocation of sop rq\n");
 704                 goto err_exit;
 705         }
 706         nb_sop_desc = rq_sop->ring.desc_count;
 707
 708         if (rq_data->in_use) {
 709                 /* Allocate data queue resources */
 710                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 711                                    nb_data_desc,
 712                                    sizeof(struct rq_enet_desc));
 713                 if (rc) {
 714                         dev_err(enic, "error in allocation of data rq\n");
 715                         goto err_free_rq_sop;
 716                 }
 717                 nb_data_desc = rq_data->ring.desc_count;
 718         }
 719         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 720                            socket_id, nb_sop_desc + nb_data_desc,
 721                            sizeof(struct cq_enet_rq_desc));
 722         if (rc) {
 723                 dev_err(enic, "error in allocation of cq for rq\n");
 724                 goto err_free_rq_data;
 725         }
 726
 727         /* Allocate the mbuf rings */
 728         rq_sop->mbuf_ring = (struct rte_mbuf **)
 729                 rte_zmalloc_socket("rq->mbuf_ring",
 730                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 731                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 732         if (rq_sop->mbuf_ring == NULL)
 733                 goto err_free_cq;
 734
 735         if (rq_data->in_use) {
 736                 rq_data->mbuf_ring = (struct rte_mbuf **)
 737                         rte_zmalloc_socket("rq->mbuf_ring",
 738                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 739                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 740                 if (rq_data->mbuf_ring == NULL)
 741                         goto err_free_sop_mbuf;
 742         }
 743
 744         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 745
 746         return 0;
 747
 748 err_free_sop_mbuf:
 749         rte_free(rq_sop->mbuf_ring);
 750 err_free_cq:
 751         /* cleanup on error */
 752         vnic_cq_free(&enic->cq[queue_idx]);
 753 err_free_rq_data:
 754         if (rq_data->in_use)
 755                 vnic_rq_free(rq_data);
 756 err_free_rq_sop:
 757         vnic_rq_free(rq_sop);
 758 err_exit:
 759         return -ENOMEM;
 760 }
 761
 762 void enic_free_wq(void *txq)
 763 {
 764         struct vnic_wq *wq;
 765         struct enic *enic;
 766
 767         if (txq == NULL)
 768                 return;
 769
 770         wq = (struct vnic_wq *)txq;
 771         enic = vnic_dev_priv(wq->vdev);
 772         rte_memzone_free(wq->cqmsg_rz);
 773         vnic_wq_free(wq);
 774         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 775 }
 776
 777 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 778         unsigned int socket_id, uint16_t nb_desc)
 779 {
 780         int err;
 781         struct vnic_wq *wq = &enic->wq[queue_idx];
 782         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 783         char name[NAME_MAX];
 784         static int instance;
 785
 786         wq->socket_id = socket_id;
 787         if (nb_desc) {
 788                 if (nb_desc > enic->config.wq_desc_count) {
 789                         dev_warning(enic,
 790                                 "WQ %d - number of tx desc in cmd line (%d)"\
 791                                 "is greater than that in the UCSM/CIMC adapter"\
 792                                 "policy.  Applying the value in the adapter "\
 793                                 "policy (%d)\n",
 794                                 queue_idx, nb_desc, enic->config.wq_desc_count);
 795                 } else if (nb_desc != enic->config.wq_desc_count) {
 796                         enic->config.wq_desc_count = nb_desc;
 797                         dev_info(enic,
 798                                 "TX Queues - effective number of descs:%d\n",
 799                                 nb_desc);
 800                 }
 801         }
 802
 803         /* Allocate queue resources */
 804         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
 805                 enic->config.wq_desc_count,
 806                 sizeof(struct wq_enet_desc));
 807         if (err) {
 808                 dev_err(enic, "error in allocation of wq\n");
 809                 return err;
 810         }
 811
 812         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
 813                 socket_id, enic->config.wq_desc_count,
 814                 sizeof(struct cq_enet_wq_desc));
 815         if (err) {
 816                 vnic_wq_free(wq);
 817                 dev_err(enic, "error in allocation of cq for wq\n");
 818         }
 819
 820         /* setup up CQ message */
 821         snprintf((char *)name, sizeof(name),
 822                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
 823                 instance++);
 824
 825         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
 826                                                    sizeof(uint32_t),
 827                                                    SOCKET_ID_ANY, 0,
 828                                                    ENIC_ALIGN);
 829         if (!wq->cqmsg_rz)
 830                 return -ENOMEM;
 831
 832         return err;
 833 }
 834
 835 int enic_disable(struct enic *enic)
 836 {
 837         unsigned int i;
 838         int err;
 839
 840         vnic_intr_mask(&enic->intr);
 841         (void)vnic_intr_masked(&enic->intr); /* flush write */
 842         rte_intr_disable(&enic->pdev->intr_handle);
 843         rte_intr_callback_unregister(&enic->pdev->intr_handle,
 844                                      enic_intr_handler,
 845                                      (void *)enic->rte_dev);
 846
 847         vnic_dev_disable(enic->vdev);
 848
 849         enic_clsf_destroy(enic);
 850
 851         if (!enic_is_sriov_vf(enic))
 852                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
 853
 854         for (i = 0; i < enic->wq_count; i++) {
 855                 err = vnic_wq_disable(&enic->wq[i]);
 856                 if (err)
 857                         return err;
 858         }
 859         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
 860                 if (enic->rq[i].in_use) {
 861                         err = vnic_rq_disable(&enic->rq[i]);
 862                         if (err)
 863                                 return err;
 864                 }
 865         }
 866
 867         /* If we were using interrupts, set the interrupt vector to -1
 868          * to disable interrupts.  We are not disabling link notifcations,
 869          * though, as we want the polling of link status to continue working.
 870          */
 871         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
 872                 vnic_dev_notify_set(enic->vdev, -1);
 873
 874         vnic_dev_set_reset_flag(enic->vdev, 1);
 875
 876         for (i = 0; i < enic->wq_count; i++)
 877                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
 878
 879         for (i = 0; i < enic_vnic_rq_count(enic); i++)
 880                 if (enic->rq[i].in_use)
 881                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
 882         for (i = 0; i < enic->cq_count; i++)
 883                 vnic_cq_clean(&enic->cq[i]);
 884         vnic_intr_clean(&enic->intr);
 885
 886         return 0;
 887 }
 888
 889 static int enic_dev_wait(struct vnic_dev *vdev,
 890         int (*start)(struct vnic_dev *, int),
 891         int (*finished)(struct vnic_dev *, int *),
 892         int arg)
 893 {
 894         int done;
 895         int err;
 896         int i;
 897
 898         err = start(vdev, arg);
 899         if (err)
 900                 return err;
 901
 902         /* Wait for func to complete...2 seconds max */
 903         for (i = 0; i < 2000; i++) {
 904                 err = finished(vdev, &done);
 905                 if (err)
 906                         return err;
 907                 if (done)
 908                         return 0;
 909                 usleep(1000);
 910         }
 911         return -ETIMEDOUT;
 912 }
 913
 914 static int enic_dev_open(struct enic *enic)
 915 {
 916         int err;
 917
 918         err = enic_dev_wait(enic->vdev, vnic_dev_open,
 919                 vnic_dev_open_done, 0);
 920         if (err)
 921                 dev_err(enic_get_dev(enic),
 922                         "vNIC device open failed, err %d\n", err);
 923
 924         return err;
 925 }
 926
 927 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
 928 {
 929         dma_addr_t rss_key_buf_pa;
 930         union vnic_rss_key *rss_key_buf_va = NULL;
 931         int err, i;
 932         u8 name[NAME_MAX];
 933
 934         RTE_ASSERT(user_key != NULL);
 935         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
 936         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
 937                 &rss_key_buf_pa, name);
 938         if (!rss_key_buf_va)
 939                 return -ENOMEM;
 940
 941         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
 942                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
 943
 944         err = enic_set_rss_key(enic,
 945                 rss_key_buf_pa,
 946                 sizeof(union vnic_rss_key));
 947
 948         /* Save for later queries */
 949         if (!err) {
 950                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
 951                            sizeof(union vnic_rss_key));
 952         }
 953         enic_free_consistent(enic, sizeof(union vnic_rss_key),
 954                 rss_key_buf_va, rss_key_buf_pa);
 955
 956         return err;
 957 }
 958
 959 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
 960 {
 961         dma_addr_t rss_cpu_buf_pa;
 962         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
 963         int err;
 964         u8 name[NAME_MAX];
 965
 966         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
 967         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
 968                 &rss_cpu_buf_pa, name);
 969         if (!rss_cpu_buf_va)
 970                 return -ENOMEM;
 971
 972         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
 973
 974         err = enic_set_rss_cpu(enic,
 975                 rss_cpu_buf_pa,
 976                 sizeof(union vnic_rss_cpu));
 977
 978         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
 979                 rss_cpu_buf_va, rss_cpu_buf_pa);
 980
 981         /* Save for later queries */
 982         if (!err)
 983                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
 984         return err;
 985 }
 986
 987 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
 988         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
 989 {
 990         const u8 tso_ipid_split_en = 0;
 991         int err;
 992
 993         err = enic_set_nic_cfg(enic,
 994                 rss_default_cpu, rss_hash_type,
 995                 rss_hash_bits, rss_base_cpu,
 996                 rss_enable, tso_ipid_split_en,
 997                 enic->ig_vlan_strip_en);
 998
 999         return err;
1000 }
1001
1002 /* Initialize RSS with defaults, called from dev_configure */
1003 int enic_init_rss_nic_cfg(struct enic *enic)
1004 {
1005         static uint8_t default_rss_key[] = {
1006                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1007                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1008                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1009                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1010         };
1011         struct rte_eth_rss_conf rss_conf;
1012         union vnic_rss_cpu rss_cpu;
1013         int ret, i;
1014
1015         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1016         /*
1017          * If setting key for the first time, and the user gives us none, then
1018          * push the default key to NIC.
1019          */
1020         if (rss_conf.rss_key == NULL) {
1021                 rss_conf.rss_key = default_rss_key;
1022                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1023         }
1024         ret = enic_set_rss_conf(enic, &rss_conf);
1025         if (ret) {
1026                 dev_err(enic, "Failed to configure RSS\n");
1027                 return ret;
1028         }
1029         if (enic->rss_enable) {
1030                 /* If enabling RSS, use the default reta */
1031                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1032                         rss_cpu.cpu[i / 4].b[i % 4] =
1033                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1034                 }
1035                 ret = enic_set_rss_reta(enic, &rss_cpu);
1036                 if (ret)
1037                         dev_err(enic, "Failed to set RSS indirection table\n");
1038         }
1039         return ret;
1040 }
1041
1042 int enic_setup_finish(struct enic *enic)
1043 {
1044         enic_init_soft_stats(enic);
1045
1046         /* Default conf */
1047         vnic_dev_packet_filter(enic->vdev,
1048                 1 /* directed  */,
1049                 1 /* multicast */,
1050                 1 /* broadcast */,
1051                 0 /* promisc   */,
1052                 1 /* allmulti  */);
1053
1054         enic->promisc = 0;
1055         enic->allmulti = 1;
1056
1057         return 0;
1058 }
1059
1060 static int enic_rss_conf_valid(struct enic *enic,
1061                                struct rte_eth_rss_conf *rss_conf)
1062 {
1063         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1064         if (enic->flow_type_rss_offloads == 0)
1065                 return 0;
1066         if (rss_conf->rss_key != NULL &&
1067             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1068                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1069                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1070                 return -EINVAL;
1071         }
1072         if (rss_conf->rss_hf != 0 &&
1073             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1074                 dev_err(enic, "Given rss_hf contains none of the supported"
1075                         " types\n");
1076                 return -EINVAL;
1077         }
1078         return 0;
1079 }
1080
1081 /* Set hash type and key according to rss_conf */
1082 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1083 {
1084         struct rte_eth_dev *eth_dev;
1085         uint64_t rss_hf;
1086         u8 rss_hash_type;
1087         u8 rss_enable;
1088         int ret;
1089
1090         RTE_ASSERT(rss_conf != NULL);
1091         ret = enic_rss_conf_valid(enic, rss_conf);
1092         if (ret) {
1093                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1094                 return ret;
1095         }
1096
1097         eth_dev = enic->rte_dev;
1098         rss_hash_type = 0;
1099         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1100         if (enic->rq_count > 1 &&
1101             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1102             rss_hf != 0) {
1103                 rss_enable = 1;
1104                 if (rss_hf & ETH_RSS_IPV4)
1105                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1106                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1107                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1108                 if (rss_hf & ETH_RSS_IPV6)
1109                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1110                 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1111                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1112                 if (rss_hf & ETH_RSS_IPV6_EX)
1113                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6_EX;
1114                 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
1115                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX;
1116         } else {
1117                 rss_enable = 0;
1118                 rss_hf = 0;
1119         }
1120
1121         /* Set the hash key if provided */
1122         if (rss_enable && rss_conf->rss_key) {
1123                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1124                 if (ret) {
1125                         dev_err(enic, "Failed to set RSS key\n");
1126                         return ret;
1127                 }
1128         }
1129
1130         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1131                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1132                               rss_enable);
1133         if (!ret) {
1134                 enic->rss_hf = rss_hf;
1135                 enic->rss_hash_type = rss_hash_type;
1136                 enic->rss_enable = rss_enable;
1137         }
1138         return 0;
1139 }
1140
1141 int enic_set_vlan_strip(struct enic *enic)
1142 {
1143         /*
1144          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1145          * together. So, re-do niccfg, preserving the current RSS settings.
1146          */
1147         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1148                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1149                                enic->rss_enable);
1150 }
1151
1152 void enic_add_packet_filter(struct enic *enic)
1153 {
1154         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1155         vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1156                 enic->promisc, enic->allmulti);
1157 }
1158
1159 int enic_get_link_status(struct enic *enic)
1160 {
1161         return vnic_dev_link_status(enic->vdev);
1162 }
1163
1164 static void enic_dev_deinit(struct enic *enic)
1165 {
1166         struct rte_eth_dev *eth_dev = enic->rte_dev;
1167
1168         /* stop link status checking */
1169         vnic_dev_notify_unset(enic->vdev);
1170
1171         rte_free(eth_dev->data->mac_addrs);
1172         rte_free(enic->cq);
1173         rte_free(enic->rq);
1174         rte_free(enic->wq);
1175 }
1176
1177
1178 int enic_set_vnic_res(struct enic *enic)
1179 {
1180         struct rte_eth_dev *eth_dev = enic->rte_dev;
1181         int rc = 0;
1182         unsigned int required_rq, required_wq, required_cq;
1183
1184         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1185         required_rq = eth_dev->data->nb_rx_queues * 2;
1186         required_wq = eth_dev->data->nb_tx_queues;
1187         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1188
1189         if (enic->conf_rq_count < required_rq) {
1190                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1191                         eth_dev->data->nb_rx_queues,
1192                         required_rq, enic->conf_rq_count);
1193                 rc = -EINVAL;
1194         }
1195         if (enic->conf_wq_count < required_wq) {
1196                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1197                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1198                 rc = -EINVAL;
1199         }
1200
1201         if (enic->conf_cq_count < required_cq) {
1202                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1203                         required_cq, enic->conf_cq_count);
1204                 rc = -EINVAL;
1205         }
1206
1207         if (rc == 0) {
1208                 enic->rq_count = eth_dev->data->nb_rx_queues;
1209                 enic->wq_count = eth_dev->data->nb_tx_queues;
1210                 enic->cq_count = enic->rq_count + enic->wq_count;
1211         }
1212
1213         return rc;
1214 }
1215
1216 /* Initialize the completion queue for an RQ */
1217 static int
1218 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1219 {
1220         struct vnic_rq *sop_rq, *data_rq;
1221         unsigned int cq_idx;
1222         int rc = 0;
1223
1224         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1225         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1226         cq_idx = rq_idx;
1227
1228         vnic_cq_clean(&enic->cq[cq_idx]);
1229         vnic_cq_init(&enic->cq[cq_idx],
1230                      0 /* flow_control_enable */,
1231                      1 /* color_enable */,
1232                      0 /* cq_head */,
1233                      0 /* cq_tail */,
1234                      1 /* cq_tail_color */,
1235                      0 /* interrupt_enable */,
1236                      1 /* cq_entry_enable */,
1237                      0 /* cq_message_enable */,
1238                      0 /* interrupt offset */,
1239                      0 /* cq_message_addr */);
1240
1241
1242         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1243                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1244                            sop_rq->ring.desc_count - 1, 1, 0);
1245         if (data_rq->in_use) {
1246                 vnic_rq_init_start(data_rq,
1247                                    enic_cq_rq(enic,
1248                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1249                                    data_rq->ring.desc_count - 1, 1, 0);
1250         }
1251
1252         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1253         if (rc)
1254                 return rc;
1255
1256         if (data_rq->in_use) {
1257                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1258                 if (rc) {
1259                         enic_rxmbuf_queue_release(enic, sop_rq);
1260                         return rc;
1261                 }
1262         }
1263
1264         return 0;
1265 }
1266
1267 /* The Cisco NIC can send and receive packets up to a max packet size
1268  * determined by the NIC type and firmware. There is also an MTU
1269  * configured into the NIC via the CIMC/UCSM management interface
1270  * which can be overridden by this function (up to the max packet size).
1271  * Depending on the network setup, doing so may cause packet drops
1272  * and unexpected behavior.
1273  */
1274 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1275 {
1276         unsigned int rq_idx;
1277         struct vnic_rq *rq;
1278         int rc = 0;
1279         uint16_t old_mtu;       /* previous setting */
1280         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1281         struct rte_eth_dev *eth_dev = enic->rte_dev;
1282
1283         old_mtu = eth_dev->data->mtu;
1284         config_mtu = enic->config.mtu;
1285
1286         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1287                 return -E_RTE_SECONDARY;
1288
1289         if (new_mtu > enic->max_mtu) {
1290                 dev_err(enic,
1291                         "MTU not updated: requested (%u) greater than max (%u)\n",
1292                         new_mtu, enic->max_mtu);
1293                 return -EINVAL;
1294         }
1295         if (new_mtu < ENIC_MIN_MTU) {
1296                 dev_info(enic,
1297                         "MTU not updated: requested (%u) less than min (%u)\n",
1298                         new_mtu, ENIC_MIN_MTU);
1299                 return -EINVAL;
1300         }
1301         if (new_mtu > config_mtu)
1302                 dev_warning(enic,
1303                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1304                         new_mtu, config_mtu);
1305
1306         /* The easy case is when scatter is disabled. However if the MTU
1307          * becomes greater than the mbuf data size, packet drops will ensue.
1308          */
1309         if (!(enic->rte_dev->data->dev_conf.rxmode.offloads &
1310               DEV_RX_OFFLOAD_SCATTER)) {
1311                 eth_dev->data->mtu = new_mtu;
1312                 goto set_mtu_done;
1313         }
1314
1315         /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to
1316          * change Rx scatter mode if necessary for better performance. I.e. if
1317          * MTU was greater than the mbuf size and now it's less, scatter Rx
1318          * doesn't have to be used and vice versa.
1319           */
1320         rte_spinlock_lock(&enic->mtu_lock);
1321
1322         /* Stop traffic on all RQs */
1323         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1324                 rq = &enic->rq[rq_idx];
1325                 if (rq->is_sop && rq->in_use) {
1326                         rc = enic_stop_rq(enic,
1327                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1328                         if (rc) {
1329                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1330                                 goto set_mtu_done;
1331                         }
1332                 }
1333         }
1334
1335         /* replace Rx function with a no-op to avoid getting stale pkts */
1336         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1337         rte_mb();
1338
1339         /* Allow time for threads to exit the real Rx function. */
1340         usleep(100000);
1341
1342         /* now it is safe to reconfigure the RQs */
1343
1344         /* update the mtu */
1345         eth_dev->data->mtu = new_mtu;
1346
1347         /* free and reallocate RQs with the new MTU */
1348         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1349                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1350
1351                 enic_free_rq(rq);
1352                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1353                                    rq->tot_nb_desc, rq->rx_free_thresh);
1354                 if (rc) {
1355                         dev_err(enic,
1356                                 "Fatal MTU alloc error- No traffic will pass\n");
1357                         goto set_mtu_done;
1358                 }
1359
1360                 rc = enic_reinit_rq(enic, rq_idx);
1361                 if (rc) {
1362                         dev_err(enic,
1363                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1364                         goto set_mtu_done;
1365                 }
1366         }
1367
1368         /* put back the real receive function */
1369         rte_mb();
1370         eth_dev->rx_pkt_burst = enic_recv_pkts;
1371         rte_mb();
1372
1373         /* restart Rx traffic */
1374         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1375                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1376                 if (rq->is_sop && rq->in_use)
1377                         enic_start_rq(enic, rq_idx);
1378         }
1379
1380 set_mtu_done:
1381         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1382         rte_spinlock_unlock(&enic->mtu_lock);
1383         return rc;
1384 }
1385
1386 static int enic_dev_init(struct enic *enic)
1387 {
1388         int err;
1389         struct rte_eth_dev *eth_dev = enic->rte_dev;
1390
1391         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1392
1393         /* Get vNIC configuration
1394         */
1395         err = enic_get_vnic_config(enic);
1396         if (err) {
1397                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1398                 return err;
1399         }
1400
1401         /* Get available resource counts */
1402         enic_get_res_counts(enic);
1403         if (enic->conf_rq_count == 1) {
1404                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1405                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1406                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1407                 return -EINVAL;
1408         }
1409         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1410         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1411                                enic->conf_cq_count, 8);
1412         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1413                                enic->conf_rq_count, 8);
1414         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1415                                enic->conf_wq_count, 8);
1416         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1417                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1418                 return -1;
1419         }
1420         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1421                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1422                 return -1;
1423         }
1424         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1425                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1426                 return -1;
1427         }
1428
1429         /* Get the supported filters */
1430         enic_fdir_info(enic);
1431
1432         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1433                                                 * ENIC_MAX_MAC_ADDR, 0);
1434         if (!eth_dev->data->mac_addrs) {
1435                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1436                 return -1;
1437         }
1438         ether_addr_copy((struct ether_addr *) enic->mac_addr,
1439                         eth_dev->data->mac_addrs);
1440
1441         vnic_dev_set_reset_flag(enic->vdev, 0);
1442
1443         LIST_INIT(&enic->flows);
1444         rte_spinlock_init(&enic->flows_lock);
1445
1446         /* set up link status checking */
1447         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1448
1449         return 0;
1450
1451 }
1452
1453 int enic_probe(struct enic *enic)
1454 {
1455         struct rte_pci_device *pdev = enic->pdev;
1456         int err = -1;
1457
1458         dev_debug(enic, " Initializing ENIC PMD\n");
1459
1460         /* if this is a secondary process the hardware is already initialized */
1461         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1462                 return 0;
1463
1464         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1465         enic->bar0.len = pdev->mem_resource[0].len;
1466
1467         /* Register vNIC device */
1468         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1469         if (!enic->vdev) {
1470                 dev_err(enic, "vNIC registration failed, aborting\n");
1471                 goto err_out;
1472         }
1473
1474         LIST_INIT(&enic->memzone_list);
1475         rte_spinlock_init(&enic->memzone_list_lock);
1476
1477         vnic_register_cbacks(enic->vdev,
1478                 enic_alloc_consistent,
1479                 enic_free_consistent);
1480
1481         /*
1482          * Allocate the consistent memory for stats upfront so both primary and
1483          * secondary processes can dump stats.
1484          */
1485         err = vnic_dev_alloc_stats_mem(enic->vdev);
1486         if (err) {
1487                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1488                 goto err_out_unregister;
1489         }
1490         /* Issue device open to get device in known state */
1491         err = enic_dev_open(enic);
1492         if (err) {
1493                 dev_err(enic, "vNIC dev open failed, aborting\n");
1494                 goto err_out_unregister;
1495         }
1496
1497         /* Set ingress vlan rewrite mode before vnic initialization */
1498         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1499                 IG_VLAN_REWRITE_MODE_PASS_THRU);
1500         if (err) {
1501                 dev_err(enic,
1502                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1503                 goto err_out_dev_close;
1504         }
1505
1506         /* Issue device init to initialize the vnic-to-switch link.
1507          * We'll start with carrier off and wait for link UP
1508          * notification later to turn on carrier.  We don't need
1509          * to wait here for the vnic-to-switch link initialization
1510          * to complete; link UP notification is the indication that
1511          * the process is complete.
1512          */
1513
1514         err = vnic_dev_init(enic->vdev, 0);
1515         if (err) {
1516                 dev_err(enic, "vNIC dev init failed, aborting\n");
1517                 goto err_out_dev_close;
1518         }
1519
1520         err = enic_dev_init(enic);
1521         if (err) {
1522                 dev_err(enic, "Device initialization failed, aborting\n");
1523                 goto err_out_dev_close;
1524         }
1525
1526         return 0;
1527
1528 err_out_dev_close:
1529         vnic_dev_close(enic->vdev);
1530 err_out_unregister:
1531         vnic_dev_unregister(enic->vdev);
1532 err_out:
1533         return err;
1534 }
1535
1536 void enic_remove(struct enic *enic)
1537 {
1538         enic_dev_deinit(enic);
1539         vnic_dev_close(enic->vdev);
1540         vnic_dev_unregister(enic->vdev);
1541 }