drivers/net/thunderx/nicvf_ethdev.c

   1 /*
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (C) Cavium networks Ltd. 2016.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Cavium networks nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <assert.h>
  34 #include <stdio.h>
  35 #include <stdbool.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43 #include <sys/queue.h>
  44 #include <sys/timerfd.h>
  45
  46 #include <rte_alarm.h>
  47 #include <rte_atomic.h>
  48 #include <rte_branch_prediction.h>
  49 #include <rte_byteorder.h>
  50 #include <rte_common.h>
  51 #include <rte_cycles.h>
  52 #include <rte_debug.h>
  53 #include <rte_dev.h>
  54 #include <rte_eal.h>
  55 #include <rte_ether.h>
  56 #include <rte_ethdev.h>
  57 #include <rte_interrupts.h>
  58 #include <rte_log.h>
  59 #include <rte_memory.h>
  60 #include <rte_memzone.h>
  61 #include <rte_malloc.h>
  62 #include <rte_random.h>
  63 #include <rte_pci.h>
  64 #include <rte_tailq.h>
  65
  66 #include "base/nicvf_plat.h"
  67
  68 #include "nicvf_ethdev.h"
  69 #include "nicvf_rxtx.h"
  70 #include "nicvf_logs.h"
  71
  72 static void nicvf_dev_stop(struct rte_eth_dev *dev);
  73
  74 static inline int
  75 nicvf_atomic_write_link_status(struct rte_eth_dev *dev,
  76                                struct rte_eth_link *link)
  77 {
  78         struct rte_eth_link *dst = &dev->data->dev_link;
  79         struct rte_eth_link *src = link;
  80
  81         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
  82                 *(uint64_t *)src) == 0)
  83                 return -1;
  84
  85         return 0;
  86 }
  87
  88 static inline void
  89 nicvf_set_eth_link_status(struct nicvf *nic, struct rte_eth_link *link)
  90 {
  91         link->link_status = nic->link_up;
  92         link->link_duplex = ETH_LINK_AUTONEG;
  93         if (nic->duplex == NICVF_HALF_DUPLEX)
  94                 link->link_duplex = ETH_LINK_HALF_DUPLEX;
  95         else if (nic->duplex == NICVF_FULL_DUPLEX)
  96                 link->link_duplex = ETH_LINK_FULL_DUPLEX;
  97         link->link_speed = nic->speed;
  98         link->link_autoneg = ETH_LINK_SPEED_AUTONEG;
  99 }
 100
 101 static void
 102 nicvf_interrupt(void *arg)
 103 {
 104         struct nicvf *nic = arg;
 105
 106         if (nicvf_reg_poll_interrupts(nic) == NIC_MBOX_MSG_BGX_LINK_CHANGE) {
 107                 if (nic->eth_dev->data->dev_conf.intr_conf.lsc)
 108                         nicvf_set_eth_link_status(nic,
 109                                         &nic->eth_dev->data->dev_link);
 110                 _rte_eth_dev_callback_process(nic->eth_dev,
 111                                 RTE_ETH_EVENT_INTR_LSC);
 112         }
 113
 114         rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
 115                                 nicvf_interrupt, nic);
 116 }
 117
 118 static int
 119 nicvf_periodic_alarm_start(struct nicvf *nic)
 120 {
 121         return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
 122                                         nicvf_interrupt, nic);
 123 }
 124
 125 static int
 126 nicvf_periodic_alarm_stop(struct nicvf *nic)
 127 {
 128         return rte_eal_alarm_cancel(nicvf_interrupt, nic);
 129 }
 130
 131 /*
 132  * Return 0 means link status changed, -1 means not changed
 133  */
 134 static int
 135 nicvf_dev_link_update(struct rte_eth_dev *dev,
 136                       int wait_to_complete __rte_unused)
 137 {
 138         struct rte_eth_link link;
 139         struct nicvf *nic = nicvf_pmd_priv(dev);
 140
 141         PMD_INIT_FUNC_TRACE();
 142
 143         memset(&link, 0, sizeof(link));
 144         nicvf_set_eth_link_status(nic, &link);
 145         return nicvf_atomic_write_link_status(dev, &link);
 146 }
 147
 148 static int
 149 nicvf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 150 {
 151         struct nicvf *nic = nicvf_pmd_priv(dev);
 152         uint32_t buffsz, frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 153
 154         PMD_INIT_FUNC_TRACE();
 155
 156         if (frame_size > NIC_HW_MAX_FRS)
 157                 return -EINVAL;
 158
 159         if (frame_size < NIC_HW_MIN_FRS)
 160                 return -EINVAL;
 161
 162         buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 163
 164         /*
 165          * Refuse mtu that requires the support of scattered packets
 166          * when this feature has not been enabled before.
 167          */
 168         if (!dev->data->scattered_rx &&
 169                 (frame_size + 2 * VLAN_TAG_SIZE > buffsz))
 170                 return -EINVAL;
 171
 172         /* check <seg size> * <max_seg>  >= max_frame */
 173         if (dev->data->scattered_rx &&
 174                 (frame_size + 2 * VLAN_TAG_SIZE > buffsz * NIC_HW_MAX_SEGS))
 175                 return -EINVAL;
 176
 177         if (frame_size > ETHER_MAX_LEN)
 178                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
 179         else
 180                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
 181
 182         if (nicvf_mbox_update_hw_max_frs(nic, frame_size))
 183                 return -EINVAL;
 184
 185         /* Update max frame size */
 186         dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)frame_size;
 187         nic->mtu = mtu;
 188         return 0;
 189 }
 190
 191 static int
 192 nicvf_dev_get_reg_length(struct rte_eth_dev *dev  __rte_unused)
 193 {
 194         return nicvf_reg_get_count();
 195 }
 196
 197 static int
 198 nicvf_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs)
 199 {
 200         uint64_t *data = regs->data;
 201         struct nicvf *nic = nicvf_pmd_priv(dev);
 202
 203         if (data == NULL)
 204                 return -EINVAL;
 205
 206         /* Support only full register dump */
 207         if ((regs->length == 0) ||
 208                 (regs->length == (uint32_t)nicvf_reg_get_count())) {
 209                 regs->version = nic->vendor_id << 16 | nic->device_id;
 210                 nicvf_reg_dump(nic, data);
 211                 return 0;
 212         }
 213         return -ENOTSUP;
 214 }
 215
 216 static void
 217 nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 218 {
 219         uint16_t qidx;
 220         struct nicvf_hw_rx_qstats rx_qstats;
 221         struct nicvf_hw_tx_qstats tx_qstats;
 222         struct nicvf_hw_stats port_stats;
 223         struct nicvf *nic = nicvf_pmd_priv(dev);
 224
 225         /* Reading per RX ring stats */
 226         for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) {
 227                 if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 228                         break;
 229
 230                 nicvf_hw_get_rx_qstats(nic, &rx_qstats, qidx);
 231                 stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes;
 232                 stats->q_ipackets[qidx] = rx_qstats.q_rx_packets;
 233         }
 234
 235         /* Reading per TX ring stats */
 236         for (qidx = 0; qidx < dev->data->nb_tx_queues; qidx++) {
 237                 if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 238                         break;
 239
 240                 nicvf_hw_get_tx_qstats(nic, &tx_qstats, qidx);
 241                 stats->q_obytes[qidx] = tx_qstats.q_tx_bytes;
 242                 stats->q_opackets[qidx] = tx_qstats.q_tx_packets;
 243         }
 244
 245         nicvf_hw_get_stats(nic, &port_stats);
 246         stats->ibytes = port_stats.rx_bytes;
 247         stats->ipackets = port_stats.rx_ucast_frames;
 248         stats->ipackets += port_stats.rx_bcast_frames;
 249         stats->ipackets += port_stats.rx_mcast_frames;
 250         stats->ierrors = port_stats.rx_l2_errors;
 251         stats->imissed = port_stats.rx_drop_red;
 252         stats->imissed += port_stats.rx_drop_overrun;
 253         stats->imissed += port_stats.rx_drop_bcast;
 254         stats->imissed += port_stats.rx_drop_mcast;
 255         stats->imissed += port_stats.rx_drop_l3_bcast;
 256         stats->imissed += port_stats.rx_drop_l3_mcast;
 257
 258         stats->obytes = port_stats.tx_bytes_ok;
 259         stats->opackets = port_stats.tx_ucast_frames_ok;
 260         stats->opackets += port_stats.tx_bcast_frames_ok;
 261         stats->opackets += port_stats.tx_mcast_frames_ok;
 262         stats->oerrors = port_stats.tx_drops;
 263 }
 264
 265 static const uint32_t *
 266 nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 267 {
 268         size_t copied;
 269         static uint32_t ptypes[32];
 270         struct nicvf *nic = nicvf_pmd_priv(dev);
 271         static const uint32_t ptypes_pass1[] = {
 272                 RTE_PTYPE_L3_IPV4,
 273                 RTE_PTYPE_L3_IPV4_EXT,
 274                 RTE_PTYPE_L3_IPV6,
 275                 RTE_PTYPE_L3_IPV6_EXT,
 276                 RTE_PTYPE_L4_TCP,
 277                 RTE_PTYPE_L4_UDP,
 278                 RTE_PTYPE_L4_FRAG,
 279         };
 280         static const uint32_t ptypes_pass2[] = {
 281                 RTE_PTYPE_TUNNEL_GRE,
 282                 RTE_PTYPE_TUNNEL_GENEVE,
 283                 RTE_PTYPE_TUNNEL_VXLAN,
 284                 RTE_PTYPE_TUNNEL_NVGRE,
 285         };
 286         static const uint32_t ptypes_end = RTE_PTYPE_UNKNOWN;
 287
 288         copied = sizeof(ptypes_pass1);
 289         memcpy(ptypes, ptypes_pass1, copied);
 290         if (nicvf_hw_version(nic) == NICVF_PASS2) {
 291                 memcpy((char *)ptypes + copied, ptypes_pass2,
 292                         sizeof(ptypes_pass2));
 293                 copied += sizeof(ptypes_pass2);
 294         }
 295
 296         memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end));
 297         if (dev->rx_pkt_burst == nicvf_recv_pkts ||
 298                 dev->rx_pkt_burst == nicvf_recv_pkts_multiseg)
 299                 return ptypes;
 300
 301         return NULL;
 302 }
 303
 304 static void
 305 nicvf_dev_stats_reset(struct rte_eth_dev *dev)
 306 {
 307         int i;
 308         uint16_t rxqs = 0, txqs = 0;
 309         struct nicvf *nic = nicvf_pmd_priv(dev);
 310
 311         for (i = 0; i < dev->data->nb_rx_queues; i++)
 312                 rxqs |= (0x3 << (i * 2));
 313         for (i = 0; i < dev->data->nb_tx_queues; i++)
 314                 txqs |= (0x3 << (i * 2));
 315
 316         nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, rxqs, txqs);
 317 }
 318
 319 /* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */
 320 static void
 321 nicvf_dev_promisc_enable(struct rte_eth_dev *dev __rte_unused)
 322 {
 323 }
 324
 325 static inline uint64_t
 326 nicvf_rss_ethdev_to_nic(struct nicvf *nic, uint64_t ethdev_rss)
 327 {
 328         uint64_t nic_rss = 0;
 329
 330         if (ethdev_rss & ETH_RSS_IPV4)
 331                 nic_rss |= RSS_IP_ENA;
 332
 333         if (ethdev_rss & ETH_RSS_IPV6)
 334                 nic_rss |= RSS_IP_ENA;
 335
 336         if (ethdev_rss & ETH_RSS_NONFRAG_IPV4_UDP)
 337                 nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
 338
 339         if (ethdev_rss & ETH_RSS_NONFRAG_IPV4_TCP)
 340                 nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
 341
 342         if (ethdev_rss & ETH_RSS_NONFRAG_IPV6_UDP)
 343                 nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
 344
 345         if (ethdev_rss & ETH_RSS_NONFRAG_IPV6_TCP)
 346                 nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
 347
 348         if (ethdev_rss & ETH_RSS_PORT)
 349                 nic_rss |= RSS_L2_EXTENDED_HASH_ENA;
 350
 351         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
 352                 if (ethdev_rss & ETH_RSS_VXLAN)
 353                         nic_rss |= RSS_TUN_VXLAN_ENA;
 354
 355                 if (ethdev_rss & ETH_RSS_GENEVE)
 356                         nic_rss |= RSS_TUN_GENEVE_ENA;
 357
 358                 if (ethdev_rss & ETH_RSS_NVGRE)
 359                         nic_rss |= RSS_TUN_NVGRE_ENA;
 360         }
 361
 362         return nic_rss;
 363 }
 364
 365 static inline uint64_t
 366 nicvf_rss_nic_to_ethdev(struct nicvf *nic,  uint64_t nic_rss)
 367 {
 368         uint64_t ethdev_rss = 0;
 369
 370         if (nic_rss & RSS_IP_ENA)
 371                 ethdev_rss |= (ETH_RSS_IPV4 | ETH_RSS_IPV6);
 372
 373         if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_TCP_ENA))
 374                 ethdev_rss |= (ETH_RSS_NONFRAG_IPV4_TCP |
 375                                 ETH_RSS_NONFRAG_IPV6_TCP);
 376
 377         if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_UDP_ENA))
 378                 ethdev_rss |= (ETH_RSS_NONFRAG_IPV4_UDP |
 379                                 ETH_RSS_NONFRAG_IPV6_UDP);
 380
 381         if (nic_rss & RSS_L2_EXTENDED_HASH_ENA)
 382                 ethdev_rss |= ETH_RSS_PORT;
 383
 384         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
 385                 if (nic_rss & RSS_TUN_VXLAN_ENA)
 386                         ethdev_rss |= ETH_RSS_VXLAN;
 387
 388                 if (nic_rss & RSS_TUN_GENEVE_ENA)
 389                         ethdev_rss |= ETH_RSS_GENEVE;
 390
 391                 if (nic_rss & RSS_TUN_NVGRE_ENA)
 392                         ethdev_rss |= ETH_RSS_NVGRE;
 393         }
 394         return ethdev_rss;
 395 }
 396
 397 static int
 398 nicvf_dev_reta_query(struct rte_eth_dev *dev,
 399                      struct rte_eth_rss_reta_entry64 *reta_conf,
 400                      uint16_t reta_size)
 401 {
 402         struct nicvf *nic = nicvf_pmd_priv(dev);
 403         uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
 404         int ret, i, j;
 405
 406         if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
 407                 RTE_LOG(ERR, PMD, "The size of hash lookup table configured "
 408                         "(%d) doesn't match the number hardware can supported "
 409                         "(%d)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
 410                 return -EINVAL;
 411         }
 412
 413         ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
 414         if (ret)
 415                 return ret;
 416
 417         /* Copy RETA table */
 418         for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_RETA_GROUP_SIZE); i++) {
 419                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
 420                         if ((reta_conf[i].mask >> j) & 0x01)
 421                                 reta_conf[i].reta[j] = tbl[j];
 422         }
 423
 424         return 0;
 425 }
 426
 427 static int
 428 nicvf_dev_reta_update(struct rte_eth_dev *dev,
 429                       struct rte_eth_rss_reta_entry64 *reta_conf,
 430                       uint16_t reta_size)
 431 {
 432         struct nicvf *nic = nicvf_pmd_priv(dev);
 433         uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
 434         int ret, i, j;
 435
 436         if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
 437                 RTE_LOG(ERR, PMD, "The size of hash lookup table configured "
 438                         "(%d) doesn't match the number hardware can supported "
 439                         "(%d)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
 440                 return -EINVAL;
 441         }
 442
 443         ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
 444         if (ret)
 445                 return ret;
 446
 447         /* Copy RETA table */
 448         for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_RETA_GROUP_SIZE); i++) {
 449                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
 450                         if ((reta_conf[i].mask >> j) & 0x01)
 451                                 tbl[j] = reta_conf[i].reta[j];
 452         }
 453
 454         return nicvf_rss_reta_update(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
 455 }
 456
 457 static int
 458 nicvf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 459                             struct rte_eth_rss_conf *rss_conf)
 460 {
 461         struct nicvf *nic = nicvf_pmd_priv(dev);
 462
 463         if (rss_conf->rss_key)
 464                 nicvf_rss_get_key(nic, rss_conf->rss_key);
 465
 466         rss_conf->rss_key_len =  RSS_HASH_KEY_BYTE_SIZE;
 467         rss_conf->rss_hf = nicvf_rss_nic_to_ethdev(nic, nicvf_rss_get_cfg(nic));
 468         return 0;
 469 }
 470
 471 static int
 472 nicvf_dev_rss_hash_update(struct rte_eth_dev *dev,
 473                           struct rte_eth_rss_conf *rss_conf)
 474 {
 475         struct nicvf *nic = nicvf_pmd_priv(dev);
 476         uint64_t nic_rss;
 477
 478         if (rss_conf->rss_key &&
 479                 rss_conf->rss_key_len != RSS_HASH_KEY_BYTE_SIZE) {
 480                 RTE_LOG(ERR, PMD, "Hash key size mismatch %d",
 481                                 rss_conf->rss_key_len);
 482                 return -EINVAL;
 483         }
 484
 485         if (rss_conf->rss_key)
 486                 nicvf_rss_set_key(nic, rss_conf->rss_key);
 487
 488         nic_rss = nicvf_rss_ethdev_to_nic(nic, rss_conf->rss_hf);
 489         nicvf_rss_set_cfg(nic, nic_rss);
 490         return 0;
 491 }
 492
 493 static int
 494 nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx,
 495                     uint32_t desc_cnt)
 496 {
 497         const struct rte_memzone *rz;
 498         uint32_t ring_size = desc_cnt * sizeof(union cq_entry_t);
 499
 500         rz = rte_eth_dma_zone_reserve(nic->eth_dev, "cq_ring", qidx, ring_size,
 501                                         NICVF_CQ_BASE_ALIGN_BYTES, nic->node);
 502         if (rz == NULL) {
 503                 PMD_INIT_LOG(ERR, "Failed to allocate mem for cq hw ring");
 504                 return -ENOMEM;
 505         }
 506
 507         memset(rz->addr, 0, ring_size);
 508
 509         rxq->phys = rz->phys_addr;
 510         rxq->desc = rz->addr;
 511         rxq->qlen_mask = desc_cnt - 1;
 512
 513         return 0;
 514 }
 515
 516 static int
 517 nicvf_qset_sq_alloc(struct nicvf *nic,  struct nicvf_txq *sq, uint16_t qidx,
 518                     uint32_t desc_cnt)
 519 {
 520         const struct rte_memzone *rz;
 521         uint32_t ring_size = desc_cnt * sizeof(union sq_entry_t);
 522
 523         rz = rte_eth_dma_zone_reserve(nic->eth_dev, "sq", qidx, ring_size,
 524                                 NICVF_SQ_BASE_ALIGN_BYTES, nic->node);
 525         if (rz == NULL) {
 526                 PMD_INIT_LOG(ERR, "Failed allocate mem for sq hw ring");
 527                 return -ENOMEM;
 528         }
 529
 530         memset(rz->addr, 0, ring_size);
 531
 532         sq->phys = rz->phys_addr;
 533         sq->desc = rz->addr;
 534         sq->qlen_mask = desc_cnt - 1;
 535
 536         return 0;
 537 }
 538
 539 static int
 540 nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz)
 541 {
 542         struct nicvf_rbdr *rbdr;
 543         const struct rte_memzone *rz;
 544         uint32_t ring_size;
 545
 546         assert(nic->rbdr == NULL);
 547         rbdr = rte_zmalloc_socket("rbdr", sizeof(struct nicvf_rbdr),
 548                                   RTE_CACHE_LINE_SIZE, nic->node);
 549         if (rbdr == NULL) {
 550                 PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr");
 551                 return -ENOMEM;
 552         }
 553
 554         ring_size = sizeof(struct rbdr_entry_t) * desc_cnt;
 555         rz = rte_eth_dma_zone_reserve(nic->eth_dev, "rbdr", 0, ring_size,
 556                                    NICVF_RBDR_BASE_ALIGN_BYTES, nic->node);
 557         if (rz == NULL) {
 558                 PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr desc ring");
 559                 return -ENOMEM;
 560         }
 561
 562         memset(rz->addr, 0, ring_size);
 563
 564         rbdr->phys = rz->phys_addr;
 565         rbdr->tail = 0;
 566         rbdr->next_tail = 0;
 567         rbdr->desc = rz->addr;
 568         rbdr->buffsz = buffsz;
 569         rbdr->qlen_mask = desc_cnt - 1;
 570         rbdr->rbdr_status =
 571                 nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_STATUS0;
 572         rbdr->rbdr_door =
 573                 nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_DOOR;
 574
 575         nic->rbdr = rbdr;
 576         return 0;
 577 }
 578
 579 static void
 580 nicvf_rbdr_release_mbuf(struct nicvf *nic, nicvf_phys_addr_t phy)
 581 {
 582         uint16_t qidx;
 583         void *obj;
 584         struct nicvf_rxq *rxq;
 585
 586         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
 587                 rxq = nic->eth_dev->data->rx_queues[qidx];
 588                 if (rxq->precharge_cnt) {
 589                         obj = (void *)nicvf_mbuff_phy2virt(phy,
 590                                                            rxq->mbuf_phys_off);
 591                         rte_mempool_put(rxq->pool, obj);
 592                         rxq->precharge_cnt--;
 593                         break;
 594                 }
 595         }
 596 }
 597
 598 static inline void
 599 nicvf_rbdr_release_mbufs(struct nicvf *nic)
 600 {
 601         uint32_t qlen_mask, head;
 602         struct rbdr_entry_t *entry;
 603         struct nicvf_rbdr *rbdr = nic->rbdr;
 604
 605         qlen_mask = rbdr->qlen_mask;
 606         head = rbdr->head;
 607         while (head != rbdr->tail) {
 608                 entry = rbdr->desc + head;
 609                 nicvf_rbdr_release_mbuf(nic, entry->full_addr);
 610                 head++;
 611                 head = head & qlen_mask;
 612         }
 613 }
 614
 615 static inline void
 616 nicvf_tx_queue_release_mbufs(struct nicvf_txq *txq)
 617 {
 618         uint32_t head;
 619
 620         head = txq->head;
 621         while (head != txq->tail) {
 622                 if (txq->txbuffs[head]) {
 623                         rte_pktmbuf_free_seg(txq->txbuffs[head]);
 624                         txq->txbuffs[head] = NULL;
 625                 }
 626                 head++;
 627                 head = head & txq->qlen_mask;
 628         }
 629 }
 630
 631 static void
 632 nicvf_tx_queue_reset(struct nicvf_txq *txq)
 633 {
 634         uint32_t txq_desc_cnt = txq->qlen_mask + 1;
 635
 636         memset(txq->desc, 0, sizeof(union sq_entry_t) * txq_desc_cnt);
 637         memset(txq->txbuffs, 0, sizeof(struct rte_mbuf *) * txq_desc_cnt);
 638         txq->tail = 0;
 639         txq->head = 0;
 640         txq->xmit_bufs = 0;
 641 }
 642
 643 static inline int
 644 nicvf_start_tx_queue(struct rte_eth_dev *dev, uint16_t qidx)
 645 {
 646         struct nicvf_txq *txq;
 647         int ret;
 648
 649         if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED)
 650                 return 0;
 651
 652         txq = dev->data->tx_queues[qidx];
 653         txq->pool = NULL;
 654         ret = nicvf_qset_sq_config(nicvf_pmd_priv(dev), qidx, txq);
 655         if (ret) {
 656                 PMD_INIT_LOG(ERR, "Failed to configure sq %d %d", qidx, ret);
 657                 goto config_sq_error;
 658         }
 659
 660         dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STARTED;
 661         return ret;
 662
 663 config_sq_error:
 664         nicvf_qset_sq_reclaim(nicvf_pmd_priv(dev), qidx);
 665         return ret;
 666 }
 667
 668 static inline int
 669 nicvf_stop_tx_queue(struct rte_eth_dev *dev, uint16_t qidx)
 670 {
 671         struct nicvf_txq *txq;
 672         int ret;
 673
 674         if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STOPPED)
 675                 return 0;
 676
 677         ret = nicvf_qset_sq_reclaim(nicvf_pmd_priv(dev), qidx);
 678         if (ret)
 679                 PMD_INIT_LOG(ERR, "Failed to reclaim sq %d %d", qidx, ret);
 680
 681         txq = dev->data->tx_queues[qidx];
 682         nicvf_tx_queue_release_mbufs(txq);
 683         nicvf_tx_queue_reset(txq);
 684
 685         dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED;
 686         return ret;
 687 }
 688
 689 static inline int
 690 nicvf_configure_cpi(struct rte_eth_dev *dev)
 691 {
 692         struct nicvf *nic = nicvf_pmd_priv(dev);
 693         uint16_t qidx, qcnt;
 694         int ret;
 695
 696         /* Count started rx queues */
 697         for (qidx = qcnt = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++)
 698                 if (dev->data->rx_queue_state[qidx] ==
 699                     RTE_ETH_QUEUE_STATE_STARTED)
 700                         qcnt++;
 701
 702         nic->cpi_alg = CPI_ALG_NONE;
 703         ret = nicvf_mbox_config_cpi(nic, qcnt);
 704         if (ret)
 705                 PMD_INIT_LOG(ERR, "Failed to configure CPI %d", ret);
 706
 707         return ret;
 708 }
 709
 710 static inline int
 711 nicvf_configure_rss(struct rte_eth_dev *dev)
 712 {
 713         struct nicvf *nic = nicvf_pmd_priv(dev);
 714         uint64_t rsshf;
 715         int ret = -EINVAL;
 716
 717         rsshf = nicvf_rss_ethdev_to_nic(nic,
 718                         dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf);
 719         PMD_DRV_LOG(INFO, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64,
 720                     dev->data->dev_conf.rxmode.mq_mode,
 721                     nic->eth_dev->data->nb_rx_queues,
 722                     nic->eth_dev->data->dev_conf.lpbk_mode, rsshf);
 723
 724         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_NONE)
 725                 ret = nicvf_rss_term(nic);
 726         else if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS)
 727                 ret = nicvf_rss_config(nic,
 728                                        nic->eth_dev->data->nb_rx_queues, rsshf);
 729         if (ret)
 730                 PMD_INIT_LOG(ERR, "Failed to configure RSS %d", ret);
 731
 732         return ret;
 733 }
 734
 735 static int
 736 nicvf_configure_rss_reta(struct rte_eth_dev *dev)
 737 {
 738         struct nicvf *nic = nicvf_pmd_priv(dev);
 739         unsigned int idx, qmap_size;
 740         uint8_t qmap[RTE_MAX_QUEUES_PER_PORT];
 741         uint8_t default_reta[NIC_MAX_RSS_IDR_TBL_SIZE];
 742
 743         if (nic->cpi_alg != CPI_ALG_NONE)
 744                 return -EINVAL;
 745
 746         /* Prepare queue map */
 747         for (idx = 0, qmap_size = 0; idx < dev->data->nb_rx_queues; idx++) {
 748                 if (dev->data->rx_queue_state[idx] ==
 749                                 RTE_ETH_QUEUE_STATE_STARTED)
 750                         qmap[qmap_size++] = idx;
 751         }
 752
 753         /* Update default RSS RETA */
 754         for (idx = 0; idx < NIC_MAX_RSS_IDR_TBL_SIZE; idx++)
 755                 default_reta[idx] = qmap[idx % qmap_size];
 756
 757         return nicvf_rss_reta_update(nic, default_reta,
 758                                      NIC_MAX_RSS_IDR_TBL_SIZE);
 759 }
 760
 761 static void
 762 nicvf_dev_tx_queue_release(void *sq)
 763 {
 764         struct nicvf_txq *txq;
 765
 766         PMD_INIT_FUNC_TRACE();
 767
 768         txq = (struct nicvf_txq *)sq;
 769         if (txq) {
 770                 if (txq->txbuffs != NULL) {
 771                         nicvf_tx_queue_release_mbufs(txq);
 772                         rte_free(txq->txbuffs);
 773                         txq->txbuffs = NULL;
 774                 }
 775                 rte_free(txq);
 776         }
 777 }
 778
 779 static void
 780 nicvf_set_tx_function(struct rte_eth_dev *dev)
 781 {
 782         struct nicvf_txq *txq;
 783         size_t i;
 784         bool multiseg = false;
 785
 786         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 787                 txq = dev->data->tx_queues[i];
 788                 if ((txq->txq_flags & ETH_TXQ_FLAGS_NOMULTSEGS) == 0) {
 789                         multiseg = true;
 790                         break;
 791                 }
 792         }
 793
 794         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
 795         if (multiseg) {
 796                 PMD_DRV_LOG(DEBUG, "Using multi-segment tx callback");
 797                 dev->tx_pkt_burst = nicvf_xmit_pkts_multiseg;
 798         } else {
 799                 PMD_DRV_LOG(DEBUG, "Using single-segment tx callback");
 800                 dev->tx_pkt_burst = nicvf_xmit_pkts;
 801         }
 802
 803         if (txq->pool_free == nicvf_single_pool_free_xmited_buffers)
 804                 PMD_DRV_LOG(DEBUG, "Using single-mempool tx free method");
 805         else
 806                 PMD_DRV_LOG(DEBUG, "Using multi-mempool tx free method");
 807 }
 808
 809 static void
 810 nicvf_set_rx_function(struct rte_eth_dev *dev)
 811 {
 812         if (dev->data->scattered_rx) {
 813                 PMD_DRV_LOG(DEBUG, "Using multi-segment rx callback");
 814                 dev->rx_pkt_burst = nicvf_recv_pkts_multiseg;
 815         } else {
 816                 PMD_DRV_LOG(DEBUG, "Using single-segment rx callback");
 817                 dev->rx_pkt_burst = nicvf_recv_pkts;
 818         }
 819 }
 820
 821 static int
 822 nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 823                          uint16_t nb_desc, unsigned int socket_id,
 824                          const struct rte_eth_txconf *tx_conf)
 825 {
 826         uint16_t tx_free_thresh;
 827         uint8_t is_single_pool;
 828         struct nicvf_txq *txq;
 829         struct nicvf *nic = nicvf_pmd_priv(dev);
 830
 831         PMD_INIT_FUNC_TRACE();
 832
 833         /* Socket id check */
 834         if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
 835                 PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
 836                 socket_id, nic->node);
 837
 838         /* Tx deferred start is not supported */
 839         if (tx_conf->tx_deferred_start) {
 840                 PMD_INIT_LOG(ERR, "Tx deferred start not supported");
 841                 return -EINVAL;
 842         }
 843
 844         /* Roundup nb_desc to available qsize and validate max number of desc */
 845         nb_desc = nicvf_qsize_sq_roundup(nb_desc);
 846         if (nb_desc == 0) {
 847                 PMD_INIT_LOG(ERR, "Value of nb_desc beyond available sq qsize");
 848                 return -EINVAL;
 849         }
 850
 851         /* Validate tx_free_thresh */
 852         tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
 853                                 tx_conf->tx_free_thresh :
 854                                 NICVF_DEFAULT_TX_FREE_THRESH);
 855
 856         if (tx_free_thresh > (nb_desc) ||
 857                 tx_free_thresh > NICVF_MAX_TX_FREE_THRESH) {
 858                 PMD_INIT_LOG(ERR,
 859                         "tx_free_thresh must be less than the number of TX "
 860                         "descriptors. (tx_free_thresh=%u port=%d "
 861                         "queue=%d)", (unsigned int)tx_free_thresh,
 862                         (int)dev->data->port_id, (int)qidx);
 863                 return -EINVAL;
 864         }
 865
 866         /* Free memory prior to re-allocation if needed. */
 867         if (dev->data->tx_queues[qidx] != NULL) {
 868                 PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
 869                                 qidx);
 870                 nicvf_dev_tx_queue_release(dev->data->tx_queues[qidx]);
 871                 dev->data->tx_queues[qidx] = NULL;
 872         }
 873
 874         /* Allocating tx queue data structure */
 875         txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq),
 876                                         RTE_CACHE_LINE_SIZE, nic->node);
 877         if (txq == NULL) {
 878                 PMD_INIT_LOG(ERR, "Failed to allocate txq=%d", qidx);
 879                 return -ENOMEM;
 880         }
 881
 882         txq->nic = nic;
 883         txq->queue_id = qidx;
 884         txq->tx_free_thresh = tx_free_thresh;
 885         txq->txq_flags = tx_conf->txq_flags;
 886         txq->sq_head = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_HEAD;
 887         txq->sq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_DOOR;
 888         is_single_pool = (txq->txq_flags & ETH_TXQ_FLAGS_NOREFCOUNT &&
 889                                 txq->txq_flags & ETH_TXQ_FLAGS_NOMULTMEMP);
 890
 891         /* Choose optimum free threshold value for multipool case */
 892         if (!is_single_pool) {
 893                 txq->tx_free_thresh = (uint16_t)
 894                 (tx_conf->tx_free_thresh == NICVF_DEFAULT_TX_FREE_THRESH ?
 895                                 NICVF_TX_FREE_MPOOL_THRESH :
 896                                 tx_conf->tx_free_thresh);
 897                 txq->pool_free = nicvf_multi_pool_free_xmited_buffers;
 898         } else {
 899                 txq->pool_free = nicvf_single_pool_free_xmited_buffers;
 900         }
 901
 902         /* Allocate software ring */
 903         txq->txbuffs = rte_zmalloc_socket("txq->txbuffs",
 904                                 nb_desc * sizeof(struct rte_mbuf *),
 905                                 RTE_CACHE_LINE_SIZE, nic->node);
 906
 907         if (txq->txbuffs == NULL) {
 908                 nicvf_dev_tx_queue_release(txq);
 909                 return -ENOMEM;
 910         }
 911
 912         if (nicvf_qset_sq_alloc(nic, txq, qidx, nb_desc)) {
 913                 PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx);
 914                 nicvf_dev_tx_queue_release(txq);
 915                 return -ENOMEM;
 916         }
 917
 918         nicvf_tx_queue_reset(txq);
 919
 920         PMD_TX_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p phys=0x%" PRIx64,
 921                         qidx, txq, nb_desc, txq->desc, txq->phys);
 922
 923         dev->data->tx_queues[qidx] = txq;
 924         dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED;
 925         return 0;
 926 }
 927
 928 static inline void
 929 nicvf_rx_queue_release_mbufs(struct nicvf_rxq *rxq)
 930 {
 931         uint32_t rxq_cnt;
 932         uint32_t nb_pkts, released_pkts = 0;
 933         uint32_t refill_cnt = 0;
 934         struct rte_eth_dev *dev = rxq->nic->eth_dev;
 935         struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH];
 936
 937         if (dev->rx_pkt_burst == NULL)
 938                 return;
 939
 940         while ((rxq_cnt = nicvf_dev_rx_queue_count(dev, rxq->queue_id))) {
 941                 nb_pkts = dev->rx_pkt_burst(rxq, rx_pkts,
 942                                         NICVF_MAX_RX_FREE_THRESH);
 943                 PMD_DRV_LOG(INFO, "nb_pkts=%d  rxq_cnt=%d", nb_pkts, rxq_cnt);
 944                 while (nb_pkts) {
 945                         rte_pktmbuf_free_seg(rx_pkts[--nb_pkts]);
 946                         released_pkts++;
 947                 }
 948         }
 949
 950         refill_cnt += nicvf_dev_rbdr_refill(dev, rxq->queue_id);
 951         PMD_DRV_LOG(INFO, "free_cnt=%d  refill_cnt=%d",
 952                     released_pkts, refill_cnt);
 953 }
 954
 955 static void
 956 nicvf_rx_queue_reset(struct nicvf_rxq *rxq)
 957 {
 958         rxq->head = 0;
 959         rxq->available_space = 0;
 960         rxq->recv_buffers = 0;
 961 }
 962
 963 static inline int
 964 nicvf_start_rx_queue(struct rte_eth_dev *dev, uint16_t qidx)
 965 {
 966         struct nicvf *nic = nicvf_pmd_priv(dev);
 967         struct nicvf_rxq *rxq;
 968         int ret;
 969
 970         if (dev->data->rx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED)
 971                 return 0;
 972
 973         /* Update rbdr pointer to all rxq */
 974         rxq = dev->data->rx_queues[qidx];
 975         rxq->shared_rbdr = nic->rbdr;
 976
 977         ret = nicvf_qset_rq_config(nic, qidx, rxq);
 978         if (ret) {
 979                 PMD_INIT_LOG(ERR, "Failed to configure rq %d %d", qidx, ret);
 980                 goto config_rq_error;
 981         }
 982         ret = nicvf_qset_cq_config(nic, qidx, rxq);
 983         if (ret) {
 984                 PMD_INIT_LOG(ERR, "Failed to configure cq %d %d", qidx, ret);
 985                 goto config_cq_error;
 986         }
 987
 988         dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STARTED;
 989         return 0;
 990
 991 config_cq_error:
 992         nicvf_qset_cq_reclaim(nic, qidx);
 993 config_rq_error:
 994         nicvf_qset_rq_reclaim(nic, qidx);
 995         return ret;
 996 }
 997
 998 static inline int
 999 nicvf_stop_rx_queue(struct rte_eth_dev *dev, uint16_t qidx)
1000 {
1001         struct nicvf *nic = nicvf_pmd_priv(dev);
1002         struct nicvf_rxq *rxq;
1003         int ret, other_error;
1004
1005         if (dev->data->rx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STOPPED)
1006                 return 0;
1007
1008         ret = nicvf_qset_rq_reclaim(nic, qidx);
1009         if (ret)
1010                 PMD_INIT_LOG(ERR, "Failed to reclaim rq %d %d", qidx, ret);
1011
1012         other_error = ret;
1013         rxq = dev->data->rx_queues[qidx];
1014         nicvf_rx_queue_release_mbufs(rxq);
1015         nicvf_rx_queue_reset(rxq);
1016
1017         ret = nicvf_qset_cq_reclaim(nic, qidx);
1018         if (ret)
1019                 PMD_INIT_LOG(ERR, "Failed to reclaim cq %d %d", qidx, ret);
1020
1021         other_error |= ret;
1022         dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED;
1023         return other_error;
1024 }
1025
1026 static void
1027 nicvf_dev_rx_queue_release(void *rx_queue)
1028 {
1029         struct nicvf_rxq *rxq = rx_queue;
1030
1031         PMD_INIT_FUNC_TRACE();
1032
1033         if (rxq)
1034                 rte_free(rxq);
1035 }
1036
1037 static int
1038 nicvf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1039 {
1040         int ret;
1041
1042         ret = nicvf_start_rx_queue(dev, qidx);
1043         if (ret)
1044                 return ret;
1045
1046         ret = nicvf_configure_cpi(dev);
1047         if (ret)
1048                 return ret;
1049
1050         return nicvf_configure_rss_reta(dev);
1051 }
1052
1053 static int
1054 nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1055 {
1056         int ret;
1057
1058         ret = nicvf_stop_rx_queue(dev, qidx);
1059         ret |= nicvf_configure_cpi(dev);
1060         ret |= nicvf_configure_rss_reta(dev);
1061         return ret;
1062 }
1063
1064 static int
1065 nicvf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1066 {
1067         return nicvf_start_tx_queue(dev, qidx);
1068 }
1069
1070 static int
1071 nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1072 {
1073         return nicvf_stop_tx_queue(dev, qidx);
1074 }
1075
1076 static int
1077 nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
1078                          uint16_t nb_desc, unsigned int socket_id,
1079                          const struct rte_eth_rxconf *rx_conf,
1080                          struct rte_mempool *mp)
1081 {
1082         uint16_t rx_free_thresh;
1083         struct nicvf_rxq *rxq;
1084         struct nicvf *nic = nicvf_pmd_priv(dev);
1085
1086         PMD_INIT_FUNC_TRACE();
1087
1088         /* Socket id check */
1089         if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
1090                 PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
1091                 socket_id, nic->node);
1092
1093         /* Mempool memory should be contiguous */
1094         if (mp->nb_mem_chunks != 1) {
1095                 PMD_INIT_LOG(ERR, "Non contiguous mempool, check huge page sz");
1096                 return -EINVAL;
1097         }
1098
1099         /* Rx deferred start is not supported */
1100         if (rx_conf->rx_deferred_start) {
1101                 PMD_INIT_LOG(ERR, "Rx deferred start not supported");
1102                 return -EINVAL;
1103         }
1104
1105         /* Roundup nb_desc to available qsize and validate max number of desc */
1106         nb_desc = nicvf_qsize_cq_roundup(nb_desc);
1107         if (nb_desc == 0) {
1108                 PMD_INIT_LOG(ERR, "Value nb_desc beyond available hw cq qsize");
1109                 return -EINVAL;
1110         }
1111
1112         /* Check rx_free_thresh upper bound */
1113         rx_free_thresh = (uint16_t)((rx_conf->rx_free_thresh) ?
1114                                 rx_conf->rx_free_thresh :
1115                                 NICVF_DEFAULT_RX_FREE_THRESH);
1116         if (rx_free_thresh > NICVF_MAX_RX_FREE_THRESH ||
1117                 rx_free_thresh >= nb_desc * .75) {
1118                 PMD_INIT_LOG(ERR, "rx_free_thresh greater than expected %d",
1119                                 rx_free_thresh);
1120                 return -EINVAL;
1121         }
1122
1123         /* Free memory prior to re-allocation if needed */
1124         if (dev->data->rx_queues[qidx] != NULL) {
1125                 PMD_RX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
1126                                 qidx);
1127                 nicvf_dev_rx_queue_release(dev->data->rx_queues[qidx]);
1128                 dev->data->rx_queues[qidx] = NULL;
1129         }
1130
1131         /* Allocate rxq memory */
1132         rxq = rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq),
1133                                         RTE_CACHE_LINE_SIZE, nic->node);
1134         if (rxq == NULL) {
1135                 PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d", qidx);
1136                 return -ENOMEM;
1137         }
1138
1139         rxq->nic = nic;
1140         rxq->pool = mp;
1141         rxq->queue_id = qidx;
1142         rxq->port_id = dev->data->port_id;
1143         rxq->rx_free_thresh = rx_free_thresh;
1144         rxq->rx_drop_en = rx_conf->rx_drop_en;
1145         rxq->cq_status = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_STATUS;
1146         rxq->cq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_DOOR;
1147         rxq->precharge_cnt = 0;
1148         rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD;
1149
1150         /* Alloc completion queue */
1151         if (nicvf_qset_cq_alloc(nic, rxq, rxq->queue_id, nb_desc)) {
1152                 PMD_INIT_LOG(ERR, "failed to allocate cq %u", rxq->queue_id);
1153                 nicvf_dev_rx_queue_release(rxq);
1154                 return -ENOMEM;
1155         }
1156
1157         nicvf_rx_queue_reset(rxq);
1158
1159         PMD_RX_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d) phy=%" PRIx64,
1160                         qidx, rxq, mp->name, nb_desc,
1161                         rte_mempool_avail_count(mp), rxq->phys);
1162
1163         dev->data->rx_queues[qidx] = rxq;
1164         dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED;
1165         return 0;
1166 }
1167
1168 static void
1169 nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1170 {
1171         struct nicvf *nic = nicvf_pmd_priv(dev);
1172
1173         PMD_INIT_FUNC_TRACE();
1174
1175         dev_info->min_rx_bufsize = ETHER_MIN_MTU;
1176         dev_info->max_rx_pktlen = NIC_HW_MAX_FRS;
1177         dev_info->max_rx_queues = (uint16_t)MAX_RCV_QUEUES_PER_QS;
1178         dev_info->max_tx_queues = (uint16_t)MAX_SND_QUEUES_PER_QS;
1179         dev_info->max_mac_addrs = 1;
1180         dev_info->max_vfs = dev->pci_dev->max_vfs;
1181
1182         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1183         dev_info->tx_offload_capa =
1184                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1185                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1186                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1187                 DEV_TX_OFFLOAD_TCP_TSO     |
1188                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
1189
1190         dev_info->reta_size = nic->rss_info.rss_size;
1191         dev_info->hash_key_size = RSS_HASH_KEY_BYTE_SIZE;
1192         dev_info->flow_type_rss_offloads = NICVF_RSS_OFFLOAD_PASS1;
1193         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING)
1194                 dev_info->flow_type_rss_offloads |= NICVF_RSS_OFFLOAD_TUNNEL;
1195
1196         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1197                 .rx_free_thresh = NICVF_DEFAULT_RX_FREE_THRESH,
1198                 .rx_drop_en = 0,
1199         };
1200
1201         dev_info->default_txconf = (struct rte_eth_txconf) {
1202                 .tx_free_thresh = NICVF_DEFAULT_TX_FREE_THRESH,
1203                 .txq_flags =
1204                         ETH_TXQ_FLAGS_NOMULTSEGS  |
1205                         ETH_TXQ_FLAGS_NOREFCOUNT  |
1206                         ETH_TXQ_FLAGS_NOMULTMEMP  |
1207                         ETH_TXQ_FLAGS_NOVLANOFFL  |
1208                         ETH_TXQ_FLAGS_NOXSUMSCTP,
1209         };
1210 }
1211
1212 static nicvf_phys_addr_t
1213 rbdr_rte_mempool_get(void *opaque)
1214 {
1215         uint16_t qidx;
1216         uintptr_t mbuf;
1217         struct nicvf_rxq *rxq;
1218         struct nicvf *nic = nicvf_pmd_priv((struct rte_eth_dev *)opaque);
1219
1220         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1221                 rxq = nic->eth_dev->data->rx_queues[qidx];
1222                 /* Maintain equal buffer count across all pools */
1223                 if (rxq->precharge_cnt >= rxq->qlen_mask)
1224                         continue;
1225                 rxq->precharge_cnt++;
1226                 mbuf = (uintptr_t)rte_pktmbuf_alloc(rxq->pool);
1227                 if (mbuf)
1228                         return nicvf_mbuff_virt2phy(mbuf, rxq->mbuf_phys_off);
1229         }
1230         return 0;
1231 }
1232
1233 static int
1234 nicvf_dev_start(struct rte_eth_dev *dev)
1235 {
1236         int ret;
1237         uint16_t qidx;
1238         uint32_t buffsz = 0, rbdrsz = 0;
1239         uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs;
1240         uint64_t mbuf_phys_off = 0;
1241         struct nicvf_rxq *rxq;
1242         struct rte_pktmbuf_pool_private *mbp_priv;
1243         struct rte_mbuf *mbuf;
1244         struct nicvf *nic = nicvf_pmd_priv(dev);
1245         struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
1246         uint16_t mtu;
1247
1248         PMD_INIT_FUNC_TRACE();
1249
1250         /* Userspace process exited without proper shutdown in last run */
1251         if (nicvf_qset_rbdr_active(nic, 0))
1252                 nicvf_dev_stop(dev);
1253
1254         /*
1255          * Thunderx nicvf PMD can support more than one pool per port only when
1256          * 1) Data payload size is same across all the pools in given port
1257          * AND
1258          * 2) All mbuffs in the pools are from the same hugepage
1259          * AND
1260          * 3) Mbuff metadata size is same across all the pools in given port
1261          *
1262          * This is to support existing application that uses multiple pool/port.
1263          * But, the purpose of using multipool for QoS will not be addressed.
1264          *
1265          */
1266
1267         /* Validate RBDR buff size */
1268         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1269                 rxq = dev->data->rx_queues[qidx];
1270                 mbp_priv = rte_mempool_get_priv(rxq->pool);
1271                 buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
1272                 if (buffsz % 128) {
1273                         PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128");
1274                         return -EINVAL;
1275                 }
1276                 if (rbdrsz == 0)
1277                         rbdrsz = buffsz;
1278                 if (rbdrsz != buffsz) {
1279                         PMD_INIT_LOG(ERR, "buffsz not same, qid=%d (%d/%d)",
1280                                      qidx, rbdrsz, buffsz);
1281                         return -EINVAL;
1282                 }
1283         }
1284
1285         /* Validate mempool attributes */
1286         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1287                 rxq = dev->data->rx_queues[qidx];
1288                 rxq->mbuf_phys_off = nicvf_mempool_phy_offset(rxq->pool);
1289                 mbuf = rte_pktmbuf_alloc(rxq->pool);
1290                 if (mbuf == NULL) {
1291                         PMD_INIT_LOG(ERR, "Failed allocate mbuf qid=%d pool=%s",
1292                                      qidx, rxq->pool->name);
1293                         return -ENOMEM;
1294                 }
1295                 rxq->mbuf_phys_off -= nicvf_mbuff_meta_length(mbuf);
1296                 rxq->mbuf_phys_off -= RTE_PKTMBUF_HEADROOM;
1297                 rte_pktmbuf_free(mbuf);
1298
1299                 if (mbuf_phys_off == 0)
1300                         mbuf_phys_off = rxq->mbuf_phys_off;
1301                 if (mbuf_phys_off != rxq->mbuf_phys_off) {
1302                         PMD_INIT_LOG(ERR, "pool params not same,%s %" PRIx64,
1303                                      rxq->pool->name, mbuf_phys_off);
1304                         return -EINVAL;
1305                 }
1306         }
1307
1308         /* Check the level of buffers in the pool */
1309         total_rxq_desc = 0;
1310         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1311                 rxq = dev->data->rx_queues[qidx];
1312                 /* Count total numbers of rxq descs */
1313                 total_rxq_desc += rxq->qlen_mask + 1;
1314                 exp_buffs = RTE_MEMPOOL_CACHE_MAX_SIZE + rxq->rx_free_thresh;
1315                 exp_buffs *= nic->eth_dev->data->nb_rx_queues;
1316                 if (rte_mempool_avail_count(rxq->pool) < exp_buffs) {
1317                         PMD_INIT_LOG(ERR, "Buff shortage in pool=%s (%d/%d)",
1318                                      rxq->pool->name,
1319                                      rte_mempool_avail_count(rxq->pool),
1320                                      exp_buffs);
1321                         return -ENOENT;
1322                 }
1323         }
1324
1325         /* Check RBDR desc overflow */
1326         ret = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1327         if (ret == 0) {
1328                 PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc");
1329                 return -ENOMEM;
1330         }
1331
1332         /* Enable qset */
1333         ret = nicvf_qset_config(nic);
1334         if (ret) {
1335                 PMD_INIT_LOG(ERR, "Failed to enable qset %d", ret);
1336                 return ret;
1337         }
1338
1339         /* Allocate RBDR and RBDR ring desc */
1340         nb_rbdr_desc = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1341         ret = nicvf_qset_rbdr_alloc(nic, nb_rbdr_desc, rbdrsz);
1342         if (ret) {
1343                 PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc");
1344                 goto qset_reclaim;
1345         }
1346
1347         /* Enable and configure RBDR registers */
1348         ret = nicvf_qset_rbdr_config(nic, 0);
1349         if (ret) {
1350                 PMD_INIT_LOG(ERR, "Failed to configure rbdr %d", ret);
1351                 goto qset_rbdr_free;
1352         }
1353
1354         /* Fill rte_mempool buffers in RBDR pool and precharge it */
1355         ret = nicvf_qset_rbdr_precharge(nic, 0, rbdr_rte_mempool_get,
1356                                         dev, total_rxq_desc);
1357         if (ret) {
1358                 PMD_INIT_LOG(ERR, "Failed to fill rbdr %d", ret);
1359                 goto qset_rbdr_reclaim;
1360         }
1361
1362         PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR",
1363                      nic->rbdr->tail, nb_rbdr_desc);
1364
1365         /* Configure RX queues */
1366         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1367                 ret = nicvf_start_rx_queue(dev, qidx);
1368                 if (ret)
1369                         goto start_rxq_error;
1370         }
1371
1372         /* Configure VLAN Strip */
1373         nicvf_vlan_hw_strip(nic, dev->data->dev_conf.rxmode.hw_vlan_strip);
1374
1375         /* Configure TX queues */
1376         for (qidx = 0; qidx < nic->eth_dev->data->nb_tx_queues; qidx++) {
1377                 ret = nicvf_start_tx_queue(dev, qidx);
1378                 if (ret)
1379                         goto start_txq_error;
1380         }
1381
1382         /* Configure CPI algorithm */
1383         ret = nicvf_configure_cpi(dev);
1384         if (ret)
1385                 goto start_txq_error;
1386
1387         /* Configure RSS */
1388         ret = nicvf_configure_rss(dev);
1389         if (ret)
1390                 goto qset_rss_error;
1391
1392         /* Configure loopback */
1393         ret = nicvf_loopback_config(nic, dev->data->dev_conf.lpbk_mode);
1394         if (ret) {
1395                 PMD_INIT_LOG(ERR, "Failed to configure loopback %d", ret);
1396                 goto qset_rss_error;
1397         }
1398
1399         /* Reset all statistics counters attached to this port */
1400         ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF);
1401         if (ret) {
1402                 PMD_INIT_LOG(ERR, "Failed to reset stat counters %d", ret);
1403                 goto qset_rss_error;
1404         }
1405
1406         /* Setup scatter mode if needed by jumbo */
1407         if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
1408                                             2 * VLAN_TAG_SIZE > buffsz)
1409                 dev->data->scattered_rx = 1;
1410         if (rx_conf->enable_scatter)
1411                 dev->data->scattered_rx = 1;
1412
1413         /* Setup MTU based on max_rx_pkt_len or default */
1414         mtu = dev->data->dev_conf.rxmode.jumbo_frame ?
1415                 dev->data->dev_conf.rxmode.max_rx_pkt_len
1416                         -  ETHER_HDR_LEN - ETHER_CRC_LEN
1417                 : ETHER_MTU;
1418
1419         if (nicvf_dev_set_mtu(dev, mtu)) {
1420                 PMD_INIT_LOG(ERR, "Failed to set default mtu size");
1421                 return -EBUSY;
1422         }
1423
1424         /* Configure callbacks based on scatter mode */
1425         nicvf_set_tx_function(dev);
1426         nicvf_set_rx_function(dev);
1427
1428         /* Done; Let PF make the BGX's RX and TX switches to ON position */
1429         nicvf_mbox_cfg_done(nic);
1430         return 0;
1431
1432 qset_rss_error:
1433         nicvf_rss_term(nic);
1434 start_txq_error:
1435         for (qidx = 0; qidx < nic->eth_dev->data->nb_tx_queues; qidx++)
1436                 nicvf_stop_tx_queue(dev, qidx);
1437 start_rxq_error:
1438         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++)
1439                 nicvf_stop_rx_queue(dev, qidx);
1440 qset_rbdr_reclaim:
1441         nicvf_qset_rbdr_reclaim(nic, 0);
1442         nicvf_rbdr_release_mbufs(nic);
1443 qset_rbdr_free:
1444         if (nic->rbdr) {
1445                 rte_free(nic->rbdr);
1446                 nic->rbdr = NULL;
1447         }
1448 qset_reclaim:
1449         nicvf_qset_reclaim(nic);
1450         return ret;
1451 }
1452
1453 static void
1454 nicvf_dev_stop(struct rte_eth_dev *dev)
1455 {
1456         int ret;
1457         uint16_t qidx;
1458         struct nicvf *nic = nicvf_pmd_priv(dev);
1459
1460         PMD_INIT_FUNC_TRACE();
1461
1462         /* Let PF make the BGX's RX and TX switches to OFF position */
1463         nicvf_mbox_shutdown(nic);
1464
1465         /* Disable loopback */
1466         ret = nicvf_loopback_config(nic, 0);
1467         if (ret)
1468                 PMD_INIT_LOG(ERR, "Failed to disable loopback %d", ret);
1469
1470         /* Disable VLAN Strip */
1471         nicvf_vlan_hw_strip(nic, 0);
1472
1473         /* Reclaim sq */
1474         for (qidx = 0; qidx < dev->data->nb_tx_queues; qidx++)
1475                 nicvf_stop_tx_queue(dev, qidx);
1476
1477         /* Reclaim rq */
1478         for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++)
1479                 nicvf_stop_rx_queue(dev, qidx);
1480
1481         /* Reclaim RBDR */
1482         ret = nicvf_qset_rbdr_reclaim(nic, 0);
1483         if (ret)
1484                 PMD_INIT_LOG(ERR, "Failed to reclaim RBDR %d", ret);
1485
1486         /* Move all charged buffers in RBDR back to pool */
1487         if (nic->rbdr != NULL)
1488                 nicvf_rbdr_release_mbufs(nic);
1489
1490         /* Reclaim CPI configuration */
1491         if (!nic->sqs_mode) {
1492                 ret = nicvf_mbox_config_cpi(nic, 0);
1493                 if (ret)
1494                         PMD_INIT_LOG(ERR, "Failed to reclaim CPI config");
1495         }
1496
1497         /* Disable qset */
1498         ret = nicvf_qset_config(nic);
1499         if (ret)
1500                 PMD_INIT_LOG(ERR, "Failed to disable qset %d", ret);
1501
1502         /* Disable all interrupts */
1503         nicvf_disable_all_interrupts(nic);
1504
1505         /* Free RBDR SW structure */
1506         if (nic->rbdr) {
1507                 rte_free(nic->rbdr);
1508                 nic->rbdr = NULL;
1509         }
1510 }
1511
1512 static void
1513 nicvf_dev_close(struct rte_eth_dev *dev)
1514 {
1515         struct nicvf *nic = nicvf_pmd_priv(dev);
1516
1517         PMD_INIT_FUNC_TRACE();
1518
1519         nicvf_dev_stop(dev);
1520         nicvf_periodic_alarm_stop(nic);
1521 }
1522
1523 static int
1524 nicvf_dev_configure(struct rte_eth_dev *dev)
1525 {
1526         struct rte_eth_conf *conf = &dev->data->dev_conf;
1527         struct rte_eth_rxmode *rxmode = &conf->rxmode;
1528         struct rte_eth_txmode *txmode = &conf->txmode;
1529         struct nicvf *nic = nicvf_pmd_priv(dev);
1530
1531         PMD_INIT_FUNC_TRACE();
1532
1533         if (!rte_eal_has_hugepages()) {
1534                 PMD_INIT_LOG(INFO, "Huge page is not configured");
1535                 return -EINVAL;
1536         }
1537
1538         if (txmode->mq_mode) {
1539                 PMD_INIT_LOG(INFO, "Tx mq_mode DCB or VMDq not supported");
1540                 return -EINVAL;
1541         }
1542
1543         if (rxmode->mq_mode != ETH_MQ_RX_NONE &&
1544                 rxmode->mq_mode != ETH_MQ_RX_RSS) {
1545                 PMD_INIT_LOG(INFO, "Unsupported rx qmode %d", rxmode->mq_mode);
1546                 return -EINVAL;
1547         }
1548
1549         if (!rxmode->hw_strip_crc) {
1550                 PMD_INIT_LOG(NOTICE, "Can't disable hw crc strip");
1551                 rxmode->hw_strip_crc = 1;
1552         }
1553
1554         if (rxmode->hw_ip_checksum) {
1555                 PMD_INIT_LOG(NOTICE, "Rxcksum not supported");
1556                 rxmode->hw_ip_checksum = 0;
1557         }
1558
1559         if (rxmode->split_hdr_size) {
1560                 PMD_INIT_LOG(INFO, "Rxmode does not support split header");
1561                 return -EINVAL;
1562         }
1563
1564         if (rxmode->hw_vlan_filter) {
1565                 PMD_INIT_LOG(INFO, "VLAN filter not supported");
1566                 return -EINVAL;
1567         }
1568
1569         if (rxmode->hw_vlan_extend) {
1570                 PMD_INIT_LOG(INFO, "VLAN extended not supported");
1571                 return -EINVAL;
1572         }
1573
1574         if (rxmode->enable_lro) {
1575                 PMD_INIT_LOG(INFO, "LRO not supported");
1576                 return -EINVAL;
1577         }
1578
1579         if (conf->link_speeds & ETH_LINK_SPEED_FIXED) {
1580                 PMD_INIT_LOG(INFO, "Setting link speed/duplex not supported");
1581                 return -EINVAL;
1582         }
1583
1584         if (conf->dcb_capability_en) {
1585                 PMD_INIT_LOG(INFO, "DCB enable not supported");
1586                 return -EINVAL;
1587         }
1588
1589         if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1590                 PMD_INIT_LOG(INFO, "Flow director not supported");
1591                 return -EINVAL;
1592         }
1593
1594         PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64,
1595                 dev->data->port_id, nicvf_hw_cap(nic));
1596
1597         return 0;
1598 }
1599
1600 /* Initialize and register driver with DPDK Application */
1601 static const struct eth_dev_ops nicvf_eth_dev_ops = {
1602         .dev_configure            = nicvf_dev_configure,
1603         .dev_start                = nicvf_dev_start,
1604         .dev_stop                 = nicvf_dev_stop,
1605         .link_update              = nicvf_dev_link_update,
1606         .dev_close                = nicvf_dev_close,
1607         .stats_get                = nicvf_dev_stats_get,
1608         .stats_reset              = nicvf_dev_stats_reset,
1609         .promiscuous_enable       = nicvf_dev_promisc_enable,
1610         .dev_infos_get            = nicvf_dev_info_get,
1611         .dev_supported_ptypes_get = nicvf_dev_supported_ptypes_get,
1612         .mtu_set                  = nicvf_dev_set_mtu,
1613         .reta_update              = nicvf_dev_reta_update,
1614         .reta_query               = nicvf_dev_reta_query,
1615         .rss_hash_update          = nicvf_dev_rss_hash_update,
1616         .rss_hash_conf_get        = nicvf_dev_rss_hash_conf_get,
1617         .rx_queue_start           = nicvf_dev_rx_queue_start,
1618         .rx_queue_stop            = nicvf_dev_rx_queue_stop,
1619         .tx_queue_start           = nicvf_dev_tx_queue_start,
1620         .tx_queue_stop            = nicvf_dev_tx_queue_stop,
1621         .rx_queue_setup           = nicvf_dev_rx_queue_setup,
1622         .rx_queue_release         = nicvf_dev_rx_queue_release,
1623         .rx_queue_count           = nicvf_dev_rx_queue_count,
1624         .tx_queue_setup           = nicvf_dev_tx_queue_setup,
1625         .tx_queue_release         = nicvf_dev_tx_queue_release,
1626         .get_reg_length           = nicvf_dev_get_reg_length,
1627         .get_reg                  = nicvf_dev_get_regs,
1628 };
1629
1630 static int
1631 nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
1632 {
1633         int ret;
1634         struct rte_pci_device *pci_dev;
1635         struct nicvf *nic = nicvf_pmd_priv(eth_dev);
1636
1637         PMD_INIT_FUNC_TRACE();
1638
1639         eth_dev->dev_ops = &nicvf_eth_dev_ops;
1640
1641         /* For secondary processes, the primary has done all the work */
1642         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1643                 /* Setup callbacks for secondary process */
1644                 nicvf_set_tx_function(eth_dev);
1645                 nicvf_set_rx_function(eth_dev);
1646                 return 0;
1647         }
1648
1649         pci_dev = eth_dev->pci_dev;
1650         rte_eth_copy_pci_info(eth_dev, pci_dev);
1651
1652         nic->device_id = pci_dev->id.device_id;
1653         nic->vendor_id = pci_dev->id.vendor_id;
1654         nic->subsystem_device_id = pci_dev->id.subsystem_device_id;
1655         nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1656         nic->eth_dev = eth_dev;
1657
1658         PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) %u:%u:%u:%u",
1659                         pci_dev->id.vendor_id, pci_dev->id.device_id,
1660                         pci_dev->addr.domain, pci_dev->addr.bus,
1661                         pci_dev->addr.devid, pci_dev->addr.function);
1662
1663         nic->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
1664         if (!nic->reg_base) {
1665                 PMD_INIT_LOG(ERR, "Failed to map BAR0");
1666                 ret = -ENODEV;
1667                 goto fail;
1668         }
1669
1670         nicvf_disable_all_interrupts(nic);
1671
1672         ret = nicvf_periodic_alarm_start(nic);
1673         if (ret) {
1674                 PMD_INIT_LOG(ERR, "Failed to start period alarm");
1675                 goto fail;
1676         }
1677
1678         ret = nicvf_mbox_check_pf_ready(nic);
1679         if (ret) {
1680                 PMD_INIT_LOG(ERR, "Failed to get ready message from PF");
1681                 goto alarm_fail;
1682         } else {
1683                 PMD_INIT_LOG(INFO,
1684                         "node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
1685                         nic->node, nic->vf_id,
1686                         nic->tns_mode == NIC_TNS_MODE ? "tns" : "tns-bypass",
1687                         nic->sqs_mode ? "true" : "false",
1688                         nic->loopback_supported ? "true" : "false"
1689                         );
1690         }
1691
1692         if (nic->sqs_mode) {
1693                 PMD_INIT_LOG(INFO, "Unsupported SQS VF detected, Detaching...");
1694                 /* Detach port by returning Positive error number */
1695                 ret = ENOTSUP;
1696                 goto alarm_fail;
1697         }
1698
1699         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0);
1700         if (eth_dev->data->mac_addrs == NULL) {
1701                 PMD_INIT_LOG(ERR, "Failed to allocate memory for mac addr");
1702                 ret = -ENOMEM;
1703                 goto alarm_fail;
1704         }
1705         if (is_zero_ether_addr((struct ether_addr *)nic->mac_addr))
1706                 eth_random_addr(&nic->mac_addr[0]);
1707
1708         ether_addr_copy((struct ether_addr *)nic->mac_addr,
1709                         &eth_dev->data->mac_addrs[0]);
1710
1711         ret = nicvf_mbox_set_mac_addr(nic, nic->mac_addr);
1712         if (ret) {
1713                 PMD_INIT_LOG(ERR, "Failed to set mac addr");
1714                 goto malloc_fail;
1715         }
1716
1717         ret = nicvf_base_init(nic);
1718         if (ret) {
1719                 PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init");
1720                 goto malloc_fail;
1721         }
1722
1723         ret = nicvf_mbox_get_rss_size(nic);
1724         if (ret) {
1725                 PMD_INIT_LOG(ERR, "Failed to get rss table size");
1726                 goto malloc_fail;
1727         }
1728
1729         PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
1730                 eth_dev->data->port_id, nic->vendor_id, nic->device_id,
1731                 nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],
1732                 nic->mac_addr[3], nic->mac_addr[4], nic->mac_addr[5]);
1733
1734         return 0;
1735
1736 malloc_fail:
1737         rte_free(eth_dev->data->mac_addrs);
1738 alarm_fail:
1739         nicvf_periodic_alarm_stop(nic);
1740 fail:
1741         return ret;
1742 }
1743
1744 static const struct rte_pci_id pci_id_nicvf_map[] = {
1745         {
1746                 .class_id = RTE_CLASS_ANY_ID,
1747                 .vendor_id = PCI_VENDOR_ID_CAVIUM,
1748                 .device_id = PCI_DEVICE_ID_THUNDERX_PASS1_NICVF,
1749                 .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
1750                 .subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS1_NICVF,
1751         },
1752         {
1753                 .class_id = RTE_CLASS_ANY_ID,
1754                 .vendor_id = PCI_VENDOR_ID_CAVIUM,
1755                 .device_id = PCI_DEVICE_ID_THUNDERX_PASS2_NICVF,
1756                 .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
1757                 .subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS2_NICVF,
1758         },
1759         {
1760                 .vendor_id = 0,
1761         },
1762 };
1763
1764 static struct eth_driver rte_nicvf_pmd = {
1765         .pci_drv = {
1766                 .name = "rte_nicvf_pmd",
1767                 .id_table = pci_id_nicvf_map,
1768                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1769         },
1770         .eth_dev_init = nicvf_eth_dev_init,
1771         .dev_private_size = sizeof(struct nicvf),
1772 };
1773
1774 static int
1775 rte_nicvf_pmd_init(const char *name __rte_unused, const char *para __rte_unused)
1776 {
1777         PMD_INIT_FUNC_TRACE();
1778         PMD_INIT_LOG(INFO, "librte_pmd_thunderx nicvf version %s",
1779                         THUNDERX_NICVF_PMD_VERSION);
1780
1781         rte_eth_driver_register(&rte_nicvf_pmd);
1782         return 0;
1783 }
1784
1785 static struct rte_driver rte_nicvf_driver = {
1786         .name = "nicvf_driver",
1787         .type = PMD_PDEV,
1788         .init = rte_nicvf_pmd_init,
1789 };
1790
1791 PMD_REGISTER_DRIVER(rte_nicvf_driver);