drivers/net/thunderx/nicvf_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016 Cavium, Inc
   3  */
   4
   5 #include <assert.h>
   6 #include <stdio.h>
   7 #include <stdbool.h>
   8 #include <errno.h>
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <stdarg.h>
  13 #include <inttypes.h>
  14 #include <netinet/in.h>
  15 #include <sys/queue.h>
  16
  17 #include <rte_alarm.h>
  18 #include <rte_branch_prediction.h>
  19 #include <rte_byteorder.h>
  20 #include <rte_common.h>
  21 #include <rte_cycles.h>
  22 #include <rte_debug.h>
  23 #include <rte_dev.h>
  24 #include <rte_eal.h>
  25 #include <rte_ether.h>
  26 #include <ethdev_driver.h>
  27 #include <ethdev_pci.h>
  28 #include <rte_interrupts.h>
  29 #include <rte_log.h>
  30 #include <rte_memory.h>
  31 #include <rte_memzone.h>
  32 #include <rte_malloc.h>
  33 #include <rte_random.h>
  34 #include <rte_pci.h>
  35 #include <rte_bus_pci.h>
  36 #include <rte_tailq.h>
  37 #include <rte_devargs.h>
  38 #include <rte_kvargs.h>
  39
  40 #include "base/nicvf_plat.h"
  41
  42 #include "nicvf_ethdev.h"
  43 #include "nicvf_rxtx.h"
  44 #include "nicvf_svf.h"
  45 #include "nicvf_logs.h"
  46
  47 static int nicvf_dev_stop(struct rte_eth_dev *dev);
  48 static void nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup);
  49 static void nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic,
  50                           bool cleanup);
  51 static int nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask);
  52 static int nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  53
  54 RTE_LOG_REGISTER_SUFFIX(nicvf_logtype_mbox, mbox, NOTICE);
  55 RTE_LOG_REGISTER_SUFFIX(nicvf_logtype_init, init, NOTICE);
  56 RTE_LOG_REGISTER_SUFFIX(nicvf_logtype_driver, driver, NOTICE);
  57
  58 static void
  59 nicvf_link_status_update(struct nicvf *nic,
  60                          struct rte_eth_link *link)
  61 {
  62         memset(link, 0, sizeof(*link));
  63
  64         link->link_status = nic->link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
  65
  66         if (nic->duplex == NICVF_HALF_DUPLEX)
  67                 link->link_duplex = ETH_LINK_HALF_DUPLEX;
  68         else if (nic->duplex == NICVF_FULL_DUPLEX)
  69                 link->link_duplex = ETH_LINK_FULL_DUPLEX;
  70         link->link_speed = nic->speed;
  71         link->link_autoneg = ETH_LINK_AUTONEG;
  72 }
  73
  74 static void
  75 nicvf_interrupt(void *arg)
  76 {
  77         struct rte_eth_dev *dev = arg;
  78         struct nicvf *nic = nicvf_pmd_priv(dev);
  79         struct rte_eth_link link;
  80
  81         if (nicvf_reg_poll_interrupts(nic) == NIC_MBOX_MSG_BGX_LINK_CHANGE) {
  82                 if (dev->data->dev_conf.intr_conf.lsc) {
  83                         nicvf_link_status_update(nic, &link);
  84                         rte_eth_linkstatus_set(dev, &link);
  85
  86                         rte_eth_dev_callback_process(dev,
  87                                                      RTE_ETH_EVENT_INTR_LSC,
  88                                                      NULL);
  89                 }
  90         }
  91
  92         rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
  93                                 nicvf_interrupt, dev);
  94 }
  95
  96 static void
  97 nicvf_vf_interrupt(void *arg)
  98 {
  99         struct nicvf *nic = arg;
 100
 101         nicvf_reg_poll_interrupts(nic);
 102
 103         rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
 104                                 nicvf_vf_interrupt, nic);
 105 }
 106
 107 static int
 108 nicvf_periodic_alarm_start(void (fn)(void *), void *arg)
 109 {
 110         return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, fn, arg);
 111 }
 112
 113 static int
 114 nicvf_periodic_alarm_stop(void (fn)(void *), void *arg)
 115 {
 116         return rte_eal_alarm_cancel(fn, arg);
 117 }
 118
 119 /*
 120  * Return 0 means link status changed, -1 means not changed
 121  */
 122 static int
 123 nicvf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 124 {
 125 #define CHECK_INTERVAL 100  /* 100ms */
 126 #define MAX_CHECK_TIME 90   /* 9s (90 * 100ms) in total */
 127         struct rte_eth_link link;
 128         struct nicvf *nic = nicvf_pmd_priv(dev);
 129         int i;
 130
 131         PMD_INIT_FUNC_TRACE();
 132
 133         if (wait_to_complete) {
 134                 /* rte_eth_link_get() might need to wait up to 9 seconds */
 135                 for (i = 0; i < MAX_CHECK_TIME; i++) {
 136                         nicvf_link_status_update(nic, &link);
 137                         if (link.link_status == ETH_LINK_UP)
 138                                 break;
 139                         rte_delay_ms(CHECK_INTERVAL);
 140                 }
 141         } else {
 142                 nicvf_link_status_update(nic, &link);
 143         }
 144
 145         return rte_eth_linkstatus_set(dev, &link);
 146 }
 147
 148 static int
 149 nicvf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 150 {
 151         struct nicvf *nic = nicvf_pmd_priv(dev);
 152         uint32_t buffsz, frame_size = mtu + NIC_HW_L2_OVERHEAD;
 153         size_t i;
 154         struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
 155
 156         PMD_INIT_FUNC_TRACE();
 157
 158         if (frame_size > NIC_HW_MAX_FRS)
 159                 return -EINVAL;
 160
 161         if (frame_size < NIC_HW_MIN_FRS)
 162                 return -EINVAL;
 163
 164         buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 165
 166         /*
 167          * Refuse mtu that requires the support of scattered packets
 168          * when this feature has not been enabled before.
 169          */
 170         if (dev->data->dev_started && !dev->data->scattered_rx &&
 171                 (frame_size + 2 * VLAN_TAG_SIZE > buffsz))
 172                 return -EINVAL;
 173
 174         /* check <seg size> * <max_seg>  >= max_frame */
 175         if (dev->data->scattered_rx &&
 176                 (frame_size + 2 * VLAN_TAG_SIZE > buffsz * NIC_HW_MAX_SEGS))
 177                 return -EINVAL;
 178
 179         if (mtu > RTE_ETHER_MTU)
 180                 rxmode->offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 181         else
 182                 rxmode->offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
 183
 184         if (nicvf_mbox_update_hw_max_frs(nic, mtu))
 185                 return -EINVAL;
 186
 187         nic->mtu = mtu;
 188
 189         for (i = 0; i < nic->sqs_count; i++)
 190                 nic->snicvf[i]->mtu = mtu;
 191
 192         return 0;
 193 }
 194
 195 static int
 196 nicvf_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs)
 197 {
 198         uint64_t *data = regs->data;
 199         struct nicvf *nic = nicvf_pmd_priv(dev);
 200
 201         if (data == NULL) {
 202                 regs->length = nicvf_reg_get_count();
 203                 regs->width = THUNDERX_REG_BYTES;
 204                 return 0;
 205         }
 206
 207         /* Support only full register dump */
 208         if ((regs->length == 0) ||
 209                 (regs->length == (uint32_t)nicvf_reg_get_count())) {
 210                 regs->version = nic->vendor_id << 16 | nic->device_id;
 211                 nicvf_reg_dump(nic, data);
 212                 return 0;
 213         }
 214         return -ENOTSUP;
 215 }
 216
 217 static int
 218 nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 219 {
 220         uint16_t qidx;
 221         struct nicvf_hw_rx_qstats rx_qstats;
 222         struct nicvf_hw_tx_qstats tx_qstats;
 223         struct nicvf_hw_stats port_stats;
 224         struct nicvf *nic = nicvf_pmd_priv(dev);
 225         uint16_t rx_start, rx_end;
 226         uint16_t tx_start, tx_end;
 227         size_t i;
 228
 229         /* RX queue indices for the first VF */
 230         nicvf_rx_range(dev, nic, &rx_start, &rx_end);
 231
 232         /* Reading per RX ring stats */
 233         for (qidx = rx_start; qidx <= rx_end; qidx++) {
 234                 if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
 235                         break;
 236
 237                 nicvf_hw_get_rx_qstats(nic, &rx_qstats, qidx);
 238                 stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes;
 239                 stats->q_ipackets[qidx] = rx_qstats.q_rx_packets;
 240         }
 241
 242         /* TX queue indices for the first VF */
 243         nicvf_tx_range(dev, nic, &tx_start, &tx_end);
 244
 245         /* Reading per TX ring stats */
 246         for (qidx = tx_start; qidx <= tx_end; qidx++) {
 247                 if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
 248                         break;
 249
 250                 nicvf_hw_get_tx_qstats(nic, &tx_qstats, qidx);
 251                 stats->q_obytes[qidx] = tx_qstats.q_tx_bytes;
 252                 stats->q_opackets[qidx] = tx_qstats.q_tx_packets;
 253         }
 254
 255         for (i = 0; i < nic->sqs_count; i++) {
 256                 struct nicvf *snic = nic->snicvf[i];
 257
 258                 if (snic == NULL)
 259                         break;
 260
 261                 /* RX queue indices for a secondary VF */
 262                 nicvf_rx_range(dev, snic, &rx_start, &rx_end);
 263
 264                 /* Reading per RX ring stats */
 265                 for (qidx = rx_start; qidx <= rx_end; qidx++) {
 266                         if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
 267                                 break;
 268
 269                         nicvf_hw_get_rx_qstats(snic, &rx_qstats,
 270                                                qidx % MAX_RCV_QUEUES_PER_QS);
 271                         stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes;
 272                         stats->q_ipackets[qidx] = rx_qstats.q_rx_packets;
 273                 }
 274
 275                 /* TX queue indices for a secondary VF */
 276                 nicvf_tx_range(dev, snic, &tx_start, &tx_end);
 277                 /* Reading per TX ring stats */
 278                 for (qidx = tx_start; qidx <= tx_end; qidx++) {
 279                         if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
 280                                 break;
 281
 282                         nicvf_hw_get_tx_qstats(snic, &tx_qstats,
 283                                                qidx % MAX_SND_QUEUES_PER_QS);
 284                         stats->q_obytes[qidx] = tx_qstats.q_tx_bytes;
 285                         stats->q_opackets[qidx] = tx_qstats.q_tx_packets;
 286                 }
 287         }
 288
 289         nicvf_hw_get_stats(nic, &port_stats);
 290         stats->ibytes = port_stats.rx_bytes;
 291         stats->ipackets = port_stats.rx_ucast_frames;
 292         stats->ipackets += port_stats.rx_bcast_frames;
 293         stats->ipackets += port_stats.rx_mcast_frames;
 294         stats->ierrors = port_stats.rx_l2_errors;
 295         stats->imissed = port_stats.rx_drop_red;
 296         stats->imissed += port_stats.rx_drop_overrun;
 297         stats->imissed += port_stats.rx_drop_bcast;
 298         stats->imissed += port_stats.rx_drop_mcast;
 299         stats->imissed += port_stats.rx_drop_l3_bcast;
 300         stats->imissed += port_stats.rx_drop_l3_mcast;
 301
 302         stats->obytes = port_stats.tx_bytes_ok;
 303         stats->opackets = port_stats.tx_ucast_frames_ok;
 304         stats->opackets += port_stats.tx_bcast_frames_ok;
 305         stats->opackets += port_stats.tx_mcast_frames_ok;
 306         stats->oerrors = port_stats.tx_drops;
 307
 308         return 0;
 309 }
 310
 311 static const uint32_t *
 312 nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 313 {
 314         size_t copied;
 315         static uint32_t ptypes[32];
 316         struct nicvf *nic = nicvf_pmd_priv(dev);
 317         static const uint32_t ptypes_common[] = {
 318                 RTE_PTYPE_L3_IPV4,
 319                 RTE_PTYPE_L3_IPV4_EXT,
 320                 RTE_PTYPE_L3_IPV6,
 321                 RTE_PTYPE_L3_IPV6_EXT,
 322                 RTE_PTYPE_L4_TCP,
 323                 RTE_PTYPE_L4_UDP,
 324                 RTE_PTYPE_L4_FRAG,
 325         };
 326         static const uint32_t ptypes_tunnel[] = {
 327                 RTE_PTYPE_TUNNEL_GRE,
 328                 RTE_PTYPE_TUNNEL_GENEVE,
 329                 RTE_PTYPE_TUNNEL_VXLAN,
 330                 RTE_PTYPE_TUNNEL_NVGRE,
 331         };
 332         static const uint32_t ptypes_end = RTE_PTYPE_UNKNOWN;
 333
 334         copied = sizeof(ptypes_common);
 335         memcpy(ptypes, ptypes_common, copied);
 336         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
 337                 memcpy((char *)ptypes + copied, ptypes_tunnel,
 338                         sizeof(ptypes_tunnel));
 339                 copied += sizeof(ptypes_tunnel);
 340         }
 341
 342         memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end));
 343
 344         /* All Ptypes are supported in all Rx functions. */
 345         return ptypes;
 346 }
 347
 348 static int
 349 nicvf_dev_stats_reset(struct rte_eth_dev *dev)
 350 {
 351         int i;
 352         uint16_t rxqs = 0, txqs = 0;
 353         struct nicvf *nic = nicvf_pmd_priv(dev);
 354         uint16_t rx_start, rx_end;
 355         uint16_t tx_start, tx_end;
 356         int ret;
 357
 358         /* Reset all primary nic counters */
 359         nicvf_rx_range(dev, nic, &rx_start, &rx_end);
 360         for (i = rx_start; i <= rx_end; i++)
 361                 rxqs |= (0x3 << (i * 2));
 362
 363         nicvf_tx_range(dev, nic, &tx_start, &tx_end);
 364         for (i = tx_start; i <= tx_end; i++)
 365                 txqs |= (0x3 << (i * 2));
 366
 367         ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, rxqs, txqs);
 368         if (ret != 0)
 369                 return ret;
 370
 371         /* Reset secondary nic queue counters */
 372         for (i = 0; i < nic->sqs_count; i++) {
 373                 struct nicvf *snic = nic->snicvf[i];
 374                 if (snic == NULL)
 375                         break;
 376
 377                 nicvf_rx_range(dev, snic, &rx_start, &rx_end);
 378                 for (i = rx_start; i <= rx_end; i++)
 379                         rxqs |= (0x3 << ((i % MAX_CMP_QUEUES_PER_QS) * 2));
 380
 381                 nicvf_tx_range(dev, snic, &tx_start, &tx_end);
 382                 for (i = tx_start; i <= tx_end; i++)
 383                         txqs |= (0x3 << ((i % MAX_SND_QUEUES_PER_QS) * 2));
 384
 385                 ret = nicvf_mbox_reset_stat_counters(snic, 0, 0, rxqs, txqs);
 386                 if (ret != 0)
 387                         return ret;
 388         }
 389
 390         return 0;
 391 }
 392
 393 /* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */
 394 static int
 395 nicvf_dev_promisc_enable(struct rte_eth_dev *dev __rte_unused)
 396 {
 397         return 0;
 398 }
 399
 400 static inline uint64_t
 401 nicvf_rss_ethdev_to_nic(struct nicvf *nic, uint64_t ethdev_rss)
 402 {
 403         uint64_t nic_rss = 0;
 404
 405         if (ethdev_rss & ETH_RSS_IPV4)
 406                 nic_rss |= RSS_IP_ENA;
 407
 408         if (ethdev_rss & ETH_RSS_IPV6)
 409                 nic_rss |= RSS_IP_ENA;
 410
 411         if (ethdev_rss & ETH_RSS_NONFRAG_IPV4_UDP)
 412                 nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
 413
 414         if (ethdev_rss & ETH_RSS_NONFRAG_IPV4_TCP)
 415                 nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
 416
 417         if (ethdev_rss & ETH_RSS_NONFRAG_IPV6_UDP)
 418                 nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
 419
 420         if (ethdev_rss & ETH_RSS_NONFRAG_IPV6_TCP)
 421                 nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
 422
 423         if (ethdev_rss & ETH_RSS_PORT)
 424                 nic_rss |= RSS_L2_EXTENDED_HASH_ENA;
 425
 426         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
 427                 if (ethdev_rss & ETH_RSS_VXLAN)
 428                         nic_rss |= RSS_TUN_VXLAN_ENA;
 429
 430                 if (ethdev_rss & ETH_RSS_GENEVE)
 431                         nic_rss |= RSS_TUN_GENEVE_ENA;
 432
 433                 if (ethdev_rss & ETH_RSS_NVGRE)
 434                         nic_rss |= RSS_TUN_NVGRE_ENA;
 435         }
 436
 437         return nic_rss;
 438 }
 439
 440 static inline uint64_t
 441 nicvf_rss_nic_to_ethdev(struct nicvf *nic,  uint64_t nic_rss)
 442 {
 443         uint64_t ethdev_rss = 0;
 444
 445         if (nic_rss & RSS_IP_ENA)
 446                 ethdev_rss |= (ETH_RSS_IPV4 | ETH_RSS_IPV6);
 447
 448         if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_TCP_ENA))
 449                 ethdev_rss |= (ETH_RSS_NONFRAG_IPV4_TCP |
 450                                 ETH_RSS_NONFRAG_IPV6_TCP);
 451
 452         if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_UDP_ENA))
 453                 ethdev_rss |= (ETH_RSS_NONFRAG_IPV4_UDP |
 454                                 ETH_RSS_NONFRAG_IPV6_UDP);
 455
 456         if (nic_rss & RSS_L2_EXTENDED_HASH_ENA)
 457                 ethdev_rss |= ETH_RSS_PORT;
 458
 459         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
 460                 if (nic_rss & RSS_TUN_VXLAN_ENA)
 461                         ethdev_rss |= ETH_RSS_VXLAN;
 462
 463                 if (nic_rss & RSS_TUN_GENEVE_ENA)
 464                         ethdev_rss |= ETH_RSS_GENEVE;
 465
 466                 if (nic_rss & RSS_TUN_NVGRE_ENA)
 467                         ethdev_rss |= ETH_RSS_NVGRE;
 468         }
 469         return ethdev_rss;
 470 }
 471
 472 static int
 473 nicvf_dev_reta_query(struct rte_eth_dev *dev,
 474                      struct rte_eth_rss_reta_entry64 *reta_conf,
 475                      uint16_t reta_size)
 476 {
 477         struct nicvf *nic = nicvf_pmd_priv(dev);
 478         uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
 479         int ret, i, j;
 480
 481         if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
 482                 PMD_DRV_LOG(ERR,
 483                             "The size of hash lookup table configured "
 484                             "(%u) doesn't match the number hardware can supported "
 485                             "(%u)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
 486                 return -EINVAL;
 487         }
 488
 489         ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
 490         if (ret)
 491                 return ret;
 492
 493         /* Copy RETA table */
 494         for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_RETA_GROUP_SIZE); i++) {
 495                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
 496                         if ((reta_conf[i].mask >> j) & 0x01)
 497                                 reta_conf[i].reta[j] = tbl[j];
 498         }
 499
 500         return 0;
 501 }
 502
 503 static int
 504 nicvf_dev_reta_update(struct rte_eth_dev *dev,
 505                       struct rte_eth_rss_reta_entry64 *reta_conf,
 506                       uint16_t reta_size)
 507 {
 508         struct nicvf *nic = nicvf_pmd_priv(dev);
 509         uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
 510         int ret, i, j;
 511
 512         if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
 513                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
 514                         "(%u) doesn't match the number hardware can supported "
 515                         "(%u)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
 516                 return -EINVAL;
 517         }
 518
 519         ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
 520         if (ret)
 521                 return ret;
 522
 523         /* Copy RETA table */
 524         for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_RETA_GROUP_SIZE); i++) {
 525                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
 526                         if ((reta_conf[i].mask >> j) & 0x01)
 527                                 tbl[j] = reta_conf[i].reta[j];
 528         }
 529
 530         return nicvf_rss_reta_update(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
 531 }
 532
 533 static int
 534 nicvf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 535                             struct rte_eth_rss_conf *rss_conf)
 536 {
 537         struct nicvf *nic = nicvf_pmd_priv(dev);
 538
 539         if (rss_conf->rss_key)
 540                 nicvf_rss_get_key(nic, rss_conf->rss_key);
 541
 542         rss_conf->rss_key_len =  RSS_HASH_KEY_BYTE_SIZE;
 543         rss_conf->rss_hf = nicvf_rss_nic_to_ethdev(nic, nicvf_rss_get_cfg(nic));
 544         return 0;
 545 }
 546
 547 static int
 548 nicvf_dev_rss_hash_update(struct rte_eth_dev *dev,
 549                           struct rte_eth_rss_conf *rss_conf)
 550 {
 551         struct nicvf *nic = nicvf_pmd_priv(dev);
 552         uint64_t nic_rss;
 553
 554         if (rss_conf->rss_key &&
 555                 rss_conf->rss_key_len != RSS_HASH_KEY_BYTE_SIZE) {
 556                 PMD_DRV_LOG(ERR, "Hash key size mismatch %u",
 557                             rss_conf->rss_key_len);
 558                 return -EINVAL;
 559         }
 560
 561         if (rss_conf->rss_key)
 562                 nicvf_rss_set_key(nic, rss_conf->rss_key);
 563
 564         nic_rss = nicvf_rss_ethdev_to_nic(nic, rss_conf->rss_hf);
 565         nicvf_rss_set_cfg(nic, nic_rss);
 566         return 0;
 567 }
 568
 569 static int
 570 nicvf_qset_cq_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
 571                     struct nicvf_rxq *rxq, uint16_t qidx, uint32_t desc_cnt)
 572 {
 573         const struct rte_memzone *rz;
 574         uint32_t ring_size = CMP_QUEUE_SZ_MAX * sizeof(union cq_entry_t);
 575
 576         rz = rte_eth_dma_zone_reserve(dev, "cq_ring",
 577                                       nicvf_netdev_qidx(nic, qidx), ring_size,
 578                                       NICVF_CQ_BASE_ALIGN_BYTES, nic->node);
 579         if (rz == NULL) {
 580                 PMD_INIT_LOG(ERR, "Failed to allocate mem for cq hw ring");
 581                 return -ENOMEM;
 582         }
 583
 584         memset(rz->addr, 0, ring_size);
 585
 586         rxq->phys = rz->iova;
 587         rxq->desc = rz->addr;
 588         rxq->qlen_mask = desc_cnt - 1;
 589
 590         return 0;
 591 }
 592
 593 static int
 594 nicvf_qset_sq_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
 595                     struct nicvf_txq *sq, uint16_t qidx, uint32_t desc_cnt)
 596 {
 597         const struct rte_memzone *rz;
 598         uint32_t ring_size = SND_QUEUE_SZ_MAX * sizeof(union sq_entry_t);
 599
 600         rz = rte_eth_dma_zone_reserve(dev, "sq",
 601                                       nicvf_netdev_qidx(nic, qidx), ring_size,
 602                                       NICVF_SQ_BASE_ALIGN_BYTES, nic->node);
 603         if (rz == NULL) {
 604                 PMD_INIT_LOG(ERR, "Failed allocate mem for sq hw ring");
 605                 return -ENOMEM;
 606         }
 607
 608         memset(rz->addr, 0, ring_size);
 609
 610         sq->phys = rz->iova;
 611         sq->desc = rz->addr;
 612         sq->qlen_mask = desc_cnt - 1;
 613
 614         return 0;
 615 }
 616
 617 static int
 618 nicvf_qset_rbdr_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
 619                       uint32_t desc_cnt, uint32_t buffsz)
 620 {
 621         struct nicvf_rbdr *rbdr;
 622         const struct rte_memzone *rz;
 623         uint32_t ring_size;
 624
 625         assert(nic->rbdr == NULL);
 626         rbdr = rte_zmalloc_socket("rbdr", sizeof(struct nicvf_rbdr),
 627                                   RTE_CACHE_LINE_SIZE, nic->node);
 628         if (rbdr == NULL) {
 629                 PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr");
 630                 return -ENOMEM;
 631         }
 632
 633         ring_size = sizeof(struct rbdr_entry_t) * RBDR_QUEUE_SZ_MAX;
 634         rz = rte_eth_dma_zone_reserve(dev, "rbdr",
 635                                       nicvf_netdev_qidx(nic, 0), ring_size,
 636                                       NICVF_RBDR_BASE_ALIGN_BYTES, nic->node);
 637         if (rz == NULL) {
 638                 PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr desc ring");
 639                 rte_free(rbdr);
 640                 return -ENOMEM;
 641         }
 642
 643         memset(rz->addr, 0, ring_size);
 644
 645         rbdr->phys = rz->iova;
 646         rbdr->tail = 0;
 647         rbdr->next_tail = 0;
 648         rbdr->desc = rz->addr;
 649         rbdr->buffsz = buffsz;
 650         rbdr->qlen_mask = desc_cnt - 1;
 651         rbdr->rbdr_status =
 652                 nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_STATUS0;
 653         rbdr->rbdr_door =
 654                 nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_DOOR;
 655
 656         nic->rbdr = rbdr;
 657         return 0;
 658 }
 659
 660 static void
 661 nicvf_rbdr_release_mbuf(struct rte_eth_dev *dev, struct nicvf *nic,
 662                         nicvf_iova_addr_t phy)
 663 {
 664         uint16_t qidx;
 665         void *obj;
 666         struct nicvf_rxq *rxq;
 667         uint16_t rx_start, rx_end;
 668
 669         /* Get queue ranges for this VF */
 670         nicvf_rx_range(dev, nic, &rx_start, &rx_end);
 671
 672         for (qidx = rx_start; qidx <= rx_end; qidx++) {
 673                 rxq = dev->data->rx_queues[qidx];
 674                 if (rxq->precharge_cnt) {
 675                         obj = (void *)nicvf_mbuff_phy2virt(phy,
 676                                                            rxq->mbuf_phys_off);
 677                         rte_mempool_put(rxq->pool, obj);
 678                         rxq->precharge_cnt--;
 679                         break;
 680                 }
 681         }
 682 }
 683
 684 static inline void
 685 nicvf_rbdr_release_mbufs(struct rte_eth_dev *dev, struct nicvf *nic)
 686 {
 687         uint32_t qlen_mask, head;
 688         struct rbdr_entry_t *entry;
 689         struct nicvf_rbdr *rbdr = nic->rbdr;
 690
 691         qlen_mask = rbdr->qlen_mask;
 692         head = rbdr->head;
 693         while (head != rbdr->tail) {
 694                 entry = rbdr->desc + head;
 695                 nicvf_rbdr_release_mbuf(dev, nic, entry->full_addr);
 696                 head++;
 697                 head = head & qlen_mask;
 698         }
 699 }
 700
 701 static inline void
 702 nicvf_tx_queue_release_mbufs(struct nicvf_txq *txq)
 703 {
 704         uint32_t head;
 705
 706         head = txq->head;
 707         while (head != txq->tail) {
 708                 if (txq->txbuffs[head]) {
 709                         rte_pktmbuf_free_seg(txq->txbuffs[head]);
 710                         txq->txbuffs[head] = NULL;
 711                 }
 712                 head++;
 713                 head = head & txq->qlen_mask;
 714         }
 715 }
 716
 717 static void
 718 nicvf_tx_queue_reset(struct nicvf_txq *txq)
 719 {
 720         uint32_t txq_desc_cnt = txq->qlen_mask + 1;
 721
 722         memset(txq->desc, 0, sizeof(union sq_entry_t) * txq_desc_cnt);
 723         memset(txq->txbuffs, 0, sizeof(struct rte_mbuf *) * txq_desc_cnt);
 724         txq->tail = 0;
 725         txq->head = 0;
 726         txq->xmit_bufs = 0;
 727 }
 728
 729 static inline int
 730 nicvf_vf_start_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
 731                         uint16_t qidx)
 732 {
 733         struct nicvf_txq *txq;
 734         int ret;
 735
 736         assert(qidx < MAX_SND_QUEUES_PER_QS);
 737
 738         if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
 739                 RTE_ETH_QUEUE_STATE_STARTED)
 740                 return 0;
 741
 742         txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)];
 743         txq->pool = NULL;
 744         ret = nicvf_qset_sq_config(nic, qidx, txq);
 745         if (ret) {
 746                 PMD_INIT_LOG(ERR, "Failed to configure sq VF%d %d %d",
 747                              nic->vf_id, qidx, ret);
 748                 goto config_sq_error;
 749         }
 750
 751         dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
 752                 RTE_ETH_QUEUE_STATE_STARTED;
 753         return ret;
 754
 755 config_sq_error:
 756         nicvf_qset_sq_reclaim(nic, qidx);
 757         return ret;
 758 }
 759
 760 static inline int
 761 nicvf_vf_stop_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
 762                        uint16_t qidx)
 763 {
 764         struct nicvf_txq *txq;
 765         int ret;
 766
 767         assert(qidx < MAX_SND_QUEUES_PER_QS);
 768
 769         if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
 770                 RTE_ETH_QUEUE_STATE_STOPPED)
 771                 return 0;
 772
 773         ret = nicvf_qset_sq_reclaim(nic, qidx);
 774         if (ret)
 775                 PMD_INIT_LOG(ERR, "Failed to reclaim sq VF%d %d %d",
 776                              nic->vf_id, qidx, ret);
 777
 778         txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)];
 779         nicvf_tx_queue_release_mbufs(txq);
 780         nicvf_tx_queue_reset(txq);
 781
 782         dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
 783                 RTE_ETH_QUEUE_STATE_STOPPED;
 784         return ret;
 785 }
 786
 787 static inline int
 788 nicvf_configure_cpi(struct rte_eth_dev *dev)
 789 {
 790         struct nicvf *nic = nicvf_pmd_priv(dev);
 791         uint16_t qidx, qcnt;
 792         int ret;
 793
 794         /* Count started rx queues */
 795         for (qidx = qcnt = 0; qidx < dev->data->nb_rx_queues; qidx++)
 796                 if (dev->data->rx_queue_state[qidx] ==
 797                     RTE_ETH_QUEUE_STATE_STARTED)
 798                         qcnt++;
 799
 800         nic->cpi_alg = CPI_ALG_NONE;
 801         ret = nicvf_mbox_config_cpi(nic, qcnt);
 802         if (ret)
 803                 PMD_INIT_LOG(ERR, "Failed to configure CPI %d", ret);
 804
 805         return ret;
 806 }
 807
 808 static inline int
 809 nicvf_configure_rss(struct rte_eth_dev *dev)
 810 {
 811         struct nicvf *nic = nicvf_pmd_priv(dev);
 812         uint64_t rsshf;
 813         int ret = -EINVAL;
 814
 815         rsshf = nicvf_rss_ethdev_to_nic(nic,
 816                         dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf);
 817         PMD_DRV_LOG(INFO, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64,
 818                     dev->data->dev_conf.rxmode.mq_mode,
 819                     dev->data->nb_rx_queues,
 820                     dev->data->dev_conf.lpbk_mode, rsshf);
 821
 822         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_NONE)
 823                 ret = nicvf_rss_term(nic);
 824         else if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS)
 825                 ret = nicvf_rss_config(nic, dev->data->nb_rx_queues, rsshf);
 826         if (ret)
 827                 PMD_INIT_LOG(ERR, "Failed to configure RSS %d", ret);
 828
 829         return ret;
 830 }
 831
 832 static int
 833 nicvf_configure_rss_reta(struct rte_eth_dev *dev)
 834 {
 835         struct nicvf *nic = nicvf_pmd_priv(dev);
 836         unsigned int idx, qmap_size;
 837         uint8_t qmap[RTE_MAX_QUEUES_PER_PORT];
 838         uint8_t default_reta[NIC_MAX_RSS_IDR_TBL_SIZE];
 839
 840         if (nic->cpi_alg != CPI_ALG_NONE)
 841                 return -EINVAL;
 842
 843         /* Prepare queue map */
 844         for (idx = 0, qmap_size = 0; idx < dev->data->nb_rx_queues; idx++) {
 845                 if (dev->data->rx_queue_state[idx] ==
 846                                 RTE_ETH_QUEUE_STATE_STARTED)
 847                         qmap[qmap_size++] = idx;
 848         }
 849
 850         /* Update default RSS RETA */
 851         for (idx = 0; idx < NIC_MAX_RSS_IDR_TBL_SIZE; idx++)
 852                 default_reta[idx] = qmap[idx % qmap_size];
 853
 854         return nicvf_rss_reta_update(nic, default_reta,
 855                                      NIC_MAX_RSS_IDR_TBL_SIZE);
 856 }
 857
 858 static void
 859 nicvf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 860 {
 861         struct nicvf_txq *txq = dev->data->tx_queues[qid];
 862
 863         PMD_INIT_FUNC_TRACE();
 864
 865         if (txq) {
 866                 if (txq->txbuffs != NULL) {
 867                         nicvf_tx_queue_release_mbufs(txq);
 868                         rte_free(txq->txbuffs);
 869                         txq->txbuffs = NULL;
 870                 }
 871                 rte_free(txq);
 872                 dev->data->tx_queues[qid] = NULL;
 873         }
 874 }
 875
 876 static void
 877 nicvf_set_tx_function(struct rte_eth_dev *dev)
 878 {
 879         struct nicvf_txq *txq = NULL;
 880         size_t i;
 881         bool multiseg = false;
 882
 883         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 884                 txq = dev->data->tx_queues[i];
 885                 if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
 886                         multiseg = true;
 887                         break;
 888                 }
 889         }
 890
 891         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
 892         if (multiseg) {
 893                 PMD_DRV_LOG(DEBUG, "Using multi-segment tx callback");
 894                 dev->tx_pkt_burst = nicvf_xmit_pkts_multiseg;
 895         } else {
 896                 PMD_DRV_LOG(DEBUG, "Using single-segment tx callback");
 897                 dev->tx_pkt_burst = nicvf_xmit_pkts;
 898         }
 899
 900         if (!txq)
 901                 return;
 902
 903         if (txq->pool_free == nicvf_single_pool_free_xmited_buffers)
 904                 PMD_DRV_LOG(DEBUG, "Using single-mempool tx free method");
 905         else
 906                 PMD_DRV_LOG(DEBUG, "Using multi-mempool tx free method");
 907 }
 908
 909 static void
 910 nicvf_set_rx_function(struct rte_eth_dev *dev)
 911 {
 912         struct nicvf *nic = nicvf_pmd_priv(dev);
 913
 914         const eth_rx_burst_t rx_burst_func[2][2][2] = {
 915         /* [NORMAL/SCATTER] [CKSUM/NO_CKSUM] [VLAN_STRIP/NO_VLAN_STRIP] */
 916                 [0][0][0] = nicvf_recv_pkts_no_offload,
 917                 [0][0][1] = nicvf_recv_pkts_vlan_strip,
 918                 [0][1][0] = nicvf_recv_pkts_cksum,
 919                 [0][1][1] = nicvf_recv_pkts_cksum_vlan_strip,
 920                 [1][0][0] = nicvf_recv_pkts_multiseg_no_offload,
 921                 [1][0][1] = nicvf_recv_pkts_multiseg_vlan_strip,
 922                 [1][1][0] = nicvf_recv_pkts_multiseg_cksum,
 923                 [1][1][1] = nicvf_recv_pkts_multiseg_cksum_vlan_strip,
 924         };
 925
 926         dev->rx_pkt_burst =
 927                 rx_burst_func[dev->data->scattered_rx]
 928                         [nic->offload_cksum][nic->vlan_strip];
 929 }
 930
 931 static int
 932 nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 933                          uint16_t nb_desc, unsigned int socket_id,
 934                          const struct rte_eth_txconf *tx_conf)
 935 {
 936         uint16_t tx_free_thresh;
 937         bool is_single_pool;
 938         struct nicvf_txq *txq;
 939         struct nicvf *nic = nicvf_pmd_priv(dev);
 940         uint64_t offloads;
 941
 942         PMD_INIT_FUNC_TRACE();
 943
 944         if (qidx >= MAX_SND_QUEUES_PER_QS)
 945                 nic = nic->snicvf[qidx / MAX_SND_QUEUES_PER_QS - 1];
 946
 947         qidx = qidx % MAX_SND_QUEUES_PER_QS;
 948
 949         /* Socket id check */
 950         if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
 951                 PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
 952                 socket_id, nic->node);
 953
 954         /* Tx deferred start is not supported */
 955         if (tx_conf->tx_deferred_start) {
 956                 PMD_INIT_LOG(ERR, "Tx deferred start not supported");
 957                 return -EINVAL;
 958         }
 959
 960         /* Roundup nb_desc to available qsize and validate max number of desc */
 961         nb_desc = nicvf_qsize_sq_roundup(nb_desc);
 962         if (nb_desc == 0) {
 963                 PMD_INIT_LOG(ERR, "Value of nb_desc beyond available sq qsize");
 964                 return -EINVAL;
 965         }
 966
 967         /* Validate tx_free_thresh */
 968         tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
 969                                 tx_conf->tx_free_thresh :
 970                                 NICVF_DEFAULT_TX_FREE_THRESH);
 971
 972         if (tx_free_thresh > (nb_desc) ||
 973                 tx_free_thresh > NICVF_MAX_TX_FREE_THRESH) {
 974                 PMD_INIT_LOG(ERR,
 975                         "tx_free_thresh must be less than the number of TX "
 976                         "descriptors. (tx_free_thresh=%u port=%d "
 977                         "queue=%d)", (unsigned int)tx_free_thresh,
 978                         (int)dev->data->port_id, (int)qidx);
 979                 return -EINVAL;
 980         }
 981
 982         /* Free memory prior to re-allocation if needed. */
 983         if (dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) {
 984                 PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
 985                                 nicvf_netdev_qidx(nic, qidx));
 986                 nicvf_dev_tx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
 987                 dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL;
 988         }
 989
 990         /* Allocating tx queue data structure */
 991         txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq),
 992                                         RTE_CACHE_LINE_SIZE, nic->node);
 993         if (txq == NULL) {
 994                 PMD_INIT_LOG(ERR, "Failed to allocate txq=%d",
 995                              nicvf_netdev_qidx(nic, qidx));
 996                 return -ENOMEM;
 997         }
 998
 999         txq->nic = nic;
1000         txq->queue_id = qidx;
1001         txq->tx_free_thresh = tx_free_thresh;
1002         txq->sq_head = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_HEAD;
1003         txq->sq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_DOOR;
1004         offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
1005         txq->offloads = offloads;
1006
1007         is_single_pool = !!(offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE);
1008
1009         /* Choose optimum free threshold value for multipool case */
1010         if (!is_single_pool) {
1011                 txq->tx_free_thresh = (uint16_t)
1012                 (tx_conf->tx_free_thresh == NICVF_DEFAULT_TX_FREE_THRESH ?
1013                                 NICVF_TX_FREE_MPOOL_THRESH :
1014                                 tx_conf->tx_free_thresh);
1015                 txq->pool_free = nicvf_multi_pool_free_xmited_buffers;
1016         } else {
1017                 txq->pool_free = nicvf_single_pool_free_xmited_buffers;
1018         }
1019
1020         dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = txq;
1021
1022         /* Allocate software ring */
1023         txq->txbuffs = rte_zmalloc_socket("txq->txbuffs",
1024                                 nb_desc * sizeof(struct rte_mbuf *),
1025                                 RTE_CACHE_LINE_SIZE, nic->node);
1026
1027         if (txq->txbuffs == NULL) {
1028                 nicvf_dev_tx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1029                 return -ENOMEM;
1030         }
1031
1032         if (nicvf_qset_sq_alloc(dev, nic, txq, qidx, nb_desc)) {
1033                 PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx);
1034                 nicvf_dev_tx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1035                 return -ENOMEM;
1036         }
1037
1038         nicvf_tx_queue_reset(txq);
1039
1040         PMD_INIT_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p"
1041                         " phys=0x%" PRIx64 " offloads=0x%" PRIx64,
1042                         nicvf_netdev_qidx(nic, qidx), txq, nb_desc, txq->desc,
1043                         txq->phys, txq->offloads);
1044
1045         dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1046                 RTE_ETH_QUEUE_STATE_STOPPED;
1047         return 0;
1048 }
1049
1050 static inline void
1051 nicvf_rx_queue_release_mbufs(struct rte_eth_dev *dev, struct nicvf_rxq *rxq)
1052 {
1053         uint32_t rxq_cnt;
1054         uint32_t nb_pkts, released_pkts = 0;
1055         uint32_t refill_cnt = 0;
1056         struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH];
1057
1058         if (dev->rx_pkt_burst == NULL)
1059                 return;
1060
1061         while ((rxq_cnt = nicvf_dev_rx_queue_count(rxq))) {
1062                 nb_pkts = dev->rx_pkt_burst(rxq, rx_pkts,
1063                                         NICVF_MAX_RX_FREE_THRESH);
1064                 PMD_DRV_LOG(INFO, "nb_pkts=%d  rxq_cnt=%d", nb_pkts, rxq_cnt);
1065                 while (nb_pkts) {
1066                         rte_pktmbuf_free_seg(rx_pkts[--nb_pkts]);
1067                         released_pkts++;
1068                 }
1069         }
1070
1071
1072         refill_cnt += nicvf_dev_rbdr_refill(dev,
1073                         nicvf_netdev_qidx(rxq->nic, rxq->queue_id));
1074
1075         PMD_DRV_LOG(INFO, "free_cnt=%d  refill_cnt=%d",
1076                     released_pkts, refill_cnt);
1077 }
1078
1079 static void
1080 nicvf_rx_queue_reset(struct nicvf_rxq *rxq)
1081 {
1082         rxq->head = 0;
1083         rxq->available_space = 0;
1084         rxq->recv_buffers = 0;
1085 }
1086
1087 static inline int
1088 nicvf_vf_start_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
1089                         uint16_t qidx)
1090 {
1091         struct nicvf_rxq *rxq;
1092         int ret;
1093
1094         assert(qidx < MAX_RCV_QUEUES_PER_QS);
1095
1096         if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
1097                 RTE_ETH_QUEUE_STATE_STARTED)
1098                 return 0;
1099
1100         /* Update rbdr pointer to all rxq */
1101         rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)];
1102         rxq->shared_rbdr = nic->rbdr;
1103
1104         ret = nicvf_qset_rq_config(nic, qidx, rxq);
1105         if (ret) {
1106                 PMD_INIT_LOG(ERR, "Failed to configure rq VF%d %d %d",
1107                              nic->vf_id, qidx, ret);
1108                 goto config_rq_error;
1109         }
1110         ret = nicvf_qset_cq_config(nic, qidx, rxq);
1111         if (ret) {
1112                 PMD_INIT_LOG(ERR, "Failed to configure cq VF%d %d %d",
1113                              nic->vf_id, qidx, ret);
1114                 goto config_cq_error;
1115         }
1116
1117         dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1118                 RTE_ETH_QUEUE_STATE_STARTED;
1119         return 0;
1120
1121 config_cq_error:
1122         nicvf_qset_cq_reclaim(nic, qidx);
1123 config_rq_error:
1124         nicvf_qset_rq_reclaim(nic, qidx);
1125         return ret;
1126 }
1127
1128 static inline int
1129 nicvf_vf_stop_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
1130                        uint16_t qidx)
1131 {
1132         struct nicvf_rxq *rxq;
1133         int ret, other_error;
1134
1135         if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
1136                 RTE_ETH_QUEUE_STATE_STOPPED)
1137                 return 0;
1138
1139         ret = nicvf_qset_rq_reclaim(nic, qidx);
1140         if (ret)
1141                 PMD_INIT_LOG(ERR, "Failed to reclaim rq VF%d %d %d",
1142                              nic->vf_id, qidx, ret);
1143
1144         other_error = ret;
1145         rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)];
1146         nicvf_rx_queue_release_mbufs(dev, rxq);
1147         nicvf_rx_queue_reset(rxq);
1148
1149         ret = nicvf_qset_cq_reclaim(nic, qidx);
1150         if (ret)
1151                 PMD_INIT_LOG(ERR, "Failed to reclaim cq VF%d %d %d",
1152                              nic->vf_id, qidx, ret);
1153
1154         other_error |= ret;
1155         dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1156                 RTE_ETH_QUEUE_STATE_STOPPED;
1157         return other_error;
1158 }
1159
1160 static void
1161 nicvf_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1162 {
1163         PMD_INIT_FUNC_TRACE();
1164
1165         rte_free(dev->data->rx_queues[qid]);
1166 }
1167
1168 static int
1169 nicvf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1170 {
1171         struct nicvf *nic = nicvf_pmd_priv(dev);
1172         int ret;
1173
1174         if (qidx >= MAX_RCV_QUEUES_PER_QS)
1175                 nic = nic->snicvf[(qidx / MAX_RCV_QUEUES_PER_QS - 1)];
1176
1177         qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1178
1179         ret = nicvf_vf_start_rx_queue(dev, nic, qidx);
1180         if (ret)
1181                 return ret;
1182
1183         ret = nicvf_configure_cpi(dev);
1184         if (ret)
1185                 return ret;
1186
1187         return nicvf_configure_rss_reta(dev);
1188 }
1189
1190 static int
1191 nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1192 {
1193         int ret;
1194         struct nicvf *nic = nicvf_pmd_priv(dev);
1195
1196         if (qidx >= MAX_SND_QUEUES_PER_QS)
1197                 nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1198
1199         qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1200
1201         ret = nicvf_vf_stop_rx_queue(dev, nic, qidx);
1202         ret |= nicvf_configure_cpi(dev);
1203         ret |= nicvf_configure_rss_reta(dev);
1204         return ret;
1205 }
1206
1207 static int
1208 nicvf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1209 {
1210         struct nicvf *nic = nicvf_pmd_priv(dev);
1211
1212         if (qidx >= MAX_SND_QUEUES_PER_QS)
1213                 nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1214
1215         qidx = qidx % MAX_SND_QUEUES_PER_QS;
1216
1217         return nicvf_vf_start_tx_queue(dev, nic, qidx);
1218 }
1219
1220 static int
1221 nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1222 {
1223         struct nicvf *nic = nicvf_pmd_priv(dev);
1224
1225         if (qidx >= MAX_SND_QUEUES_PER_QS)
1226                 nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1227
1228         qidx = qidx % MAX_SND_QUEUES_PER_QS;
1229
1230         return nicvf_vf_stop_tx_queue(dev, nic, qidx);
1231 }
1232
1233 static inline void
1234 nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq)
1235 {
1236         uintptr_t p;
1237         struct rte_mbuf mb_def;
1238         struct nicvf *nic = rxq->nic;
1239
1240         RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8);
1241         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0);
1242         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) -
1243                                 offsetof(struct rte_mbuf, data_off) != 2);
1244         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) -
1245                                 offsetof(struct rte_mbuf, data_off) != 4);
1246         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) -
1247                                 offsetof(struct rte_mbuf, data_off) != 6);
1248         RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq, rxq_fastpath_data_end) -
1249                                 offsetof(struct nicvf_rxq,
1250                                         rxq_fastpath_data_start) > 128);
1251         mb_def.nb_segs = 1;
1252         mb_def.data_off = RTE_PKTMBUF_HEADROOM + (nic->skip_bytes);
1253         mb_def.port = rxq->port_id;
1254         rte_mbuf_refcnt_set(&mb_def, 1);
1255
1256         /* Prevent compiler reordering: rearm_data covers previous fields */
1257         rte_compiler_barrier();
1258         p = (uintptr_t)&mb_def.rearm_data;
1259         rxq->mbuf_initializer.value = *(uint64_t *)p;
1260 }
1261
1262 static int
1263 nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
1264                          uint16_t nb_desc, unsigned int socket_id,
1265                          const struct rte_eth_rxconf *rx_conf,
1266                          struct rte_mempool *mp)
1267 {
1268         uint16_t rx_free_thresh;
1269         struct nicvf_rxq *rxq;
1270         struct nicvf *nic = nicvf_pmd_priv(dev);
1271         uint64_t offloads;
1272         uint32_t buffsz;
1273         struct rte_pktmbuf_pool_private *mbp_priv;
1274
1275         PMD_INIT_FUNC_TRACE();
1276
1277         /* First skip check */
1278         mbp_priv = rte_mempool_get_priv(mp);
1279         buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
1280         if (buffsz < (uint32_t)(nic->skip_bytes)) {
1281                 PMD_INIT_LOG(ERR, "First skip is more than configured buffer size");
1282                 return -EINVAL;
1283         }
1284
1285         if (qidx >= MAX_RCV_QUEUES_PER_QS)
1286                 nic = nic->snicvf[qidx / MAX_RCV_QUEUES_PER_QS - 1];
1287
1288         qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1289
1290         /* Socket id check */
1291         if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
1292                 PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
1293                 socket_id, nic->node);
1294
1295         /* Mempool memory must be contiguous, so must be one memory segment*/
1296         if (mp->nb_mem_chunks != 1) {
1297                 PMD_INIT_LOG(ERR, "Non-contiguous mempool, add more huge pages");
1298                 return -EINVAL;
1299         }
1300
1301         /* Mempool memory must be physically contiguous */
1302         if (mp->flags & RTE_MEMPOOL_F_NO_IOVA_CONTIG) {
1303                 PMD_INIT_LOG(ERR, "Mempool memory must be physically contiguous");
1304                 return -EINVAL;
1305         }
1306
1307         /* Rx deferred start is not supported */
1308         if (rx_conf->rx_deferred_start) {
1309                 PMD_INIT_LOG(ERR, "Rx deferred start not supported");
1310                 return -EINVAL;
1311         }
1312
1313         /* Roundup nb_desc to available qsize and validate max number of desc */
1314         nb_desc = nicvf_qsize_cq_roundup(nb_desc);
1315         if (nb_desc == 0) {
1316                 PMD_INIT_LOG(ERR, "Value nb_desc beyond available hw cq qsize");
1317                 return -EINVAL;
1318         }
1319
1320
1321         /* Check rx_free_thresh upper bound */
1322         rx_free_thresh = (uint16_t)((rx_conf->rx_free_thresh) ?
1323                                 rx_conf->rx_free_thresh :
1324                                 NICVF_DEFAULT_RX_FREE_THRESH);
1325         if (rx_free_thresh > NICVF_MAX_RX_FREE_THRESH ||
1326                 rx_free_thresh >= nb_desc * .75) {
1327                 PMD_INIT_LOG(ERR, "rx_free_thresh greater than expected %d",
1328                                 rx_free_thresh);
1329                 return -EINVAL;
1330         }
1331
1332         /* Free memory prior to re-allocation if needed */
1333         if (dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) {
1334                 PMD_RX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
1335                                 nicvf_netdev_qidx(nic, qidx));
1336                 nicvf_dev_rx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1337                 dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL;
1338         }
1339
1340         /* Allocate rxq memory */
1341         rxq = rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq),
1342                                         RTE_CACHE_LINE_SIZE, nic->node);
1343         if (rxq == NULL) {
1344                 PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d",
1345                              nicvf_netdev_qidx(nic, qidx));
1346                 return -ENOMEM;
1347         }
1348
1349         rxq->nic = nic;
1350         rxq->pool = mp;
1351         rxq->queue_id = qidx;
1352         rxq->port_id = dev->data->port_id;
1353         rxq->rx_free_thresh = rx_free_thresh;
1354         rxq->rx_drop_en = rx_conf->rx_drop_en;
1355         rxq->cq_status = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_STATUS;
1356         rxq->cq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_DOOR;
1357         rxq->precharge_cnt = 0;
1358
1359         if (nicvf_hw_cap(nic) & NICVF_CAP_CQE_RX2)
1360                 rxq->rbptr_offset = NICVF_CQE_RX2_RBPTR_WORD;
1361         else
1362                 rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD;
1363
1364         dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = rxq;
1365
1366         nicvf_rxq_mbuf_setup(rxq);
1367
1368         /* Alloc completion queue */
1369         if (nicvf_qset_cq_alloc(dev, nic, rxq, rxq->queue_id, nb_desc)) {
1370                 PMD_INIT_LOG(ERR, "failed to allocate cq %u", rxq->queue_id);
1371                 nicvf_dev_rx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1372                 return -ENOMEM;
1373         }
1374
1375         nicvf_rx_queue_reset(rxq);
1376
1377         offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
1378         PMD_INIT_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d)"
1379                         " phy=0x%" PRIx64 " offloads=0x%" PRIx64,
1380                         nicvf_netdev_qidx(nic, qidx), rxq, mp->name, nb_desc,
1381                         rte_mempool_avail_count(mp), rxq->phys, offloads);
1382
1383         dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1384                 RTE_ETH_QUEUE_STATE_STOPPED;
1385         return 0;
1386 }
1387
1388 static int
1389 nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1390 {
1391         struct nicvf *nic = nicvf_pmd_priv(dev);
1392         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1393
1394         PMD_INIT_FUNC_TRACE();
1395
1396         /* Autonegotiation may be disabled */
1397         dev_info->speed_capa = ETH_LINK_SPEED_FIXED;
1398         dev_info->speed_capa |= ETH_LINK_SPEED_10M | ETH_LINK_SPEED_100M |
1399                                  ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
1400         if (nicvf_hw_version(nic) != PCI_SUB_DEVICE_ID_CN81XX_NICVF)
1401                 dev_info->speed_capa |= ETH_LINK_SPEED_40G;
1402
1403         dev_info->min_rx_bufsize = RTE_ETHER_MIN_MTU;
1404         dev_info->max_rx_pktlen = NIC_HW_MAX_MTU + RTE_ETHER_HDR_LEN;
1405         dev_info->max_rx_queues =
1406                         (uint16_t)MAX_RCV_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
1407         dev_info->max_tx_queues =
1408                         (uint16_t)MAX_SND_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
1409         dev_info->max_mac_addrs = 1;
1410         dev_info->max_vfs = pci_dev->max_vfs;
1411
1412         dev_info->rx_offload_capa = NICVF_RX_OFFLOAD_CAPA;
1413         dev_info->tx_offload_capa = NICVF_TX_OFFLOAD_CAPA;
1414         dev_info->rx_queue_offload_capa = NICVF_RX_OFFLOAD_CAPA;
1415         dev_info->tx_queue_offload_capa = NICVF_TX_OFFLOAD_CAPA;
1416
1417         dev_info->reta_size = nic->rss_info.rss_size;
1418         dev_info->hash_key_size = RSS_HASH_KEY_BYTE_SIZE;
1419         dev_info->flow_type_rss_offloads = NICVF_RSS_OFFLOAD_PASS1;
1420         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING)
1421                 dev_info->flow_type_rss_offloads |= NICVF_RSS_OFFLOAD_TUNNEL;
1422
1423         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1424                 .rx_free_thresh = NICVF_DEFAULT_RX_FREE_THRESH,
1425                 .rx_drop_en = 0,
1426         };
1427
1428         dev_info->default_txconf = (struct rte_eth_txconf) {
1429                 .tx_free_thresh = NICVF_DEFAULT_TX_FREE_THRESH,
1430                 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE |
1431                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM   |
1432                         DEV_TX_OFFLOAD_UDP_CKSUM          |
1433                         DEV_TX_OFFLOAD_TCP_CKSUM,
1434         };
1435
1436         return 0;
1437 }
1438
1439 static nicvf_iova_addr_t
1440 rbdr_rte_mempool_get(void *dev, void *opaque)
1441 {
1442         uint16_t qidx;
1443         uintptr_t mbuf;
1444         struct nicvf_rxq *rxq;
1445         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)dev;
1446         struct nicvf *nic = (struct nicvf *)opaque;
1447         uint16_t rx_start, rx_end;
1448
1449         /* Get queue ranges for this VF */
1450         nicvf_rx_range(eth_dev, nic, &rx_start, &rx_end);
1451
1452         for (qidx = rx_start; qidx <= rx_end; qidx++) {
1453                 rxq = eth_dev->data->rx_queues[qidx];
1454                 /* Maintain equal buffer count across all pools */
1455                 if (rxq->precharge_cnt >= rxq->qlen_mask)
1456                         continue;
1457                 rxq->precharge_cnt++;
1458                 mbuf = (uintptr_t)rte_pktmbuf_alloc(rxq->pool);
1459                 if (mbuf)
1460                         return nicvf_mbuff_virt2phy(mbuf, rxq->mbuf_phys_off);
1461         }
1462         return 0;
1463 }
1464
1465 static int
1466 nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
1467 {
1468         int ret;
1469         uint16_t qidx, data_off;
1470         uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs;
1471         uint64_t mbuf_phys_off = 0;
1472         struct nicvf_rxq *rxq;
1473         struct rte_mbuf *mbuf;
1474         uint16_t rx_start, rx_end;
1475         uint16_t tx_start, tx_end;
1476         int mask;
1477
1478         PMD_INIT_FUNC_TRACE();
1479
1480         /* Userspace process exited without proper shutdown in last run */
1481         if (nicvf_qset_rbdr_active(nic, 0))
1482                 nicvf_vf_stop(dev, nic, false);
1483
1484         /* Get queue ranges for this VF */
1485         nicvf_rx_range(dev, nic, &rx_start, &rx_end);
1486
1487         /*
1488          * Thunderx nicvf PMD can support more than one pool per port only when
1489          * 1) Data payload size is same across all the pools in given port
1490          * AND
1491          * 2) All mbuffs in the pools are from the same hugepage
1492          * AND
1493          * 3) Mbuff metadata size is same across all the pools in given port
1494          *
1495          * This is to support existing application that uses multiple pool/port.
1496          * But, the purpose of using multipool for QoS will not be addressed.
1497          *
1498          */
1499
1500         /* Validate mempool attributes */
1501         for (qidx = rx_start; qidx <= rx_end; qidx++) {
1502                 rxq = dev->data->rx_queues[qidx];
1503                 rxq->mbuf_phys_off = nicvf_mempool_phy_offset(rxq->pool);
1504                 mbuf = rte_pktmbuf_alloc(rxq->pool);
1505                 if (mbuf == NULL) {
1506                         PMD_INIT_LOG(ERR, "Failed allocate mbuf VF%d qid=%d "
1507                                      "pool=%s",
1508                                      nic->vf_id, qidx, rxq->pool->name);
1509                         return -ENOMEM;
1510                 }
1511                 data_off = nicvf_mbuff_meta_length(mbuf);
1512                 data_off += RTE_PKTMBUF_HEADROOM;
1513                 rte_pktmbuf_free(mbuf);
1514
1515                 if (data_off % RTE_CACHE_LINE_SIZE) {
1516                         PMD_INIT_LOG(ERR, "%s: unaligned data_off=%d delta=%d",
1517                                 rxq->pool->name, data_off,
1518                                 data_off % RTE_CACHE_LINE_SIZE);
1519                         return -EINVAL;
1520                 }
1521                 rxq->mbuf_phys_off -= data_off;
1522                 rxq->mbuf_phys_off -= nic->skip_bytes;
1523
1524                 if (mbuf_phys_off == 0)
1525                         mbuf_phys_off = rxq->mbuf_phys_off;
1526                 if (mbuf_phys_off != rxq->mbuf_phys_off) {
1527                         PMD_INIT_LOG(ERR, "pool params not same,%s VF%d %"
1528                                      PRIx64, rxq->pool->name, nic->vf_id,
1529                                      mbuf_phys_off);
1530                         return -EINVAL;
1531                 }
1532         }
1533
1534         /* Check the level of buffers in the pool */
1535         total_rxq_desc = 0;
1536         for (qidx = rx_start; qidx <= rx_end; qidx++) {
1537                 rxq = dev->data->rx_queues[qidx];
1538                 /* Count total numbers of rxq descs */
1539                 total_rxq_desc += rxq->qlen_mask + 1;
1540                 exp_buffs = RTE_MEMPOOL_CACHE_MAX_SIZE + rxq->rx_free_thresh;
1541                 exp_buffs *= dev->data->nb_rx_queues;
1542                 if (rte_mempool_avail_count(rxq->pool) < exp_buffs) {
1543                         PMD_INIT_LOG(ERR, "Buff shortage in pool=%s (%d/%d)",
1544                                      rxq->pool->name,
1545                                      rte_mempool_avail_count(rxq->pool),
1546                                      exp_buffs);
1547                         return -ENOENT;
1548                 }
1549         }
1550
1551         /* Check RBDR desc overflow */
1552         ret = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1553         if (ret == 0) {
1554                 PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc "
1555                              "VF%d", nic->vf_id);
1556                 return -ENOMEM;
1557         }
1558
1559         /* Enable qset */
1560         ret = nicvf_qset_config(nic);
1561         if (ret) {
1562                 PMD_INIT_LOG(ERR, "Failed to enable qset %d VF%d", ret,
1563                              nic->vf_id);
1564                 return ret;
1565         }
1566
1567         /* Allocate RBDR and RBDR ring desc */
1568         nb_rbdr_desc = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1569         ret = nicvf_qset_rbdr_alloc(dev, nic, nb_rbdr_desc, rbdrsz);
1570         if (ret) {
1571                 PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc "
1572                              "VF%d", nic->vf_id);
1573                 goto qset_reclaim;
1574         }
1575
1576         /* Enable and configure RBDR registers */
1577         ret = nicvf_qset_rbdr_config(nic, 0);
1578         if (ret) {
1579                 PMD_INIT_LOG(ERR, "Failed to configure rbdr %d VF%d", ret,
1580                              nic->vf_id);
1581                 goto qset_rbdr_free;
1582         }
1583
1584         /* Fill rte_mempool buffers in RBDR pool and precharge it */
1585         ret = nicvf_qset_rbdr_precharge(dev, nic, 0, rbdr_rte_mempool_get,
1586                                         total_rxq_desc);
1587         if (ret) {
1588                 PMD_INIT_LOG(ERR, "Failed to fill rbdr %d VF%d", ret,
1589                              nic->vf_id);
1590                 goto qset_rbdr_reclaim;
1591         }
1592
1593         PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR VF%d",
1594                      nic->rbdr->tail, nb_rbdr_desc, nic->vf_id);
1595
1596         /* Configure VLAN Strip */
1597         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
1598                 ETH_VLAN_EXTEND_MASK;
1599         ret = nicvf_vlan_offload_config(dev, mask);
1600
1601         /* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data
1602          * to the 64bit memory address.
1603          * The alignment creates a hole in mbuf(between the end of headroom and
1604          * packet data start). The new revision of the HW provides an option to
1605          * disable the L3 alignment feature and make mbuf layout looks
1606          * more like other NICs. For better application compatibility, disabling
1607          * l3 alignment feature on the hardware revisions it supports
1608          */
1609         nicvf_apad_config(nic, false);
1610
1611         /* Get queue ranges for this VF */
1612         nicvf_tx_range(dev, nic, &tx_start, &tx_end);
1613
1614         /* Configure TX queues */
1615         for (qidx = tx_start; qidx <= tx_end; qidx++) {
1616                 ret = nicvf_vf_start_tx_queue(dev, nic,
1617                         qidx % MAX_SND_QUEUES_PER_QS);
1618                 if (ret)
1619                         goto start_txq_error;
1620         }
1621
1622         /* Configure RX queues */
1623         for (qidx = rx_start; qidx <= rx_end; qidx++) {
1624                 ret = nicvf_vf_start_rx_queue(dev, nic,
1625                         qidx % MAX_RCV_QUEUES_PER_QS);
1626                 if (ret)
1627                         goto start_rxq_error;
1628         }
1629
1630         if (!nic->sqs_mode) {
1631                 /* Configure CPI algorithm */
1632                 ret = nicvf_configure_cpi(dev);
1633                 if (ret)
1634                         goto start_txq_error;
1635
1636                 ret = nicvf_mbox_get_rss_size(nic);
1637                 if (ret) {
1638                         PMD_INIT_LOG(ERR, "Failed to get rss table size");
1639                         goto qset_rss_error;
1640                 }
1641
1642                 /* Configure RSS */
1643                 ret = nicvf_configure_rss(dev);
1644                 if (ret)
1645                         goto qset_rss_error;
1646         }
1647
1648         /* Done; Let PF make the BGX's RX and TX switches to ON position */
1649         nicvf_mbox_cfg_done(nic);
1650         return 0;
1651
1652 qset_rss_error:
1653         nicvf_rss_term(nic);
1654 start_rxq_error:
1655         for (qidx = rx_start; qidx <= rx_end; qidx++)
1656                 nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS);
1657 start_txq_error:
1658         for (qidx = tx_start; qidx <= tx_end; qidx++)
1659                 nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS);
1660 qset_rbdr_reclaim:
1661         nicvf_qset_rbdr_reclaim(nic, 0);
1662         nicvf_rbdr_release_mbufs(dev, nic);
1663 qset_rbdr_free:
1664         if (nic->rbdr) {
1665                 rte_free(nic->rbdr);
1666                 nic->rbdr = NULL;
1667         }
1668 qset_reclaim:
1669         nicvf_qset_reclaim(nic);
1670         return ret;
1671 }
1672
1673 static int
1674 nicvf_dev_start(struct rte_eth_dev *dev)
1675 {
1676         uint16_t qidx;
1677         int ret;
1678         size_t i;
1679         struct nicvf *nic = nicvf_pmd_priv(dev);
1680         struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
1681         uint16_t mtu;
1682         uint32_t buffsz = 0, rbdrsz = 0;
1683         struct rte_pktmbuf_pool_private *mbp_priv;
1684         struct nicvf_rxq *rxq;
1685
1686         PMD_INIT_FUNC_TRACE();
1687
1688         /* This function must be called for a primary device */
1689         assert_primary(nic);
1690
1691         /* Validate RBDR buff size */
1692         for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) {
1693                 rxq = dev->data->rx_queues[qidx];
1694                 mbp_priv = rte_mempool_get_priv(rxq->pool);
1695                 buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
1696                 if (buffsz % 128) {
1697                         PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128");
1698                         return -EINVAL;
1699                 }
1700                 if (rbdrsz == 0)
1701                         rbdrsz = buffsz;
1702                 if (rbdrsz != buffsz) {
1703                         PMD_INIT_LOG(ERR, "buffsz not same, qidx=%d (%d/%d)",
1704                                      qidx, rbdrsz, buffsz);
1705                         return -EINVAL;
1706                 }
1707         }
1708
1709         /* Configure loopback */
1710         ret = nicvf_loopback_config(nic, dev->data->dev_conf.lpbk_mode);
1711         if (ret) {
1712                 PMD_INIT_LOG(ERR, "Failed to configure loopback %d", ret);
1713                 return ret;
1714         }
1715
1716         /* Reset all statistics counters attached to this port */
1717         ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF);
1718         if (ret) {
1719                 PMD_INIT_LOG(ERR, "Failed to reset stat counters %d", ret);
1720                 return ret;
1721         }
1722
1723         /* Setup scatter mode if needed by jumbo */
1724         if (dev->data->mtu + (uint32_t)NIC_HW_L2_OVERHEAD + 2 * VLAN_TAG_SIZE > buffsz)
1725                 dev->data->scattered_rx = 1;
1726         if ((rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER) != 0)
1727                 dev->data->scattered_rx = 1;
1728
1729         /* Setup MTU */
1730         mtu = dev->data->mtu;
1731
1732         if (nicvf_dev_set_mtu(dev, mtu)) {
1733                 PMD_INIT_LOG(ERR, "Failed to set default mtu size");
1734                 return -EBUSY;
1735         }
1736
1737         ret = nicvf_vf_start(dev, nic, rbdrsz);
1738         if (ret != 0)
1739                 return ret;
1740
1741         for (i = 0; i < nic->sqs_count; i++) {
1742                 assert(nic->snicvf[i]);
1743
1744                 ret = nicvf_vf_start(dev, nic->snicvf[i], rbdrsz);
1745                 if (ret != 0)
1746                         return ret;
1747         }
1748
1749         /* Configure callbacks based on offloads */
1750         nicvf_set_tx_function(dev);
1751         nicvf_set_rx_function(dev);
1752
1753         return 0;
1754 }
1755
1756 static void
1757 nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup)
1758 {
1759         size_t i;
1760         int ret;
1761         struct nicvf *nic = nicvf_pmd_priv(dev);
1762
1763         PMD_INIT_FUNC_TRACE();
1764         dev->data->dev_started = 0;
1765
1766         /* Teardown secondary vf first */
1767         for (i = 0; i < nic->sqs_count; i++) {
1768                 if (!nic->snicvf[i])
1769                         continue;
1770
1771                 nicvf_vf_stop(dev, nic->snicvf[i], cleanup);
1772         }
1773
1774         /* Stop the primary VF now */
1775         nicvf_vf_stop(dev, nic, cleanup);
1776
1777         /* Disable loopback */
1778         ret = nicvf_loopback_config(nic, 0);
1779         if (ret)
1780                 PMD_INIT_LOG(ERR, "Failed to disable loopback %d", ret);
1781
1782         /* Reclaim CPI configuration */
1783         ret = nicvf_mbox_config_cpi(nic, 0);
1784         if (ret)
1785                 PMD_INIT_LOG(ERR, "Failed to reclaim CPI config %d", ret);
1786 }
1787
1788 static int
1789 nicvf_dev_stop(struct rte_eth_dev *dev)
1790 {
1791         PMD_INIT_FUNC_TRACE();
1792
1793         nicvf_dev_stop_cleanup(dev, false);
1794
1795         return 0;
1796 }
1797
1798 static void
1799 nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic, bool cleanup)
1800 {
1801         int ret;
1802         uint16_t qidx;
1803         uint16_t tx_start, tx_end;
1804         uint16_t rx_start, rx_end;
1805
1806         PMD_INIT_FUNC_TRACE();
1807
1808         if (cleanup) {
1809                 /* Let PF make the BGX's RX and TX switches to OFF position */
1810                 nicvf_mbox_shutdown(nic);
1811         }
1812
1813         /* Disable VLAN Strip */
1814         nicvf_vlan_hw_strip(nic, 0);
1815
1816         /* Get queue ranges for this VF */
1817         nicvf_tx_range(dev, nic, &tx_start, &tx_end);
1818
1819         for (qidx = tx_start; qidx <= tx_end; qidx++)
1820                 nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS);
1821
1822         /* Get queue ranges for this VF */
1823         nicvf_rx_range(dev, nic, &rx_start, &rx_end);
1824
1825         /* Reclaim rq */
1826         for (qidx = rx_start; qidx <= rx_end; qidx++)
1827                 nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS);
1828
1829         /* Reclaim RBDR */
1830         ret = nicvf_qset_rbdr_reclaim(nic, 0);
1831         if (ret)
1832                 PMD_INIT_LOG(ERR, "Failed to reclaim RBDR %d", ret);
1833
1834         /* Move all charged buffers in RBDR back to pool */
1835         if (nic->rbdr != NULL)
1836                 nicvf_rbdr_release_mbufs(dev, nic);
1837
1838         /* Disable qset */
1839         ret = nicvf_qset_reclaim(nic);
1840         if (ret)
1841                 PMD_INIT_LOG(ERR, "Failed to disable qset %d", ret);
1842
1843         /* Disable all interrupts */
1844         nicvf_disable_all_interrupts(nic);
1845
1846         /* Free RBDR SW structure */
1847         if (nic->rbdr) {
1848                 rte_free(nic->rbdr);
1849                 nic->rbdr = NULL;
1850         }
1851 }
1852
1853 static int
1854 nicvf_dev_close(struct rte_eth_dev *dev)
1855 {
1856         size_t i;
1857         struct nicvf *nic = nicvf_pmd_priv(dev);
1858
1859         PMD_INIT_FUNC_TRACE();
1860         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1861                 return 0;
1862
1863         nicvf_dev_stop_cleanup(dev, true);
1864         nicvf_periodic_alarm_stop(nicvf_interrupt, dev);
1865
1866         for (i = 0; i < nic->sqs_count; i++) {
1867                 if (!nic->snicvf[i])
1868                         continue;
1869
1870                 nicvf_periodic_alarm_stop(nicvf_vf_interrupt, nic->snicvf[i]);
1871         }
1872
1873         return 0;
1874 }
1875
1876 static int
1877 nicvf_request_sqs(struct nicvf *nic)
1878 {
1879         size_t i;
1880
1881         assert_primary(nic);
1882         assert(nic->sqs_count > 0);
1883         assert(nic->sqs_count <= MAX_SQS_PER_VF);
1884
1885         /* Set no of Rx/Tx queues in each of the SQsets */
1886         for (i = 0; i < nic->sqs_count; i++) {
1887                 if (nicvf_svf_empty())
1888                         rte_panic("Cannot assign sufficient number of "
1889                                   "secondary queues to primary VF%" PRIu8 "\n",
1890                                   nic->vf_id);
1891
1892                 nic->snicvf[i] = nicvf_svf_pop();
1893                 nic->snicvf[i]->sqs_id = i;
1894         }
1895
1896         return nicvf_mbox_request_sqs(nic);
1897 }
1898
1899 static int
1900 nicvf_dev_configure(struct rte_eth_dev *dev)
1901 {
1902         struct rte_eth_dev_data *data = dev->data;
1903         struct rte_eth_conf *conf = &data->dev_conf;
1904         struct rte_eth_rxmode *rxmode = &conf->rxmode;
1905         struct rte_eth_txmode *txmode = &conf->txmode;
1906         struct nicvf *nic = nicvf_pmd_priv(dev);
1907         uint8_t cqcount;
1908
1909         PMD_INIT_FUNC_TRACE();
1910
1911         if (rxmode->mq_mode & ETH_MQ_RX_RSS_FLAG)
1912                 rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH;
1913
1914         if (!rte_eal_has_hugepages()) {
1915                 PMD_INIT_LOG(INFO, "Huge page is not configured");
1916                 return -EINVAL;
1917         }
1918
1919         if (txmode->mq_mode) {
1920                 PMD_INIT_LOG(INFO, "Tx mq_mode DCB or VMDq not supported");
1921                 return -EINVAL;
1922         }
1923
1924         if (rxmode->mq_mode != ETH_MQ_RX_NONE &&
1925                 rxmode->mq_mode != ETH_MQ_RX_RSS) {
1926                 PMD_INIT_LOG(INFO, "Unsupported rx qmode %d", rxmode->mq_mode);
1927                 return -EINVAL;
1928         }
1929
1930         if (rxmode->split_hdr_size) {
1931                 PMD_INIT_LOG(INFO, "Rxmode does not support split header");
1932                 return -EINVAL;
1933         }
1934
1935         if (conf->link_speeds & ETH_LINK_SPEED_FIXED) {
1936                 PMD_INIT_LOG(INFO, "Setting link speed/duplex not supported");
1937                 return -EINVAL;
1938         }
1939
1940         if (conf->dcb_capability_en) {
1941                 PMD_INIT_LOG(INFO, "DCB enable not supported");
1942                 return -EINVAL;
1943         }
1944
1945         if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1946                 PMD_INIT_LOG(INFO, "Flow director not supported");
1947                 return -EINVAL;
1948         }
1949
1950         assert_primary(nic);
1951         NICVF_STATIC_ASSERT(MAX_RCV_QUEUES_PER_QS == MAX_SND_QUEUES_PER_QS);
1952         cqcount = RTE_MAX(data->nb_tx_queues, data->nb_rx_queues);
1953         if (cqcount > MAX_RCV_QUEUES_PER_QS) {
1954                 nic->sqs_count = RTE_ALIGN_CEIL(cqcount, MAX_RCV_QUEUES_PER_QS);
1955                 nic->sqs_count = (nic->sqs_count / MAX_RCV_QUEUES_PER_QS) - 1;
1956         } else {
1957                 nic->sqs_count = 0;
1958         }
1959
1960         assert(nic->sqs_count <= MAX_SQS_PER_VF);
1961
1962         if (nic->sqs_count > 0) {
1963                 if (nicvf_request_sqs(nic)) {
1964                         rte_panic("Cannot assign sufficient number of "
1965                                   "secondary queues to PORT%d VF%" PRIu8 "\n",
1966                                   dev->data->port_id, nic->vf_id);
1967                 }
1968         }
1969
1970         if (rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM)
1971                 nic->offload_cksum = 1;
1972
1973         PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64,
1974                 dev->data->port_id, nicvf_hw_cap(nic));
1975
1976         return 0;
1977 }
1978
1979 static int
1980 nicvf_dev_set_link_up(struct rte_eth_dev *dev)
1981 {
1982         struct nicvf *nic = nicvf_pmd_priv(dev);
1983         int rc, i;
1984
1985         rc = nicvf_mbox_set_link_up_down(nic, true);
1986         if (rc)
1987                 goto done;
1988
1989         /* Start tx queues  */
1990         for (i = 0; i < dev->data->nb_tx_queues; i++)
1991                 nicvf_dev_tx_queue_start(dev, i);
1992
1993 done:
1994         return rc;
1995 }
1996
1997 static int
1998 nicvf_dev_set_link_down(struct rte_eth_dev *dev)
1999 {
2000         struct nicvf *nic = nicvf_pmd_priv(dev);
2001         int i;
2002
2003         /* Stop tx queues  */
2004         for (i = 0; i < dev->data->nb_tx_queues; i++)
2005                 nicvf_dev_tx_queue_stop(dev, i);
2006
2007         return nicvf_mbox_set_link_up_down(nic, false);
2008 }
2009
2010 /* Initialize and register driver with DPDK Application */
2011 static const struct eth_dev_ops nicvf_eth_dev_ops = {
2012         .dev_configure            = nicvf_dev_configure,
2013         .dev_start                = nicvf_dev_start,
2014         .dev_stop                 = nicvf_dev_stop,
2015         .link_update              = nicvf_dev_link_update,
2016         .dev_close                = nicvf_dev_close,
2017         .stats_get                = nicvf_dev_stats_get,
2018         .stats_reset              = nicvf_dev_stats_reset,
2019         .promiscuous_enable       = nicvf_dev_promisc_enable,
2020         .dev_infos_get            = nicvf_dev_info_get,
2021         .dev_supported_ptypes_get = nicvf_dev_supported_ptypes_get,
2022         .mtu_set                  = nicvf_dev_set_mtu,
2023         .vlan_offload_set         = nicvf_vlan_offload_set,
2024         .reta_update              = nicvf_dev_reta_update,
2025         .reta_query               = nicvf_dev_reta_query,
2026         .rss_hash_update          = nicvf_dev_rss_hash_update,
2027         .rss_hash_conf_get        = nicvf_dev_rss_hash_conf_get,
2028         .rx_queue_start           = nicvf_dev_rx_queue_start,
2029         .rx_queue_stop            = nicvf_dev_rx_queue_stop,
2030         .tx_queue_start           = nicvf_dev_tx_queue_start,
2031         .tx_queue_stop            = nicvf_dev_tx_queue_stop,
2032         .rx_queue_setup           = nicvf_dev_rx_queue_setup,
2033         .rx_queue_release         = nicvf_dev_rx_queue_release,
2034         .tx_queue_setup           = nicvf_dev_tx_queue_setup,
2035         .tx_queue_release         = nicvf_dev_tx_queue_release,
2036         .dev_set_link_up          = nicvf_dev_set_link_up,
2037         .dev_set_link_down        = nicvf_dev_set_link_down,
2038         .get_reg                  = nicvf_dev_get_regs,
2039 };
2040
2041 static int
2042 nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask)
2043 {
2044         struct rte_eth_rxmode *rxmode;
2045         struct nicvf *nic = nicvf_pmd_priv(dev);
2046         rxmode = &dev->data->dev_conf.rxmode;
2047         if (mask & ETH_VLAN_STRIP_MASK) {
2048                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
2049                         nicvf_vlan_hw_strip(nic, true);
2050                 else
2051                         nicvf_vlan_hw_strip(nic, false);
2052         }
2053
2054         return 0;
2055 }
2056
2057 static int
2058 nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2059 {
2060         nicvf_vlan_offload_config(dev, mask);
2061
2062         return 0;
2063 }
2064
2065 static inline int
2066 nicvf_set_first_skip(struct rte_eth_dev *dev)
2067 {
2068         int bytes_to_skip = 0;
2069         int ret = 0;
2070         unsigned int i;
2071         struct rte_kvargs *kvlist;
2072         static const char *const skip[] = {
2073                 SKIP_DATA_BYTES,
2074                 NULL};
2075         struct nicvf *nic = nicvf_pmd_priv(dev);
2076
2077         if (!dev->device->devargs) {
2078                 nicvf_first_skip_config(nic, 0);
2079                 return ret;
2080         }
2081
2082         kvlist = rte_kvargs_parse(dev->device->devargs->args, skip);
2083         if (!kvlist)
2084                 return -EINVAL;
2085
2086         if (kvlist->count == 0)
2087                 goto exit;
2088
2089         for (i = 0; i != kvlist->count; ++i) {
2090                 const struct rte_kvargs_pair *pair = &kvlist->pairs[i];
2091
2092                 if (!strcmp(pair->key, SKIP_DATA_BYTES))
2093                         bytes_to_skip = atoi(pair->value);
2094         }
2095
2096         /*128 bytes amounts to one cache line*/
2097         if (bytes_to_skip >= 0 && bytes_to_skip < 128) {
2098                 if (!(bytes_to_skip % 8)) {
2099                         nicvf_first_skip_config(nic, (bytes_to_skip / 8));
2100                         nic->skip_bytes = bytes_to_skip;
2101                         goto kvlist_free;
2102                 } else {
2103                         PMD_INIT_LOG(ERR, "skip_data_bytes should be multiple of 8");
2104                         ret = -EINVAL;
2105                         goto exit;
2106                 }
2107         } else {
2108                 PMD_INIT_LOG(ERR, "skip_data_bytes should be less than 128");
2109                 ret = -EINVAL;
2110                 goto exit;
2111         }
2112 exit:
2113         nicvf_first_skip_config(nic, 0);
2114 kvlist_free:
2115         rte_kvargs_free(kvlist);
2116         return ret;
2117 }
2118 static int
2119 nicvf_eth_dev_uninit(struct rte_eth_dev *dev)
2120 {
2121         PMD_INIT_FUNC_TRACE();
2122         nicvf_dev_close(dev);
2123         return 0;
2124 }
2125 static int
2126 nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
2127 {
2128         int ret;
2129         struct rte_pci_device *pci_dev;
2130         struct nicvf *nic = nicvf_pmd_priv(eth_dev);
2131
2132         PMD_INIT_FUNC_TRACE();
2133
2134         eth_dev->dev_ops = &nicvf_eth_dev_ops;
2135         eth_dev->rx_queue_count = nicvf_dev_rx_queue_count;
2136
2137         /* For secondary processes, the primary has done all the work */
2138         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2139                 if (nic) {
2140                         /* Setup callbacks for secondary process */
2141                         nicvf_set_tx_function(eth_dev);
2142                         nicvf_set_rx_function(eth_dev);
2143                         return 0;
2144                 } else {
2145                         /* If nic == NULL than it is secondary function
2146                          * so ethdev need to be released by caller */
2147                         return ENOTSUP;
2148                 }
2149         }
2150
2151         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2152         rte_eth_copy_pci_info(eth_dev, pci_dev);
2153         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
2154
2155         nic->device_id = pci_dev->id.device_id;
2156         nic->vendor_id = pci_dev->id.vendor_id;
2157         nic->subsystem_device_id = pci_dev->id.subsystem_device_id;
2158         nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
2159
2160         PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) %u:%u:%u:%u",
2161                         pci_dev->id.vendor_id, pci_dev->id.device_id,
2162                         pci_dev->addr.domain, pci_dev->addr.bus,
2163                         pci_dev->addr.devid, pci_dev->addr.function);
2164
2165         nic->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
2166         if (!nic->reg_base) {
2167                 PMD_INIT_LOG(ERR, "Failed to map BAR0");
2168                 ret = -ENODEV;
2169                 goto fail;
2170         }
2171
2172         nicvf_disable_all_interrupts(nic);
2173
2174         ret = nicvf_periodic_alarm_start(nicvf_interrupt, eth_dev);
2175         if (ret) {
2176                 PMD_INIT_LOG(ERR, "Failed to start period alarm");
2177                 goto fail;
2178         }
2179
2180         ret = nicvf_mbox_check_pf_ready(nic);
2181         if (ret) {
2182                 PMD_INIT_LOG(ERR, "Failed to get ready message from PF");
2183                 goto alarm_fail;
2184         } else {
2185                 PMD_INIT_LOG(INFO,
2186                         "node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
2187                         nic->node, nic->vf_id,
2188                         nic->tns_mode == NIC_TNS_MODE ? "tns" : "tns-bypass",
2189                         nic->sqs_mode ? "true" : "false",
2190                         nic->loopback_supported ? "true" : "false"
2191                         );
2192         }
2193
2194         ret = nicvf_base_init(nic);
2195         if (ret) {
2196                 PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init");
2197                 goto malloc_fail;
2198         }
2199
2200         if (nic->sqs_mode) {
2201                 /* Push nic to stack of secondary vfs */
2202                 nicvf_svf_push(nic);
2203
2204                 /* Steal nic pointer from the device for further reuse */
2205                 eth_dev->data->dev_private = NULL;
2206
2207                 nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev);
2208                 ret = nicvf_periodic_alarm_start(nicvf_vf_interrupt, nic);
2209                 if (ret) {
2210                         PMD_INIT_LOG(ERR, "Failed to start period alarm");
2211                         goto fail;
2212                 }
2213
2214                 /* Detach port by returning positive error number */
2215                 return ENOTSUP;
2216         }
2217
2218         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
2219                                         RTE_ETHER_ADDR_LEN, 0);
2220         if (eth_dev->data->mac_addrs == NULL) {
2221                 PMD_INIT_LOG(ERR, "Failed to allocate memory for mac addr");
2222                 ret = -ENOMEM;
2223                 goto alarm_fail;
2224         }
2225         if (rte_is_zero_ether_addr((struct rte_ether_addr *)nic->mac_addr))
2226                 rte_eth_random_addr(&nic->mac_addr[0]);
2227
2228         rte_ether_addr_copy((struct rte_ether_addr *)nic->mac_addr,
2229                         &eth_dev->data->mac_addrs[0]);
2230
2231         ret = nicvf_mbox_set_mac_addr(nic, nic->mac_addr);
2232         if (ret) {
2233                 PMD_INIT_LOG(ERR, "Failed to set mac addr");
2234                 goto malloc_fail;
2235         }
2236
2237         ret = nicvf_set_first_skip(eth_dev);
2238         if (ret) {
2239                 PMD_INIT_LOG(ERR, "Failed to configure first skip");
2240                 goto malloc_fail;
2241         }
2242         PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=" RTE_ETHER_ADDR_PRT_FMT,
2243                 eth_dev->data->port_id, nic->vendor_id, nic->device_id,
2244                 nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],
2245                 nic->mac_addr[3], nic->mac_addr[4], nic->mac_addr[5]);
2246
2247         return 0;
2248
2249 malloc_fail:
2250         rte_free(eth_dev->data->mac_addrs);
2251         eth_dev->data->mac_addrs = NULL;
2252 alarm_fail:
2253         nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev);
2254 fail:
2255         return ret;
2256 }
2257
2258 static const struct rte_pci_id pci_id_nicvf_map[] = {
2259         {
2260                 .class_id = RTE_CLASS_ANY_ID,
2261                 .vendor_id = PCI_VENDOR_ID_CAVIUM,
2262                 .device_id = PCI_DEVICE_ID_THUNDERX_CN88XX_PASS1_NICVF,
2263                 .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2264                 .subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS1_NICVF,
2265         },
2266         {
2267                 .class_id = RTE_CLASS_ANY_ID,
2268                 .vendor_id = PCI_VENDOR_ID_CAVIUM,
2269                 .device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2270                 .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2271                 .subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF,
2272         },
2273         {
2274                 .class_id = RTE_CLASS_ANY_ID,
2275                 .vendor_id = PCI_VENDOR_ID_CAVIUM,
2276                 .device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2277                 .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2278                 .subsystem_device_id = PCI_SUB_DEVICE_ID_CN81XX_NICVF,
2279         },
2280         {
2281                 .class_id = RTE_CLASS_ANY_ID,
2282                 .vendor_id = PCI_VENDOR_ID_CAVIUM,
2283                 .device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2284                 .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2285                 .subsystem_device_id = PCI_SUB_DEVICE_ID_CN83XX_NICVF,
2286         },
2287         {
2288                 .vendor_id = 0,
2289         },
2290 };
2291
2292 static int nicvf_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2293         struct rte_pci_device *pci_dev)
2294 {
2295         return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct nicvf),
2296                 nicvf_eth_dev_init);
2297 }
2298
2299 static int nicvf_eth_pci_remove(struct rte_pci_device *pci_dev)
2300 {
2301         return rte_eth_dev_pci_generic_remove(pci_dev, nicvf_eth_dev_uninit);
2302 }
2303
2304 static struct rte_pci_driver rte_nicvf_pmd = {
2305         .id_table = pci_id_nicvf_map,
2306         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_KEEP_MAPPED_RES |
2307                         RTE_PCI_DRV_INTR_LSC,
2308         .probe = nicvf_eth_pci_probe,
2309         .remove = nicvf_eth_pci_remove,
2310 };
2311
2312 RTE_PMD_REGISTER_PCI(net_thunderx, rte_nicvf_pmd);
2313 RTE_PMD_REGISTER_PCI_TABLE(net_thunderx, pci_id_nicvf_map);
2314 RTE_PMD_REGISTER_KMOD_DEP(net_thunderx, "* igb_uio | uio_pci_generic | vfio-pci");
2315 RTE_PMD_REGISTER_PARAM_STRING(net_thunderx, SKIP_DATA_BYTES "=<int>");